Skip to content

Commit 5e9f1dc

Browse files
committed
gh-115103: Implement delayed free mechanism for free-threaded builds
This adds `_PyMem_FreeDelayed()` and supporting functions. The `_PyMem_FreeDelayed()` function frees memory with the same allocator as `PyMem_Free()`, but after some delay to ensure that concurrent lock-free readers have finished.
1 parent 5903190 commit 5e9f1dc

File tree

8 files changed

+210
-0
lines changed

8 files changed

+210
-0
lines changed

Include/internal/pycore_interp.h

+1
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,7 @@ struct _is {
231231

232232
struct _Py_dict_state dict_state;
233233
struct _Py_exc_state exc_state;
234+
struct _Py_mem_interp_free_queue mem_free_queue;
234235

235236
struct ast_state ast;
236237
struct types_state types;

Include/internal/pycore_pymem.h

+19
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#ifndef Py_INTERNAL_PYMEM_H
22
#define Py_INTERNAL_PYMEM_H
33

4+
#include "pycore_llist.h" // struct llist_node
45
#include "pycore_lock.h" // PyMutex
56

67
#ifdef __cplusplus
@@ -48,6 +49,11 @@ struct _pymem_allocators {
4849
PyObjectArenaAllocator obj_arena;
4950
};
5051

52+
struct _Py_mem_interp_free_queue {
53+
int has_work; // true if the queue is not empty
54+
PyMutex mutex; // protects the queue
55+
struct llist_node head; // queue of _mem_work_chunk items
56+
};
5157

5258
/* Set the memory allocator of the specified domain to the default.
5359
Save the old allocator into *old_alloc if it's non-NULL.
@@ -110,6 +116,19 @@ extern int _PyMem_SetupAllocators(PyMemAllocatorName allocator);
110116
/* Is the debug allocator enabled? */
111117
extern int _PyMem_DebugEnabled(void);
112118

119+
// Enqueue a pointer to be freed possibly after some delay.
120+
extern void _PyMem_FreeDelayed(void *ptr);
121+
122+
// Periodically process delayed free requests.
123+
extern void _PyMem_ProcessDelayed(PyThreadState *tstate);
124+
125+
// Abandon all thread-local delayed free requests and push them to the
126+
// interpreter's queue.
127+
extern void _PyMem_AbandonDelayed(PyThreadState *tstate);
128+
129+
// On interpreter shutdown, frees all delayed free requests.
130+
extern void _PyMem_FiniDelayed(PyInterpreterState *interp);
131+
113132
#ifdef __cplusplus
114133
}
115134
#endif

Include/internal/pycore_pymem_init.h

+5
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,11 @@ extern void _PyMem_ArenaFree(void *, void *, size_t);
9292
{ NULL, _PyMem_ArenaAlloc, _PyMem_ArenaFree }
9393

9494

95+
#define _Py_mem_free_queue_INIT(queue) \
96+
{ \
97+
.head = LLIST_INIT(queue.head), \
98+
}
99+
95100
#ifdef __cplusplus
96101
}
97102
#endif

Include/internal/pycore_runtime_init.h

+1
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,7 @@ extern PyTypeObject _PyExc_MemoryError;
176176
}, \
177177
.dtoa = _dtoa_state_INIT(&(INTERP)), \
178178
.dict_state = _dict_state_INIT, \
179+
.mem_free_queue = _Py_mem_free_queue_INIT(INTERP.mem_free_queue), \
179180
.func_state = { \
180181
.next_version = 1, \
181182
}, \

Include/internal/pycore_tstate.h

+1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ typedef struct _PyThreadStateImpl {
2929
PyThreadState base;
3030

3131
struct _qsbr_thread_state *qsbr; // only used by free-threaded build
32+
struct llist_node mem_free_queue; // delayed free queue
3233

3334
#ifdef Py_GIL_DISABLED
3435
struct _gc_thread_state gc;

Objects/obmalloc.c

+174
Original file line numberDiff line numberDiff line change
@@ -948,6 +948,180 @@ _PyMem_Strdup(const char *str)
948948
return copy;
949949
}
950950

951+
/***********************************************/
952+
/* Delayed freeing support for Py_GIL_DISABLED */
953+
/***********************************************/
954+
955+
// So that sizeof(struct _mem_work_chunk) is 4096 bytes on 64-bit platforms.
956+
#define WORK_ITEMS_PER_CHUNK 254
957+
958+
// A pointer to be freed once the QSBR read sequence reaches qsbr_goal.
959+
struct _mem_work_item {
960+
void *ptr;
961+
uint64_t qsbr_goal;
962+
};
963+
964+
// A fixed-size buffer of pointers to be freed
965+
struct _mem_work_chunk {
966+
// Linked list node of chunks in queue
967+
struct llist_node node;
968+
969+
Py_ssize_t rd_idx; // index of next item to read
970+
Py_ssize_t wr_idx; // index of next item to write
971+
struct _mem_work_item array[WORK_ITEMS_PER_CHUNK];
972+
};
973+
974+
void
975+
_PyMem_FreeDelayed(void *ptr)
976+
{
977+
#ifndef Py_GIL_DISABLED
978+
PyMem_Free(ptr);
979+
#else
980+
if (_PyRuntime.stoptheworld.world_stopped) {
981+
// Free immediately if the world is stopped, including during
982+
// interpreter shutdown.
983+
PyMem_Free(ptr);
984+
return;
985+
}
986+
987+
_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
988+
struct llist_node *head = &tstate->mem_free_queue;
989+
990+
struct _mem_work_chunk *buf = NULL;
991+
if (!llist_empty(head)) {
992+
// Try to re-use the last buffer
993+
buf = llist_data(head->prev, struct _mem_work_chunk, node);
994+
if (buf->wr_idx == WORK_ITEMS_PER_CHUNK) {
995+
// already full
996+
buf = NULL;
997+
}
998+
}
999+
1000+
if (buf == NULL) {
1001+
buf = PyMem_Calloc(1, sizeof(*buf));
1002+
if (buf != NULL) {
1003+
llist_insert_tail(head, &buf->node);
1004+
}
1005+
}
1006+
1007+
if (buf == NULL) {
1008+
// failed to allocate a buffer, free immediately
1009+
_PyEval_StopTheWorld(tstate->base.interp);
1010+
PyMem_Free(ptr);
1011+
_PyEval_StartTheWorld(tstate->base.interp);
1012+
return;
1013+
}
1014+
1015+
assert(buf != NULL && buf->wr_idx < WORK_ITEMS_PER_CHUNK);
1016+
uint64_t seq = _Py_qsbr_deferred_advance(tstate->qsbr);
1017+
buf->array[buf->wr_idx].ptr = ptr;
1018+
buf->array[buf->wr_idx].qsbr_goal = seq;
1019+
buf->wr_idx++;
1020+
1021+
if (buf->wr_idx == WORK_ITEMS_PER_CHUNK) {
1022+
_PyMem_ProcessDelayed((PyThreadState *)tstate);
1023+
}
1024+
#endif
1025+
}
1026+
1027+
static struct _mem_work_chunk *
1028+
work_queue_first(struct llist_node *head)
1029+
{
1030+
return llist_data(head->next, struct _mem_work_chunk, node);
1031+
}
1032+
1033+
static void
1034+
process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr)
1035+
{
1036+
while (!llist_empty(head)) {
1037+
struct _mem_work_chunk *buf = work_queue_first(head);
1038+
1039+
if (buf->rd_idx == buf->wr_idx) {
1040+
llist_remove(&buf->node);
1041+
PyMem_Free(buf);
1042+
continue;
1043+
}
1044+
1045+
struct _mem_work_item *item = &buf->array[buf->rd_idx];
1046+
if (!_Py_qsbr_poll(qsbr, item->qsbr_goal)) {
1047+
return;
1048+
}
1049+
1050+
PyMem_Free(item->ptr);
1051+
buf->rd_idx++;
1052+
}
1053+
}
1054+
1055+
static void
1056+
process_interp_queue(struct _Py_mem_interp_free_queue *queue,
1057+
struct _qsbr_thread_state *qsbr)
1058+
{
1059+
if (!_Py_atomic_load_int_relaxed(&queue->has_work)) {
1060+
return;
1061+
}
1062+
1063+
// Try to acquire the lock, but don't block if it's already held.
1064+
if (_PyMutex_LockTimed(&queue->mutex, 0, 0) == PY_LOCK_ACQUIRED) {
1065+
process_queue(&queue->head, qsbr);
1066+
1067+
int more_work = !llist_empty(&queue->head);
1068+
_Py_atomic_store_int_relaxed(&queue->has_work, more_work);
1069+
1070+
PyMutex_Unlock(&queue->mutex);
1071+
}
1072+
}
1073+
1074+
void
1075+
_PyMem_ProcessDelayed(PyThreadState *tstate)
1076+
{
1077+
PyInterpreterState *interp = tstate->interp;
1078+
_PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate;
1079+
1080+
// Process thread-local work
1081+
process_queue(&tstate_impl->mem_free_queue, tstate_impl->qsbr);
1082+
1083+
// Process shared interpreter work
1084+
process_interp_queue(&interp->mem_free_queue, tstate_impl->qsbr);
1085+
}
1086+
1087+
void
1088+
_PyMem_AbandonDelayed(PyThreadState *tstate)
1089+
{
1090+
PyInterpreterState *interp = tstate->interp;
1091+
struct llist_node *queue = &((_PyThreadStateImpl *)tstate)->mem_free_queue;
1092+
1093+
if (llist_empty(queue)) {
1094+
return;
1095+
}
1096+
1097+
// Merge the thread's work queue into the interpreter's work queue.
1098+
PyMutex_Lock(&interp->mem_free_queue.mutex);
1099+
llist_concat(&interp->mem_free_queue.head, queue);
1100+
_Py_atomic_store_int_relaxed(&interp->mem_free_queue.has_work, 1);
1101+
PyMutex_Unlock(&interp->mem_free_queue.mutex);
1102+
1103+
assert(llist_empty(queue)); // the thread's queue is now empty
1104+
}
1105+
1106+
void
1107+
_PyMem_FiniDelayed(PyInterpreterState *interp)
1108+
{
1109+
struct llist_node *head = &interp->mem_free_queue.head;
1110+
while (!llist_empty(head)) {
1111+
struct _mem_work_chunk *buf = work_queue_first(head);
1112+
1113+
while (buf->rd_idx < buf->wr_idx) {
1114+
// Free the remaining items immediately. There should be no other
1115+
// threads accessing the memory at this point during shutdown.
1116+
struct _mem_work_item *item = &buf->array[buf->rd_idx];
1117+
PyMem_Free(item->ptr);
1118+
buf->rd_idx++;
1119+
}
1120+
1121+
llist_remove(&buf->node);
1122+
PyMem_Free(buf);
1123+
}
1124+
}
9511125

9521126
/**************************/
9531127
/* the "object" allocator */

Python/pylifecycle.c

+3
Original file line numberDiff line numberDiff line change
@@ -1834,6 +1834,9 @@ finalize_interp_clear(PyThreadState *tstate)
18341834

18351835
finalize_interp_types(tstate->interp);
18361836

1837+
/* Free any delayed free requests immediately */
1838+
_PyMem_FiniDelayed(tstate->interp);
1839+
18371840
/* finalize_interp_types may allocate Python objects so we may need to
18381841
abandon mimalloc segments again */
18391842
_PyThreadState_ClearMimallocHeaps(tstate);

Python/pystate.c

+6
Original file line numberDiff line numberDiff line change
@@ -617,6 +617,7 @@ init_interpreter(PyInterpreterState *interp,
617617
#ifdef Py_GIL_DISABLED
618618
_Py_brc_init_state(interp);
619619
#endif
620+
llist_init(&interp->mem_free_queue.head);
620621
for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) {
621622
interp->monitors.tools[i] = 0;
622623
}
@@ -1351,6 +1352,7 @@ init_threadstate(_PyThreadStateImpl *_tstate,
13511352
// Initialize biased reference counting inter-thread queue
13521353
_Py_brc_init_thread(tstate);
13531354
#endif
1355+
llist_init(&_tstate->mem_free_queue);
13541356

13551357
if (interp->stoptheworld.requested || _PyRuntime.stoptheworld.requested) {
13561358
// Start in the suspended state if there is an ongoing stop-the-world.
@@ -1572,6 +1574,7 @@ PyThreadState_Clear(PyThreadState *tstate)
15721574
// don't call _PyInterpreterState_SetNotRunningMain() yet.
15731575
tstate->on_delete(tstate->on_delete_data);
15741576
}
1577+
15751578
#ifdef Py_GIL_DISABLED
15761579
// Each thread should clear own freelists in free-threading builds.
15771580
struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
@@ -1581,6 +1584,9 @@ PyThreadState_Clear(PyThreadState *tstate)
15811584
_Py_brc_remove_thread(tstate);
15821585
#endif
15831586

1587+
// Merge our queue of pointers to be freed into the interpreter queue.
1588+
_PyMem_AbandonDelayed(tstate);
1589+
15841590
_PyThreadState_ClearMimallocHeaps(tstate);
15851591

15861592
tstate->_status.cleared = 1;

0 commit comments

Comments
 (0)