Forward-port generational GC.

Co-Authored-By: Neil Schemenauer <nas@arctrix.com>
Co-Authored-By: Sergey Miryanov <sergey.miryanov@gmail.com>

Author: sergey-miryanov

Include/internal/pycore_gc.h

@@ -131,7 +131,6 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) {
  * When object are moved from the pending space, old[gcstate->visited_space^1]
  * into the increment, the old space bit is flipped.
 */
-#define _PyGC_NEXT_MASK_OLD_SPACE_1    1
 
 #define _PyGC_PREV_SHIFT           2
 #define _PyGC_PREV_MASK            (((uintptr_t) -1) << _PyGC_PREV_SHIFT)
@@ -159,13 +158,11 @@ typedef enum {
 // Lowest bit of _gc_next is used for flags only in GC.
 // But it is always 0 for normal code.
 static inline PyGC_Head* _PyGCHead_NEXT(PyGC_Head *gc) {
-    uintptr_t next = gc->_gc_next & _PyGC_PREV_MASK;
+    uintptr_t next = gc->_gc_next;
     return (PyGC_Head*)next;
 }
 static inline void _PyGCHead_SET_NEXT(PyGC_Head *gc, PyGC_Head *next) {
-    uintptr_t unext = (uintptr_t)next;
-    assert((unext & ~_PyGC_PREV_MASK) == 0);
-    gc->_gc_next = (gc->_gc_next & ~_PyGC_PREV_MASK) | unext;
+    gc->_gc_next = (uintptr_t)next;
 }
 
 // Lowest two bits of _gc_prev is used for _PyGC_PREV_MASK_* flags.
@@ -207,10 +204,6 @@ static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) {
 
 extern void _Py_ScheduleGC(PyThreadState *tstate);
 
-#ifndef Py_GIL_DISABLED
-extern void _Py_TriggerGC(struct _gc_runtime_state *gcstate);
-#endif
-
 
 /* Tell the GC to track this object.
  *
@@ -244,19 +237,12 @@ static inline void _PyObject_GC_TRACK(
                           "object is in generation which is garbage collected",
                           filename, lineno, __func__);
 
-    struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc;
-    PyGC_Head *generation0 = &gcstate->young.head;
+    PyGC_Head *generation0 = _PyInterpreterState_GET()->gc.generation0;
     PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev);
     _PyGCHead_SET_NEXT(last, gc);
     _PyGCHead_SET_PREV(gc, last);
-    uintptr_t not_visited = 1 ^ gcstate->visited_space;
-    gc->_gc_next = ((uintptr_t)generation0) | not_visited;
+    _PyGCHead_SET_NEXT(gc, generation0);
     generation0->_gc_prev = (uintptr_t)gc;
-    gcstate->young.count++; /* number of tracked GC objects */
-    gcstate->heap_size++;
-    if (gcstate->young.count > gcstate->young.threshold) {
-        _Py_TriggerGC(gcstate);
-    }
 #endif
 }
 
@@ -291,11 +277,6 @@ static inline void _PyObject_GC_UNTRACK(
     _PyGCHead_SET_PREV(next, prev);
     gc->_gc_next = 0;
     gc->_gc_prev &= _PyGC_PREV_MASK_FINALIZED;
-    struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc;
-    if (gcstate->young.count > 0) {
-        gcstate->young.count--;
-    }
-    gcstate->heap_size--;
 #endif
 }
 

Include/internal/pycore_interp_structs.h

@@ -215,8 +215,13 @@ struct _gc_runtime_state {
     int enabled;
     int debug;
     /* linked lists of container objects */
+#ifndef Py_GIL_DISABLED
+    struct gc_generation generations[NUM_GENERATIONS];
+    PyGC_Head *generation0;
+#else
     struct gc_generation young;
     struct gc_generation old[2];
+#endif
     /* a permanent generation which won't be collected */
     struct gc_generation permanent_generation;
     struct gc_generation_stats generation_stats[NUM_GENERATIONS];
@@ -227,13 +232,6 @@ struct _gc_runtime_state {
     /* a list of callbacks to be invoked when collection is performed */
     PyObject *callbacks;
 
-    Py_ssize_t heap_size;
-    Py_ssize_t work_to_do;
-    /* Which of the old spaces is the visited space */
-    int visited_space;
-    int phase;
-
-#ifdef Py_GIL_DISABLED
     /* This is the number of objects that survived the last full
        collection. It approximates the number of long lived objects
        tracked by the GC.
@@ -246,6 +244,7 @@ struct _gc_runtime_state {
        the first time. */
     Py_ssize_t long_lived_pending;
 
+#ifdef Py_GIL_DISABLED
     /* True if gc.freeze() has been used. */
     int freeze_active;
 
@@ -261,6 +260,22 @@ struct _gc_runtime_state {
 #endif
 };
 
+#ifndef Py_GIL_DISABLED
+#define GC_GENERATION_INIT \
+    .generations = { \
+        { .threshold = 2000, }, \
+        { .threshold = 10, }, \
+        { .threshold = 10, }, \
+    },
+#else
+#define GC_GENERATION_INIT \
+    .young = { .threshold = 2000, }, \
+    .old = { \
+        { .threshold = 10, }, \
+        { .threshold = 10, }, \
+    },
+#endif
+
 #include "pycore_gil.h"           // struct _gil_runtime_state
 
 /**** Import ********/

Include/internal/pycore_runtime_init.h

@@ -137,13 +137,7 @@ extern PyTypeObject _PyExc_MemoryError;
         }, \
         .gc = { \
             .enabled = 1, \
-            .young = { .threshold = 2000, }, \
-            .old = { \
-                { .threshold = 10, }, \
-                { .threshold = 0, }, \
-            }, \
-            .work_to_do = -5000, \
-            .phase = GC_PHASE_MARK, \
+            GC_GENERATION_INIT \
         }, \
         .qsbr = { \
             .wr_seq = QSBR_INITIAL, \

Modules/_testinternalcapi.c

@@ -2346,7 +2346,8 @@ has_deferred_refcount(PyObject *self, PyObject *op)
 static PyObject *
 get_tracked_heap_size(PyObject *self, PyObject *Py_UNUSED(ignored))
 {
-    return PyLong_FromInt64(PyInterpreterState_Get()->gc.heap_size);
+    // Generational GC doesn't track heap_size, return -1.
+    return PyLong_FromInt64(-1);
 }
 
 static PyObject *

Modules/gcmodule.c

@@ -159,13 +159,23 @@ gc_set_threshold_impl(PyObject *module, int threshold0, int group_right_1,
 {
     GCState *gcstate = get_gc_state();
 
+#ifndef Py_GIL_DISABLED
+    gcstate->generations[0].threshold = threshold0;
+    if (group_right_1) {
+        gcstate->generations[1].threshold = threshold1;
+    }
+    if (group_right_2) {
+        gcstate->generations[2].threshold = threshold2;
+    }
+#else
     gcstate->young.threshold = threshold0;
     if (group_right_1) {
         gcstate->old[0].threshold = threshold1;
     }
     if (group_right_2) {
         gcstate->old[1].threshold = threshold2;
     }
+#endif
     Py_RETURN_NONE;
 }
 
@@ -180,10 +190,17 @@ gc_get_threshold_impl(PyObject *module)
 /*[clinic end generated code: output=7902bc9f41ecbbd8 input=286d79918034d6e6]*/
 {
     GCState *gcstate = get_gc_state();
+#ifndef Py_GIL_DISABLED
+    return Py_BuildValue("(iii)",
+                         gcstate->generations[0].threshold,
+                         gcstate->generations[1].threshold,
+                         gcstate->generations[2].threshold);
+#else
     return Py_BuildValue("(iii)",
                          gcstate->young.threshold,
                          gcstate->old[0].threshold,
-                         0);
+                         gcstate->old[1].threshold);
+#endif
 }
 
 /*[clinic input]
@@ -207,10 +224,17 @@ gc_get_count_impl(PyObject *module)
     gc->alloc_count = 0;
 #endif
 
+#ifndef Py_GIL_DISABLED
+    return Py_BuildValue("(iii)",
+                         gcstate->generations[0].count,
+                         gcstate->generations[1].count,
+                         gcstate->generations[2].count);
+#else
     return Py_BuildValue("(iii)",
                          gcstate->young.count,
-                         gcstate->old[gcstate->visited_space].count,
-                         gcstate->old[gcstate->visited_space^1].count);
+                         gcstate->old[0].count,
+                         gcstate->old[1].count);
+#endif
 }
 
 /*[clinic input]