Tracing region

Include/internal/pycore_cown.h

@@ -0,0 +1,29 @@
+#ifndef Py_INTERNAL_COWN_H
+#define Py_INTERNAL_COWN_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "Py_BUILD_CORE must be defined to include this header"
+#endif
+
+#include "object.h"
+#include "exports.h"
+
+typedef struct _PyCownObject _PyCownObject;
+#define _PyCownObject_CAST(op) _Py_CAST(_PyCownObject*, op)
+
+PyAPI_DATA(PyTypeObject) _PyCown_Type;
+
+typedef uint64_t _PyCown_ipid_t;
+typedef uint64_t _PyCown_thread_id_t;
+
+PyAPI_FUNC(_PyCown_ipid_t) _PyCown_ThisInterpreterId(void);
+PyAPI_FUNC(_PyCown_thread_id_t) _PyCown_ThisThreadId(void);
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_COWN_H */

Include/internal/pycore_gc.h

@@ -352,6 +352,25 @@ extern PyObject *_PyGC_GetObjects(PyInterpreterState *interp, int generation);
 extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs);
 
 // Functions to clear types free lists
+/* Disposal of a list of objects that are known to be unreachable. Used by the
+ * collector itself and by anything else that owns a set of objects it has
+ * established to be garbage, such as a closed tracing region.
+ *
+ * `_PyGC_FinalizeGarbage()` runs the finalizer of every object in `collectable`,
+ * before anything is cleared, so that a `__del__` still sees its object intact.
+ *
+ * `_PyGC_DeleteGarbage()` then breaks the references between them, deallocating
+ * every object whose reference count reaches zero. Objects that a finalizer kept
+ * alive are moved to `old` instead.
+ *
+ * Neither may be called with an exception set. Only available in the default
+ * build; the free-threaded collector has its own implementation.
+ */
+#ifndef Py_GIL_DISABLED
+extern void _PyGC_FinalizeGarbage(PyGC_Head *collectable);
+extern void _PyGC_DeleteGarbage(PyGC_Head *collectable, PyGC_Head *old);
+#endif
+
 extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp);
 extern void _Py_ScheduleGC(PyThreadState *tstate);
 extern void _Py_RunGC(PyThreadState *tstate);

Include/internal/pycore_immutability.h

@@ -8,6 +8,10 @@ extern "C" {
 #  error "Py_BUILD_CORE must be defined to include this header"
 #endif
 
+PyAPI_DATA(PyTypeObject) _PyTracingRegion_Type;
+PyAPI_FUNC(int) _PyTracingRegion_Close(PyObject* region);
+PyAPI_FUNC(int) _PyTracingRegion_IsClosed(PyObject* region);
+
 struct _Py_immutability_state {
     int late_init_done;
     struct _Py_hashtable_t *shallow_immutable_types;

Lib/immutable.py

@@ -21,6 +21,8 @@
 FREEZABLE_PROXY = _c.FREEZABLE_PROXY
 InterpreterLocal = _c.InterpreterLocal
 SharedField = _c.SharedField
+TracingRegion = _c.TracingRegion
+Cown = _c.Cown
 
 # FIXME(immutable): For the longest time we used the name `isfrozen`
 # without the underscore. This keeps the function name for now, but

Lib/test/test_freeze/test_implicit.py

@@ -1,3 +1,4 @@
+import sys
 import unittest
 from immutable import freeze, is_frozen
 
@@ -139,6 +140,26 @@ def test_deeply_nested_no_stack_overflow(self):
             obj = (obj,)
         self.assertTrue(is_frozen(obj))
 
+    def test_abandoned_walk_keeps_references(self):
+        """An aborted walk must not drop references it never took.
+
+        The walk pushes objects onto a worklist without increfing them, so
+        anything still on the worklist when a mutable object aborts the walk
+        used to be decrefed when the worklist was released. That freed the
+        object while its real owners were still pointing at it, which showed
+        up much later as a negative refcount.
+        """
+        # Built at runtime so it is neither interned nor immortal, which makes
+        # its reference count fully accounted for by this test.
+        item = "".join(["abandoned", "-", "worklist", "-", "entry"])
+        # Tuples are traversed back to front, so `item` reaches the worklist
+        # before the dict aborts the walk.
+        obj = ({"mutable": 1}, item)
+
+        before = sys.getrefcount(item)
+        self.assertFalse(is_frozen(obj))
+        self.assertEqual(sys.getrefcount(item), before)
+
 
 if __name__ == '__main__':
     unittest.main()

Lib/test/test_freeze/test_tracing_region.py

@@ -0,0 +1,253 @@
+import re
+import sys
+import unittest
+from immutable import freeze, is_frozen, freezable
+from immutable import TracingRegion as Region
+from immutable import Cown
+
+def sort_region_error(msg):
+    """Normalize a 'region could not be closed' message by masking the object
+    addresses and sorting its per-object lines. Useful for deterministic test
+    assertions, since the addresses differ per run and the object order comes
+    from hashtable iteration and isn't stable."""
+    header, *lines = re.sub(r"0x[0-9a-fA-F]+", "0x...", msg).splitlines()
+    return [header, *sorted(lines)]
+
+class TestTraceRefs(unittest.TestCase):
+    def test_release_error(self):
+        x = [1]
+        y = [2]
+
+        c = Cown(Region())
+        c.value.x = x
+        c.value.y = y
+
+        with self.assertRaises(RuntimeError) as cm:
+            c.release()
+
+        self.assertEqual(
+            sort_region_error(str(cm.exception)),
+            [
+                "The region could not be closed due to:",
+                "- 1 incoming reference to list '[1]'",
+                "- 1 incoming reference to list '[2]'"
+            ])
+
+    def test_release_error_capped_output(self):
+        # The object order in the error message is based on the address
+        # and therefore fairly random. All elements look the same of
+        # make testing stable.
+        l = [[1], [1], [1], [1], [1], [1], [1], [1]]
+
+        c = Cown(Region())
+        c.value.x = []
+
+        for i in range(len(l)):
+            c.value.x.append(l[i])
+
+        with self.assertRaises(RuntimeError) as cm:
+            c.release()
+
+        self.assertEqual(
+            sort_region_error(str(cm.exception)),
+            [
+                "The region could not be closed due to:",
+                "- 1 incoming reference to list '[1]'",
+                "- 1 incoming reference to list '[1]'",
+                "- 1 incoming reference to list '[1]'",
+                "- 1 incoming reference to list '[1]'",
+                "- 1 incoming reference to list '[1]'",
+                "- 3 references to other objects",
+            ])
+
+        # The cown should now be released
+        l = None
+        c.release()
+
+    def test_release_error_in_subregion(self):
+        x = [1]
+
+        c = Cown(Region())
+        child = Region()
+        child.x = x
+        c.value.child = child
+
+        with self.assertRaises(RuntimeError) as cm:
+            c.release()
+
+        self.assertEqual(
+            sort_region_error(str(cm.exception)),
+            [
+                "The region could not be closed due to:",
+                "- 1 incoming reference to list '[1]'",
+            ])
+
+
+class TestRegionOpening(unittest.TestCase):
+    def test_open_after_acquire(self):
+        c = Cown(Region())
+        c.value.x = []
+        self.assertFalse(c._is_closed())
+
+        c.release()
+        c.acquire()
+
+        self.assertTrue(c._is_closed())
+        c.value.x = None
+        self.assertFalse(c._is_closed())
+
+    def test_release_closed_region(self):
+        c = Cown(Region())
+        c.value.x = []
+        self.assertFalse(c._is_closed())
+
+        c.release()
+        c.acquire()
+
+        self.assertTrue(c._is_closed())
+
+        c.release()
+
+    def test_bridge_refs_keep_region_closed(self):
+        c = Cown(Region())
+        c.release()
+        c.acquire()
+        self.assertTrue(c._is_closed())
+
+        # Adding new references to the bridge object should keep it closed.
+        # only attribute accesses should open it.
+        r1 = c.value
+        r2 = c.value
+        self.assertTrue(c._is_closed())
+
+        # However, these references should prevent the cown from being released
+        with self.assertRaises(RuntimeError) as cm:
+            c.release()
+
+        self.assertEqual(
+            str(cm.exception),
+            "the cown couldn't be released, due to the bridge having incoming references")
+
+        # The release should succeed once all refs have been killed
+        del r1
+        del r2
+        c.release()
+
+    def test_sub_region_closing(self):
+        @freezable
+        class A:
+            pass
+        c = Cown(Region())
+        c.value.a = A()
+        c.value.a.child = Region()
+        c.value.a.child.b = A()
+
+        c.release()
+        c.acquire()
+
+        r2 = c.value.a.child
+        c2 = Cown(r2)
+
+        self.assertTrue(c2._is_closed())
+
+    def test_sub_region_multiple_refs(self):
+        @freezable
+        class A:
+            pass
+        c = Cown(Region())
+        c.value.a = A()
+        sub = Region()
+        c.value.a.child_a = sub
+        c.value.a.child_b = sub
+        # A reference to the bridge of a sub-region counts as an incoming
+        # reference into the parent region, see
+        # test_ref_to_sub_region_bridge_keeps_parent_open.
+        del sub
+
+        c.release()
+        c.acquire()
+
+        r2 = c.value.a.child_a
+        c2 = Cown(r2)
+
+        self.assertTrue(c2._is_closed())
+
+    def test_ref_to_sub_region_bridge_keeps_parent_open(self):
+        c1 = Cown(Region())
+        c2 = Cown(Region())
+        c1.value.child = c2.value
+
+        self.assertFalse(c2._is_closed())
+
+        with self.assertRaises(RuntimeError) as cm:
+            c1.release()
+
+        # Attempting to close the region c1 should have closed c2 and then
+        # failed due to the incoming reference to the bridge stored in c2
+        self.assertTrue(c2._is_closed())
+
+
+        self.assertEqual(
+            sort_region_error(str(cm.exception)),
+            [
+                "The region could not be closed due to:",
+                "- 1 incoming reference to TracingRegion '<TracingRegion closed>'",
+            ])
+
+
+
+class TestImplicitFreeze(unittest.TestCase):
+    def test_implicit_freeze_func(self):
+        @freezable
+        def some_func():
+            pass
+        c = Cown(Region())
+
+        c.value.obj = some_func
+        self.assertFalse(is_frozen(c.value.obj))
+        c.release()
+        c.acquire()
+        self.assertTrue(is_frozen(c.value.obj))
+
+    def test_implicit_freeze_type(self):
+        @freezable
+        class A:
+            pass
+        c = Cown(Region())
+
+        c.value.obj = A
+        self.assertFalse(is_frozen(c.value.obj))
+        c.release()
+        c.acquire()
+        self.assertTrue(is_frozen(c.value.obj))
+
+    def test_implicit_freeze_module(self):
+        import random;
+        c = Cown(Region())
+
+        c.value.obj = random
+        self.assertFalse(is_frozen(c.value.obj))
+        c.release()
+        c.acquire()
+        self.assertTrue(is_frozen(c.value.obj))
+
+        # Unimport module
+        sys.modules.pop("random", None)
+        sys.mut_modules.pop("random", None)
+
+    def test_implicit_freeze_str(self):
+        c = Cown(Region())
+
+        c.value.obj = "Ducks are cool"
+        c.release()
+        c.acquire()
+        self.assertTrue(is_frozen(c.value.obj))
+
+    def test_implicit_freeze_int(self):
+        c = Cown(Region())
+
+        c.value.obj = 17
+        c.release()
+        c.acquire()
+        self.assertTrue(is_frozen(c.value.obj))
+

Makefile.pre.in

@@ -528,6 +528,7 @@ OBJECT_OBJS=	\
 		Objects/classobject.o \
 		Objects/codeobject.o \
 		Objects/complexobject.o \
+		Objects/cownobject.o \
 		Objects/descrobject.o \
 		Objects/enumobject.o \
 		Objects/exceptions.o \
@@ -555,6 +556,7 @@ OBJECT_OBJS=	\
 		Objects/sliceobject.o \
 		Objects/structseq.o \
 		Objects/templateobject.o \
+		Objects/tracingregionobject.o \
 		Objects/tupleobject.o \
 		Objects/typeobject.o \
 		Objects/typevarobject.o \