[pull] main from python:main

Doc/c-api/threads.rst

@@ -736,10 +736,10 @@ Low-level APIs
 .. c:function:: PyObject* PyThreadState_GetDict()
 
    Return a dictionary in which extensions can store thread-specific state
-   information.  Each extension should use a unique key to use to store state in
+   information.  Each extension should use a unique key to store a state in
    the dictionary.  It is okay to call this function when no :term:`thread state`
    is :term:`attached <attached thread state>`. If this function returns
-   ``NULL``, no exception has been raised and the caller should assume no
+   ``NULL`` and no exception has been raised, then the caller should assume no
    thread state is attached.
 
 

Include/internal/pycore_interp_structs.h

@@ -1001,7 +1001,7 @@ struct _is {
     struct ast_state ast;
     struct types_state types;
     struct callable_cache callable_cache;
-    PyObject *common_consts[NUM_COMMON_CONSTANTS];
+    _PyStackRef common_consts[NUM_COMMON_CONSTANTS];
     bool jit;
     bool compiling;
 

Include/internal/pycore_stackref.h

@@ -263,6 +263,18 @@ _PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber)
 }
 #define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__)
 
+static inline _PyStackRef
+_PyStackRef_DupImmortal(_PyStackRef ref, const char *filename, int linenumber)
+{
+    assert(!PyStackRef_IsError(ref));
+    assert(!PyStackRef_IsTaggedInt(ref));
+    assert(!PyStackRef_RefcountOnObject(ref));
+    PyObject *obj = _Py_stackref_get_object(ref);
+    assert(_Py_IsImmortal(obj));
+    return _Py_stackref_create(obj, Py_TAG_REFCNT, filename, linenumber);
+}
+#define PyStackRef_DupImmortal(REF) _PyStackRef_DupImmortal((REF), __FILE__, __LINE__)
+
 static inline void
 _PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct, const char *filename, int linenumber)
 {
@@ -633,6 +645,15 @@ PyStackRef_DUP(_PyStackRef ref)
 }
 #endif
 
+static inline _PyStackRef
+PyStackRef_DupImmortal(_PyStackRef ref)
+{
+    assert(!PyStackRef_IsNull(ref));
+    assert(!PyStackRef_RefcountOnObject(ref));
+    assert(_Py_IsImmortal(BITS_TO_PTR_MASKED(ref)));
+    return ref;
+}
+
 static inline bool
 PyStackRef_IsHeapSafe(_PyStackRef ref)
 {

InternalDocs/qsbr.md

@@ -101,15 +101,39 @@ Periodically, a polling mechanism processes this deferred-free list:
 
 To reduce memory contention from frequent updates to the global `wr_seq`, its
 advancement is sometimes deferred. Instead of incrementing `wr_seq` on every
-reclamation request, each thread tracks its number of deferrals locally. Once
-the deferral count reaches a limit (QSBR_DEFERRED_LIMIT, currently 10), the
-thread advances the global `wr_seq` and resets its local count.
-
-When an object is added to the deferred-free list, its qsbr_goal is set to
-`wr_seq` + 2. By setting the goal to the next sequence value, we ensure it's safe
-to defer the global counter advancement. This optimization improves runtime
-speed but may increase peak memory usage by slightly delaying when memory can
-be reclaimed.
+reclamation request, the object's qsbr_goal is set to `wr_seq` + 2 (the value
+the counter *would* take on its next advance) without actually advancing the
+global counter. This is safe because the goal still corresponds to a future
+sequence value that no thread has yet observed as quiescent.
+
+Whether to actually advance `wr_seq` is decided per request, based on how
+much memory and how many items the calling thread has already deferred since
+its last advance:
+
+* For deferred object frees (`_PyMem_FreeDelayed`), the thread tracks both a
+  count (`deferred_count`) and an estimate of the held memory
+  (`deferred_memory`). The global `wr_seq` is advanced when the freed block
+  is larger than `QSBR_FREE_MEM_LIMIT` (1 MiB), when the accumulated deferred
+  memory exceeds that limit, or when the count exceeds `QSBR_DEFERRED_LIMIT`
+  (127, sized so a chunk of work items is processed before it overflows).
+  Crossing any of these thresholds also sets a per-thread `should_process`
+  flag, signalling that the deferred-free list should be drained.
+
+* For mimalloc pages held by QSBR, the thread tracks `deferred_page_memory`
+  and advances `wr_seq` when either the individual page or the accumulated
+  page memory exceeds `QSBR_PAGE_MEM_LIMIT` (4096 * 20 bytes). Advancing
+  promptly here matters because a held page cannot be reused for a different
+  size class or by a different thread.
+
+Processing of the deferred-free list normally happens from the eval breaker
+(rather than from inside `_PyMem_FreeDelayed`), which gives the global
+`rd_seq` a better chance to have advanced far enough that items can actually
+be freed. `_PyMem_ProcessDelayed` is still called from the free path as a
+safety valve when a work-item chunk fills up.
+
+This optimization improves runtime speed but may increase peak memory usage
+by slightly delaying when memory can be reclaimed; the size-based thresholds
+above bound that extra memory.
 
 
 ## Limitations

Lib/test/support/__init__.py

@@ -73,6 +73,7 @@
     "run_no_yield_async_fn", "run_yielding_async_fn", "async_yield",
     "reset_code", "on_github_actions",
     "requires_root_user", "requires_non_root_user",
+    "skip_if_double_rounding",
     ]
 
 
@@ -514,6 +515,15 @@ def dec(*args, **kwargs):
     float.__getformat__("double").startswith("IEEE"),
     "test requires IEEE 754 doubles")
 
+# detect evidence of double-rounding:
+x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer
+HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4)
+skip_if_double_rounding = unittest.skipIf(HAVE_DOUBLE_ROUNDING,
+                                          "accuracy not guaranteed on "
+                                          "machines with double rounding")
+del x, y, HAVE_DOUBLE_ROUNDING
+
+
 def requires_zlib(reason='requires zlib'):
     try:
         import zlib

Lib/test/test_builtin.py

@@ -39,19 +39,14 @@
 from test.support.script_helper import assert_python_ok
 from test.support.testcase import ComplexesAreIdenticalMixin
 from test.support.warnings_helper import check_warnings
-from test.support import requires_IEEE_754
+from test.support import requires_IEEE_754, skip_if_double_rounding
 from unittest.mock import MagicMock, patch
 try:
     import pty, signal
 except ImportError:
     pty = signal = None
 
 
-# Detect evidence of double-rounding: sum() does not always
-# get improved accuracy on machines that suffer from double rounding.
-x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer
-HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4)
-
 # used as proof of globals being used
 A_GLOBAL_VALUE = 123
 A_SENTINEL = sentinel("A_SENTINEL")
@@ -2235,8 +2230,7 @@ def __getitem__(self, index):
                                          complex(2, -0.0))
 
     @requires_IEEE_754
-    @unittest.skipIf(HAVE_DOUBLE_ROUNDING,
-                         "sum accuracy not guaranteed on machines with double rounding")
+    @skip_if_double_rounding
     @support.cpython_only    # Other implementations may choose a different algorithm
     def test_sum_accuracy(self):
         self.assertEqual(sum([0.1] * 10), 1.0)

Lib/test/test_math.py

@@ -1,7 +1,8 @@
 # Python test set -- math module
 # XXXX Should not do tests around zero only
 
-from test.support import verbose, requires_IEEE_754
+from test.support import (verbose, requires_IEEE_754,
+                          skip_if_double_rounding)
 from test import support
 import unittest
 import fractions
@@ -23,11 +24,6 @@
 FLOAT_MAX = sys.float_info.max
 FLOAT_MIN = sys.float_info.min
 
-# detect evidence of double-rounding: fsum is not always correctly
-# rounded on machines that suffer from double rounding.
-x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer
-HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4)
-
 # locate file with test values
 if __name__ == '__main__':
     file = sys.argv[0]
@@ -683,8 +679,7 @@ def testfrexp(name, result, expected):
         self.assertTrue(math.isnan(math.frexp(NAN)[0]))
 
     @requires_IEEE_754
-    @unittest.skipIf(HAVE_DOUBLE_ROUNDING,
-                         "fsum is not exact on machines with double rounding")
+    @skip_if_double_rounding
     def testFsum(self):
         # math.fsum relies on exact rounding for correct operation.
         # There's a known problem with IA32 floating-point that causes
@@ -920,8 +915,7 @@ def testHypot(self):
         self.assertRaises(TypeError, math.hypot, *([1.0]*18), 'spam')
 
     @requires_IEEE_754
-    @unittest.skipIf(HAVE_DOUBLE_ROUNDING,
-                     "hypot() loses accuracy on machines with double rounding")
+    @skip_if_double_rounding
     @support.skip_on_newlib
     def testHypotAccuracy(self):
         # Verify improved accuracy in cases that were known to be inaccurate.
@@ -1412,8 +1406,7 @@ def __rmul__(self, other):
         self.assertEqual(sumprod(*args), 0.0)
 
     @requires_IEEE_754
-    @unittest.skipIf(HAVE_DOUBLE_ROUNDING,
-                         "sumprod() accuracy not guaranteed on machines with double rounding")
+    @skip_if_double_rounding
     @support.cpython_only    # Other implementations may choose a different algorithm
     def test_sumprod_accuracy(self):
         sumprod = math.sumprod
@@ -1498,8 +1491,7 @@ def run(func, *args):
                         )
 
     @requires_IEEE_754
-    @unittest.skipIf(HAVE_DOUBLE_ROUNDING,
-                         "sumprod() accuracy not guaranteed on machines with double rounding")
+    @skip_if_double_rounding
     @support.cpython_only    # Other implementations may choose a different algorithm
     @support.requires_resource('cpu')
     def test_sumprod_extended_precision_accuracy(self):

Lib/test/test_statistics.py

@@ -16,7 +16,8 @@
 import sys
 import unittest
 from test import support
-from test.support import import_helper, requires_IEEE_754, skip_on_newlib
+from test.support import (import_helper, requires_IEEE_754,
+                          skip_if_double_rounding, skip_on_newlib)
 
 from decimal import Decimal
 from fractions import Fraction
@@ -28,12 +29,6 @@
 
 # === Helper functions and class ===
 
-# Test copied from Lib/test/test_math.py
-# detect evidence of double-rounding: fsum is not always correctly
-# rounded on machines that suffer from double rounding.
-x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer
-HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4)
-
 def sign(x):
     """Return -1.0 for negatives, including -0.0, otherwise +1.0."""
     return math.copysign(1, x)
@@ -2796,8 +2791,7 @@ def test_sqrtprod_helper_function_fundamentals(self):
                 self.assertEqual(sign(actual), sign(expected))
 
     @requires_IEEE_754
-    @unittest.skipIf(HAVE_DOUBLE_ROUNDING,
-                     "accuracy not guaranteed on machines with double rounding")
+    @skip_if_double_rounding
     @support.cpython_only    # Allow for a weaker sumprod() implementation
     @skip_on_newlib
     def test_sqrtprod_helper_function_improved_accuracy(self):

Python/bytecodes.c

@@ -1974,7 +1974,7 @@ dummy_func(
         inst(LOAD_COMMON_CONSTANT, ( -- value)) {
             // Keep in sync with _common_constants in opcode.py
             assert(oparg < NUM_COMMON_CONSTANTS);
-            value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]);
+            value = PyStackRef_DupImmortal(tstate->interp->common_consts[oparg]);
         }
 
         inst(LOAD_BUILD_CLASS, ( -- bc)) {

Python/flowgraph.c

@@ -11,6 +11,7 @@
 #include "pycore_opcode_utils.h"
 #include "pycore_opcode_metadata.h" // OPCODE_HAS_ARG, etc
 #include "pycore_pystate.h"         // _PyInterpreterState_GET()
+#include "pycore_stackref.h"        // PyStackRef_AsPyObjectBorrow()
 
 #include <stdbool.h>
 
@@ -1330,7 +1331,8 @@ get_const_value(int opcode, int oparg, PyObject *co_consts)
     }
     if (opcode == LOAD_COMMON_CONSTANT) {
         assert(oparg < NUM_COMMON_CONSTANTS);
-        return Py_NewRef(_PyInterpreterState_GET()->common_consts[oparg]);
+        return PyStackRef_AsPyObjectBorrow(
+            _PyInterpreterState_GET()->common_consts[oparg]);
     }
 
     if (constant == NULL) {

Python/optimizer_bytecodes.c

@@ -2,6 +2,7 @@
 #include "pycore_long.h"
 #include "pycore_opcode_utils.h"
 #include "pycore_optimizer.h"
+#include "pycore_stackref.h"
 #include "pycore_typeobject.h"
 #include "pycore_uops.h"
 #include "pycore_uop_ids.h"
@@ -870,15 +871,11 @@ dummy_func(void) {
 
     op(_LOAD_COMMON_CONSTANT, (-- value)) {
         assert(oparg < NUM_COMMON_CONSTANTS);
-        PyObject *val = _PyInterpreterState_GET()->common_consts[oparg];
-        if (_Py_IsImmortal(val)) {
-            ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val);
-            value = PyJitRef_Borrow(sym_new_const(ctx, val));
-        }
-        else {
-            ADD_OP(_LOAD_CONST_INLINE, 0, (uintptr_t)val);
-            value = sym_new_const(ctx, val);
-        }
+        PyObject *val = PyStackRef_AsPyObjectBorrow(
+            _PyInterpreterState_GET()->common_consts[oparg]);
+        assert(_Py_IsImmortal(val));
+        ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val);
+        value = PyJitRef_Borrow(sym_new_const(ctx, val));
     }
 
     op(_LOAD_SMALL_INT, (-- value)) {