Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions Lib/test/test__interpreters.py
Original file line number Diff line number Diff line change
Expand Up @@ -466,6 +466,39 @@ def test_still_running(self):
_interpreters.destroy(interp)
self.assertTrue(_interpreters.is_running(interp))

def test_concurrent_run_and_destroy(self):
# gh-127041: An interpreter could start running during destroy().
for _ in range(50):
interp = _interpreters.create()
barrier = threading.Barrier(2)

def runner():
barrier.wait()
try:
_interpreters.run_string(interp, "x = 1")
except _interpreters.InterpreterError:
pass

def destroyer():
barrier.wait()
try:
_interpreters.destroy(interp)
except _interpreters.InterpreterError:
pass

threads = [
threading.Thread(target=runner),
threading.Thread(target=destroyer),
]
for t in threads:
t.start()
for t in threads:
t.join()
try:
_interpreters.destroy(interp)
except _interpreters.InterpreterError:
pass


class CommonTests(TestBase):
def setUp(self):
Expand Down
22 changes: 22 additions & 0 deletions Python/crossinterp.c
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#include "pycore_import.h" // _PyImport_SetModule()
#include "pycore_initconfig.h" // _PyStatus_OK()
#include "pycore_namespace.h" // _PyNamespace_New()
#include "pycore_pystate.h" // _PyInterpreterGuard_TryAcquire()
#include "pycore_pythonrun.h" // _Py_SourceAsString()
#include "pycore_runtime.h" // _PyRuntime
#include "pycore_setobject.h" // _PySet_NextEntry()
Expand Down Expand Up @@ -2546,6 +2547,10 @@ struct xi_session {
// once the session exits. Do not access this directly; use
// _PyXI_Preserve() and _PyXI_GetPreserved() instead;
PyObject *_preserved;

// PEP 788 finalization guard to prevent the interpreter from being
// destroyed/finalized while this session is active.
PyInterpreterGuard guard;
};

_PyXI_session *
Expand Down Expand Up @@ -2606,6 +2611,7 @@ _enter_session(_PyXI_session *session, PyInterpreterState *interp)
.init_tstate = tstate,
.prev_tstate = prev,
.own_init_tstate = !same_interp,
.guard = session->guard,
};
}

Expand Down Expand Up @@ -2641,6 +2647,12 @@ _exit_session(_PyXI_session *session)
assert(!session->own_init_tstate);
}

// Release PEP 788 finalization guard after thread state cleanup & swap back.
if (session->guard.interp != NULL) {
_PyInterpreterGuard_Release(&session->guard);
session->guard.interp = NULL;
}

*session = (_PyXI_session){0};
}

Expand Down Expand Up @@ -2714,6 +2726,16 @@ _PyXI_Enter(_PyXI_session *session,
}
}

// Acquire PEP 788 finalization guard before switching or running.
// If finalization has already started, TryAcquire will fail safely.
if (_PyInterpreterGuard_TryAcquire(interp, &session->guard) < 0) {

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should try to use the public API (PyInterpreterGuard_FromCurrent) and store the result as a pointer. That would also remove the need for the extra has_guard field, because NULL could act as has_guard = 0.

@utam-1 utam-1 Nov 15, 2026 •

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I tried using the PyInterpreterGuard_FromCurrent() but that resulted in a deadlock in _PyThreadState_Attach, reason could be the behaviour of _enter_session ( Not super sure though).

Instead, since PyInterpreterGuard wraps PyInterpreterState *interp (NULL when zero-initialized), I removed has_guard by checking session->guard.interp != NULL while keeping _PyInterpreterGuard_TryAcquire before _enter_session().

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please determine the source of the deadlock. I think it's a bad idea to rely on guard internals here.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the Python and C stack trace (I symbolised it a little bit because it's easier to follow this way):

test_concurrent_run_and_destroy (test.test__interpreters.DestroyTests.test_concurrent_run_and_destroy) ...
=== C Stack Trace on Freed tstate->interp ===
  #01 _PyThreadState_CheckConsistency (Python\pystate.c:3371)
  #02 take_gil (Python\ceval_gil.c:411)
  #03 _PyEval_AcquireLock (Python\ceval_gil.c:593)
  #04 _PyThreadState_Attach (Python\pystate.c:2405)
  #05 _PyThreadState_Swap (Python\pystate.c:2801)
  #06 PyThreadState_Swap (Python\pystate.c:2810)
  #07 _enter_session (Python\crossinterp.c:2603)
  #08 _PyXI_Enter (Python\crossinterp.c:2731)
  #09 _run_in_interpreter (Modules\_interpretersmodule.c:676)
  #10 _interpreters_run_string_impl (Modules\_interpretersmodule.c:1199)
  #11 _interpreters_run_string (Modules\clinic\_interpretersmodule.c.h:583)

Assertion failed: !_PyMem_IsPtrFreed(tstate->interp), file Python\pystate.c, line 3371
Fatal Python error: Aborted

Current thread 0x00003e8c [Thread-1 (runner)] (most recent call first):
  File "Lib\test\test__interpreters.py", line 478 in runner
  File "Lib\threading.py", line 1161 in run
  File "Lib\threading.py", line 1219 in _bootstrap_inner
  File "Lib\threading.py", line 1181 in _bootstrap

Based on the further analysis I found two things related to public API:

  • It takes a void argument, that leads to it guarding the caller's interpreter not the sub-interpreter we require.
  • Calling it after _enter_session() leaves PyThreadState_Swap() unguarded—so a concurrent destroy() frees interp or marks the new thread state SHUTTING_DOWN, hanging in _PyThreadState_HangThread() inside take_gil().

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's an abort, not a deadlock. Can you show the diff?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm sorry for mixing it up, I'm sharing the diff:

diff --git a/Python/crossinterp.c b/Python/crossinterp.c
index b8dd5f11c2a..e998fb627da 100644
--- a/Python/crossinterp.c
+++ b/Python/crossinterp.c
@@ -2546,6 +2546,8 @@ struct xi_session {
     // once the session exits.  Do not access this directly; use
     // _PyXI_Preserve() and _PyXI_GetPreserved() instead;
     PyObject *_preserved;
+
+    PyInterpreterGuard *guard;
 };
 
 _PyXI_session *
@@ -2641,6 +2643,11 @@ _exit_session(_PyXI_session *session)
         assert(!session->own_init_tstate);
     }
 
+    if (session->guard != NULL) {
+        PyInterpreterGuard_Close(session->guard);
+        session->guard = NULL;
+    }
+
     *session = (_PyXI_session){0};
 }
 
@@ -2717,6 +2724,11 @@ _PyXI_Enter(_PyXI_session *session,
     // Switch to the requested interpreter (if necessary).
     _enter_session(session, interp);
     _PyXI_failure override = XI_FAILURE_INIT;
     override.code = _PyXI_ERR_UNCAUGHT_EXCEPTION;
 #ifndef NDEBUG
     tstate = _PyThreadState_GET();
 #endif
 
+    session->guard = PyInterpreterGuard_FromCurrent();
+    if (session->guard == NULL) {
+        goto error;
+    }
+
     // Ensure this thread owns __main__.
     if (_PyInterpreterState_SetRunningMain(interp) < 0) {

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, right, we need to attach to an interpreter to use PyInterpreterGuard_FromCurrent, so that won't work

But, I think the current approach is thread-unsafe as currently written, because there's nothing stopping an interpreter from being deleted during the call to _PyInterpreterGuard_TryAcquire. PEP 788 comes with interpreter views for this exact use-case, so let's try those. Something like this should work:

  1. Switch the PyInterpreterState *interp parameter of _PyXI_Enter to a PyInterpreterView *view.
  2. Call PyInterpreterGuard_FromView before trying to attach.
  3. Release the guard after attaching to the interpreter (we don't want to block finalization for all of the subinterpreter's execution; just the part where it can be potentially deleted by another thread).

if (sharedns != NULL) {
_destroy_sharedns(sharedns);
}
PyErr_SetString(PyExc_InterpreterError, "interpreter is being destroyed");
return -1;
}

// Switch to the requested interpreter (if necessary).
_enter_session(session, interp);
_PyXI_failure override = XI_FAILURE_INIT;
Expand Down
Loading