From 47eabce82c56432ebf37e25e14f6b4da7e6c662a Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 13 Nov 2026 09:50:41 -0500 Subject: [PATCH 1/2] fix: add smart_holder keep_alive patient before publishing the instance smart_holder_from_unique_ptr and smart_holder_from_shared_ptr called keep_alive_impl() after init_instance(). On free-threaded builds another thread could find the registered instance and read its flags while has_patients was written. The generic cast path already uses this order. Part of #6214. Assisted-by: ClaudeCode:claude-fable-5-2 --- include/pybind11/detail/type_caster_base.h | 10 ++++++---- tests/test_thread.cpp | 8 ++++++++ tests/test_thread.py | 8 +++++++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 4c7691ff8d..b4d3217bc5 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -450,12 +450,13 @@ handle smart_holder_from_unique_ptr(std::unique_ptr &&src, src_raw_void_ptr = nullptr; } auto smhldr = smart_holder::from_unique_ptr(std::move(src), src_raw_void_ptr); - tinfo->init_instance(inst_raw_ptr, static_cast(&smhldr)); - + // Before init_instance, which publishes the instance to other threads. if (policy == return_value_policy::reference_internal) { keep_alive_impl(inst, parent); } + tinfo->init_instance(inst_raw_ptr, static_cast(&smhldr)); + return inst.release(); } @@ -513,12 +514,13 @@ handle smart_holder_from_shared_ptr(const std::shared_ptr &src, valueptr = src_raw_void_ptr; auto smhldr = smart_holder::from_shared_ptr(std::shared_ptr(src, src_raw_void_ptr)); - tinfo->init_instance(inst_raw_ptr, static_cast(&smhldr)); - + // Before init_instance, which publishes the instance to other threads. if (policy == return_value_policy::reference_internal) { keep_alive_impl(inst, parent); } + tinfo->init_instance(inst_raw_ptr, static_cast(&smhldr)); + return inst.release(); } diff --git a/tests/test_thread.cpp b/tests/test_thread.cpp index fcef8bef0f..1e67dc4a5a 100644 --- a/tests/test_thread.cpp +++ b/tests/test_thread.cpp @@ -235,6 +235,14 @@ TEST_SUBMODULE(thread, m) { m.def("get_shared_object_classic", &get_shared); py::classh(m, "SharedObjectSh").def("value", &SharedObjectSh::value); m.def("get_shared_object_sh", &get_shared); + m.def( + "get_shared_object_sh_keep_alive", + // By reference: a by-value return has its policy replaced with `move`. + [](py::handle /*parent*/) -> const std::shared_ptr & { + static const auto instance = get_shared(); + return instance; + }, + py::return_value_policy::reference_internal); #if defined(PYBIND11_HAS_STD_BARRIER) // In the free-threaded build, during PyThreadState_Clear, removing the thread from the biased diff --git a/tests/test_thread.py b/tests/test_thread.py index f5a71a9e6a..16d7e45d21 100644 --- a/tests/test_thread.py +++ b/tests/test_thread.py @@ -75,7 +75,13 @@ def access_shared_instance(): @pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") @pytest.mark.parametrize( - "get_shared", [m.get_shared_object_classic, m.get_shared_object_sh] + "get_shared", + [ + m.get_shared_object_classic, + m.get_shared_object_sh, + # reference_internal adds a patient to the new wrapper. + lambda: m.get_shared_object_sh_keep_alive(m), + ], ) def test_concurrent_cast_of_shared_cpp_object(get_shared): # Only fails under ThreadSanitizer on free-threaded builds; otherwise the race is silent. From e18e2c447c73e8dec75b5ac39c718a5136fea3b6 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 21 Nov 2026 13:02:56 -0500 Subject: [PATCH 2/2] fix: activate smart_holder keep_alive before the instance owns the pointer keep_alive_impl() can throw (no parent). It ran after the wrapper had owned = true and the raw pointer, but before the holder was constructed, so the wrapper cleanup and the temporary smart_holder both deleted the object. Call it immediately after make_new_instance() instead. Assisted-by: ClaudeCode:claude-fable-5-2 --- include/pybind11/detail/type_caster_base.h | 20 ++++++++++---------- tests/test_class_sh_basic.cpp | 13 +++++++++++++ tests/test_class_sh_basic.py | 14 ++++++++++++++ 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index b4d3217bc5..075073d57b 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -439,6 +439,11 @@ handle smart_holder_from_unique_ptr(std::unique_ptr &&src, } auto inst = reinterpret_steal(make_new_instance(tinfo->type)); + // Before the instance owns the pointer (this can throw), and before init_instance + // publishes the instance to other threads. + if (policy == return_value_policy::reference_internal) { + keep_alive_impl(inst, parent); + } auto *inst_raw_ptr = get_instance(inst.ptr()); inst_raw_ptr->owned = true; void *&valueptr = values_and_holders(inst_raw_ptr).begin()->value_ptr(); @@ -450,11 +455,6 @@ handle smart_holder_from_unique_ptr(std::unique_ptr &&src, src_raw_void_ptr = nullptr; } auto smhldr = smart_holder::from_unique_ptr(std::move(src), src_raw_void_ptr); - // Before init_instance, which publishes the instance to other threads. - if (policy == return_value_policy::reference_internal) { - keep_alive_impl(inst, parent); - } - tinfo->init_instance(inst_raw_ptr, static_cast(&smhldr)); return inst.release(); @@ -508,17 +508,17 @@ handle smart_holder_from_shared_ptr(const std::shared_ptr &src, } auto inst = reinterpret_steal(make_new_instance(tinfo->type)); + // Before the instance owns the pointer (this can throw), and before init_instance + // publishes the instance to other threads. + if (policy == return_value_policy::reference_internal) { + keep_alive_impl(inst, parent); + } auto *inst_raw_ptr = get_instance(inst.ptr()); inst_raw_ptr->owned = true; void *&valueptr = values_and_holders(inst_raw_ptr).begin()->value_ptr(); valueptr = src_raw_void_ptr; auto smhldr = smart_holder::from_shared_ptr(std::shared_ptr(src, src_raw_void_ptr)); - // Before init_instance, which publishes the instance to other threads. - if (policy == return_value_policy::reference_internal) { - keep_alive_impl(inst, parent); - } - tinfo->init_instance(inst_raw_ptr, static_cast(&smhldr)); return inst.release(); diff --git a/tests/test_class_sh_basic.cpp b/tests/test_class_sh_basic.cpp index d9cbe1f6be..dec111e069 100644 --- a/tests/test_class_sh_basic.cpp +++ b/tests/test_class_sh_basic.cpp @@ -235,6 +235,19 @@ TEST_SUBMODULE(class_sh_basic, m) { []() { return std::unique_ptr(new atyp("rtrn_uq_automatic_reference")); }, pybind11::return_value_policy::automatic_reference); + // reference_internal without a parent: keep_alive fails before the holder exists. + m.def("cast_uq_reference_internal_no_parent", []() { + return py::cast(std::unique_ptr(new atyp("uq_no_parent")), + py::return_value_policy::reference_internal); + }); + m.def( + "rtrn_shcr_reference_internal_no_parent", + []() -> const std::shared_ptr & { + static const std::shared_ptr obj(new atyp("sh_no_parent")); + return obj; + }, + py::return_value_policy::reference_internal); + m.def("pass_shared_ptr_ptr", [](std::shared_ptr *) {}); py::classh(m, "LocalUnusualOpRef"); diff --git a/tests/test_class_sh_basic.py b/tests/test_class_sh_basic.py index aaea87b8f3..1ad8430e43 100644 --- a/tests/test_class_sh_basic.py +++ b/tests/test_class_sh_basic.py @@ -232,6 +232,20 @@ def test_unique_ptr_return_value_policy_automatic_reference(): assert m.get_mtxt(m.rtrn_uq_automatic_reference()) == "rtrn_uq_automatic_reference" +@pytest.mark.parametrize( + "rtrn_f", + [ + m.cast_uq_reference_internal_no_parent, + m.rtrn_shcr_reference_internal_no_parent, + ], +) +def test_reference_internal_no_parent(rtrn_f): + # Was a double free: the wrapper and the temporary holder both owned the pointer. + for _ in range(2): + with pytest.raises(RuntimeError, match="Could not activate keep_alive!"): + rtrn_f() + + def test_pass_shared_ptr_ptr(): obj = m.atyp() with pytest.raises(RuntimeError) as excinfo: