From 1dcb9ed9a14d4febfe5125ad853ccfe0136bd632 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 16 Nov 2026 10:55:57 -0500 Subject: [PATCH 1/3] fix(stl): keep element casters alive when elements refer to them A container caster destroyed the type caster of each element before the bound function ran. Elements that refer to data in their caster, such as the pointers in `std::vector`, were then dangling. Add the `cast_refers_to_caster` trait. The container casters keep the element casters for the duration of the call only if the trait is true. A custom caster can opt in with a `refers_to_caster` member. Fixes #2527, fixes #2245. Assisted-by: ClaudeCode:claude-fable-5-2 --- docs/advanced/cast/custom.rst | 9 +++ docs/limitations.rst | 6 +- include/pybind11/cast.h | 16 +++++ include/pybind11/stl.h | 91 ++++++++++++++++++++++--- tests/test_stl.cpp | 123 ++++++++++++++++++++++++++++++++++ tests/test_stl.py | 29 ++++++++ 6 files changed, 263 insertions(+), 11 deletions(-) diff --git a/docs/advanced/cast/custom.rst b/docs/advanced/cast/custom.rst index de673b2d86..9ddd18e100 100644 --- a/docs/advanced/cast/custom.rst +++ b/docs/advanced/cast/custom.rst @@ -138,6 +138,15 @@ type is explicitly allowed. intended only for type-caster implementations. See :ref:`string_view_lifetime` for a concrete example. +.. note:: + + If a custom ``load()`` instead produces a non-owning view of storage that + the type caster itself owns (for example, a span over a ``std::vector`` + member), add ``static constexpr bool refers_to_caster = true;`` to the type + caster. The STL container casters then keep the type caster of each element + alive, so that a ``std::vector`` of such views stays valid for the duration + of a bound-function call. + .. warning:: When using custom type casters, it's important to declare them consistently diff --git a/docs/limitations.rst b/docs/limitations.rst index 1b06ea8728..a64179b1f2 100644 --- a/docs/limitations.rst +++ b/docs/limitations.rst @@ -45,10 +45,10 @@ Known limitations These are issues that are probably solvable, but have not been fixed yet. A clean, well written patch would likely be accepted to solve them. -- Type casters are not kept alive recursively. +- ``py::cast()`` does not keep type casters alive. A ``T`` that refers to + data that its type caster owns, such as a container of ``char *``, is valid + only as an argument of a bound function. `#2527 `_ - One consequence is that containers of ``char *`` are currently not supported. - `#2245 `_ Python 3.9.0 warning ^^^^^^^^^^^^^^^^^^^^ diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 79e743d9bb..59eea39629 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -159,6 +159,10 @@ bool isinstance_native_enum(handle obj, const std::type_info &tp) { return isinstance_native_enum_impl>(obj, tp); } +// Defined below, after `cast_is_temporary_value_reference`. +template +struct cast_refers_to_caster; + template class type_caster> { private: @@ -176,6 +180,7 @@ class type_caster> { public: bool load(handle src, bool convert) { return subcaster.load(src, convert); } static constexpr auto name = caster_t::name; + static constexpr bool refers_to_caster = cast_refers_to_caster::value; static handle cast(const std::reference_wrapper &src, return_value_policy policy, handle parent) { // It is definitely wrong to take ownership of this pointer, so mask that rvp @@ -800,6 +805,7 @@ class tuple_caster { static constexpr auto name = const_name("tuple[") + ::pybind11::detail::concat(make_caster::name...) + const_name("]"); + static constexpr bool refers_to_caster = any_of...>::value; template using cast_op_type = type; @@ -1637,6 +1643,16 @@ using cast_is_temporary_value_reference && !std::is_base_of>::value && !std::is_same, void>::value>; +// Detect whether the result of a cast to `T` refers to data that the type_caster owns. The +// type_caster must then stay alive for as long as the result is in use. This is the case for +// `cast_is_temporary_value_reference`, and for a type_caster that has a +// `static constexpr bool refers_to_caster = true` member. A type_caster needs that member if +// it returns, by value, a type that does not own its data (a container of pointers, a span). +template +struct cast_refers_to_caster : cast_is_temporary_value_reference {}; +template +struct cast_refers_to_caster::refers_to_caster>> : std::true_type {}; + // When a value returned from a C++ function is being cast back to Python, we almost always want to // force `policy = move`, regardless of the return value policy the function/method was declared // with. diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index b73d0e95b2..dae9500ab6 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -167,10 +167,46 @@ constexpr forwarded_type forward_like(U &&u) { template using has_reserve_method = std::is_same().reserve(0)), void>; +// Provides the type_caster for each element that a container type_caster loads. The element +// type_casters stay alive with this object only if the elements refer to them, for example the +// elements of `std::vector`. +template ::value> +class element_casters { +public: + struct slot { + explicit slot(element_casters &) {} + make_caster caster; + }; + void clear() {} +}; + +template +class element_casters { +public: + struct slot { + explicit slot(element_casters &owner) : caster(owner.add()) {} + make_caster &caster; + }; + void clear() { kept.clear(); } + + element_casters() = default; + // Not copyable: the loaded elements refer to the type_casters of the original. + element_casters(element_casters &&) = default; + element_casters &operator=(element_casters &&) = default; + +private: + make_caster &add() { + kept.emplace_back(); + return kept.back(); + } + std::list> kept; // The elements refer to the nodes, which do not move. +}; + template struct set_caster { using type = Type; using key_conv = make_caster; + static constexpr bool refers_to_caster = cast_refers_to_caster::value; private: template ::value, int> = 0> @@ -179,9 +215,13 @@ struct set_caster { } void reserve_maybe(const anyset &, void *) {} + element_casters key_casters; + bool convert_iterable(const iterable &itbl, bool convert) { + key_casters.clear(); for (const auto &it : itbl) { - key_conv conv; + typename element_casters::slot slot(key_casters); + key_conv &conv = slot.caster; if (!conv.load(it, convert)) { return false; } @@ -238,8 +278,13 @@ template struct map_caster { using key_conv = make_caster; using value_conv = make_caster; + static constexpr bool refers_to_caster + = any_of, cast_refers_to_caster>::value; private: + element_casters key_casters; + element_casters value_casters; + template ::value, int> = 0> void reserve_maybe(const dict &d, Type *) { value.reserve(d.size()); @@ -248,10 +293,14 @@ struct map_caster { bool convert_elements(const dict &d, bool convert) { value.clear(); + key_casters.clear(); + value_casters.clear(); reserve_maybe(d, &value); for (const auto &it : d) { - key_conv kconv; - value_conv vconv; + typename element_casters::slot kslot(key_casters); + typename element_casters::slot vslot(value_casters); + key_conv &kconv = kslot.caster; + value_conv &vconv = vslot.caster; if (!kconv.load(it.first.ptr(), convert) || !vconv.load(it.second.ptr(), convert)) { return false; } @@ -310,6 +359,7 @@ struct map_caster { template struct list_caster { using value_conv = make_caster; + static constexpr bool refers_to_caster = cast_refers_to_caster::value; bool load(handle src, bool convert) { if (!object_is_convertible_to_std_vector(src)) { @@ -330,6 +380,8 @@ struct list_caster { } private: + element_casters value_casters; + template ::value, int> = 0> void reserve_maybe(const sequence &s, Type *) { value.reserve(s.size()); @@ -339,9 +391,11 @@ struct list_caster { bool convert_elements(handle seq, bool convert) { auto s = reinterpret_borrow(seq); value.clear(); + value_casters.clear(); reserve_maybe(s, &value); for (const auto &it : seq) { - value_conv conv; + typename element_casters::slot slot(value_casters); + value_conv &conv = slot.caster; if (!conv.load(it, convert)) { return false; } @@ -397,9 +451,11 @@ ArrayType vector_to_array(V &&v) { template struct array_caster { using value_conv = make_caster; + static constexpr bool refers_to_caster = cast_refers_to_caster::value; private: std::unique_ptr value; + element_casters value_casters; // Both `convert_elements` overloads index `l` up to a size that is determined only once, // rather than iterating over `l`: the accesses stay in bounds even if the length of `l` @@ -415,8 +471,11 @@ struct array_caster { // For `std::valarray`, this is a requirement: // https://en.cppreference.com/w/cpp/named_req/NumericType value->resize(size); + value_casters.clear(); for (size_t i = 0; i < size; i++) { - value_conv conv; // Constructed before the item is fetched. + // Constructed before the item is fetched. + typename element_casters::slot slot(value_casters); + value_conv &conv = slot.caster; const object item = l[i]; // Keeps the item alive while `conv` is in use. if (!conv.load(item, convert)) { return false; @@ -440,8 +499,11 @@ struct array_caster { // more significant than the runtime overhead for the `temp` storage. std::vector temp; temp.reserve(size); + value_casters.clear(); for (size_t i = 0; i < size; i++) { - value_conv conv; // Constructed before the item is fetched. + // Constructed before the item is fetched. + typename element_casters::slot slot(value_casters); + value_conv &conv = slot.caster; const object item = l[i]; // Keeps the item alive while `conv` is in use. if (!conv.load(item, convert)) { return false; @@ -547,6 +609,7 @@ struct type_caster> template struct optional_caster { using value_conv = make_caster; + static constexpr bool refers_to_caster = cast_refers_to_caster::value; template static handle cast(T &&src, return_value_policy policy, handle parent) { @@ -567,7 +630,9 @@ struct optional_caster { if (src.is_none()) { return true; // default-constructed value is already empty } - value_conv inner_caster; + value_casters.clear(); + typename element_casters::slot slot(value_casters); + value_conv &inner_caster = slot.caster; if (!inner_caster.load(src, convert)) { return false; } @@ -577,6 +642,9 @@ struct optional_caster { } PYBIND11_TYPE_CASTER(Type, value_conv::name | make_caster::name); + +private: + element_casters value_casters; }; #if defined(PYBIND11_HAS_OPTIONAL) @@ -629,10 +697,14 @@ struct variant_caster; template