From 0bcbff402289eeb7a954b955e6fee7f6a5d31c4b Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 13 Nov 2026 23:24:24 -0800 Subject: [PATCH 1/9] fix(stl): rework array_caster element loading Load `std::array` and `std::valarray` elements in one of two ways: * General path: default-construct the array, then assign the elements. * Fallback (`std::array` only): temporary `std::vector` plus an `index_sequence` of `Size` elements, for element types that are not default constructible, or not assignable from what their caster yields. Since #5305 all `std::array` types went through the `index_sequence`, which is very slow to compile for large `Size` (#5765). All element loops now determine the length of the sequence only once and index up to it. Before, a sequence reporting different lengths could cause out-of-bounds accesses (a write for `std::valarray`, a read for `std::array`). Assisted-by: ClaudeCode:claude-fable-5-2 --- include/pybind11/stl.h | 60 +++++++++++++++++------- tests/test_stl.cpp | 45 +++++++++++++++++- tests/test_stl.py | 103 ++++++++++++++++++++++++++++++++++++----- 3 files changed, 177 insertions(+), 31 deletions(-) diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index b73d0e95b2..ddb5650573 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -399,45 +399,68 @@ struct array_caster { using value_conv = make_caster; private: + // On the heap because `ArrayType` is not necessarily default constructible. std::unique_ptr value; + // What `cast_op` yields, e.g. `Value &` for types registered with `py::class_`. + using value_cast_op_type = typename value_conv::template cast_op_type; + + // A `std::array` can only be populated in one of two ways: + // * Default-construct it, then assign the elements. This is the general path, which + // `std::valarray` shares. + // * Aggregate-initialize it from `Size` initializers. This is the fallback for `Value` + // types that rule out the general path. It needs a temporary `std::vector` and an + // `index_sequence` of `Size` elements, which is very slow to compile for large `Size` + // (#5765). + using assign_elements + = bool_constant::value + && std::is_assignable::value)>; + + // Returns a default-constructed array for `size` elements, or `nullptr` if `size` is not + // acceptable. `ArrayType()` rather than `ArrayType{}`: the latter is ill-formed for `Value` + // types with an explicit default constructor. + static std::unique_ptr make_array(size_t size, std::true_type /*Resizable*/) { + std::unique_ptr array(new ArrayType()); + // For the `resize` to work, `Value` must be default constructible. + // For `std::valarray`, this is a requirement: + // https://en.cppreference.com/w/cpp/named_req/NumericType + array->resize(size); + return array; + } + static std::unique_ptr make_array(size_t size, std::false_type /*Resizable*/) { + return std::unique_ptr(size == Size ? new ArrayType() : nullptr); + } + // Both `convert_elements` overloads index `l` up to a size that is determined only once, // rather than iterating over `l`: the accesses stay in bounds even if the length of `l` // changes underneath (e.g. if another thread resizes a list in a free-threaded build). - template = 0> - bool convert_elements(handle seq, bool convert) { + bool convert_elements(handle seq, bool convert, std::true_type /*assign_elements*/) { auto l = reinterpret_borrow(seq); const size_t size = l.size(); - value.reset(new ArrayType{}); - // Using `resize` to preserve the behavior exactly as it was before PR #5305 - // For the `resize` to work, `Value` must be default constructible. - // For `std::valarray`, this is a requirement: - // https://en.cppreference.com/w/cpp/named_req/NumericType - value->resize(size); + auto result = make_array(size, bool_constant{}); + if (!result) { + return false; + } for (size_t i = 0; i < size; i++) { value_conv conv; // Constructed before the item is fetched. const object item = l[i]; // Keeps the item alive while `conv` is in use. if (!conv.load(item, convert)) { return false; } - (*value)[i] = cast_op(std::move(conv)); + (*result)[i] = cast_op(std::move(conv)); } + value = std::move(result); return true; } - template = 0> - bool convert_elements(handle seq, bool convert) { + bool convert_elements(handle seq, bool convert, std::false_type /*assign_elements*/) { auto l = reinterpret_borrow(seq); const size_t size = l.size(); if (size != Size) { return false; } - // The `temp` storage is needed to support `Value` types that are not - // default-constructible. - // Deliberate choice: no template specializations, for simplicity, and - // because the compile time overhead for the specializations is deemed - // more significant than the runtime overhead for the `temp` storage. std::vector temp; temp.reserve(size); for (size_t i = 0; i < size; i++) { @@ -458,7 +481,7 @@ struct array_caster { return false; } if (isinstance(src)) { - return convert_elements(src, convert); + return convert_elements(src, convert, assign_elements{}); } if (!convert) { return false; @@ -468,7 +491,8 @@ struct array_caster { // the generator is not left in an unpredictable (to the caller) partially-consumed // state. assert(isinstance(src)); - return convert_elements(tuple(reinterpret_borrow(src)), convert); + return convert_elements( + tuple(reinterpret_borrow(src)), convert, assign_elements{}); } template diff --git a/tests/test_stl.cpp b/tests/test_stl.cpp index 9306b20670..3bcd9309fa 100644 --- a/tests/test_stl.cpp +++ b/tests/test_stl.cpp @@ -18,7 +18,10 @@ #include +#include +#include #include +#include #include #if defined(PYBIND11_TEST_BOOST) @@ -215,11 +218,51 @@ TEST_SUBMODULE(stl, m) { }; // test_array_no_default_ctor - py::class_(m, "NoDefaultCtor").def_readonly("val", &NoDefaultCtor::val); + py::class_(m, "NoDefaultCtor") + .def(py::init()) + .def_readonly("val", &NoDefaultCtor::val); py::class_(m, "NoDefaultCtorArray") .def(py::init()) .def_readwrite("arr", &NoDefaultCtorArray::arr); + // test_array_element_types + // `std::array{}` is ill-formed, `std::array()` + // is not. + struct ExplicitDefaultCtor { + explicit ExplicitDefaultCtor() = default; + explicit ExplicitDefaultCtor(int val) : val{val} {} + int val = 0; + }; + // Default constructible and move assignable, but the elements of an array cannot be + // assigned: the caster for a type registered with `py::class_` yields an lvalue reference. + struct NotCopyAssignable { + NotCopyAssignable() = default; + explicit NotCopyAssignable(int val) : val{val} {} + NotCopyAssignable(const NotCopyAssignable &) = default; + NotCopyAssignable &operator=(const NotCopyAssignable &) = delete; + NotCopyAssignable &operator=(NotCopyAssignable &&) = default; + int val = 0; + }; + py::class_(m, "ExplicitDefaultCtor").def(py::init()); + py::class_(m, "NotCopyAssignable").def(py::init()); + m.def("load_array_explicit_default_ctor", [](const std::array &a) { + return std::vector{a[0].val, a[1].val}; + }); + m.def("load_array_not_copy_assignable", [](const std::array &a) { + return std::vector{a[0].val, a[1].val}; + }); + + // test_array_nested + m.def("nested_array_cast_sequence", + [](std::array, 2> x) { return x; }); + + // test_array_large + // #5765: this was very slow to compile when `array_caster` needed an `index_sequence` of + // `Size` elements for all element types. + m.def("sum_large_array", [](const std::array &a) { + return std::accumulate(a.begin(), a.end(), 0LL); + }); + // test_valarray m.def("cast_valarray", []() { return std::valarray{1, 4, 9}; }); m.def("load_valarray", [](const std::valarray &v) { diff --git a/tests/test_stl.py b/tests/test_stl.py index 7c1f76daa6..72052bc8df 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -193,6 +193,46 @@ def test_array_no_default_ctor(): assert [e.val for e in lst.arr] == [13, 23] lst.arr = m.NoDefaultCtorArray(4).arr assert [e.val for e in lst.arr] == [14, 24] + lst.arr = (m.NoDefaultCtor(i) for i in (5, 6)) + assert [e.val for e in lst.arr] == [5, 6] + with pytest.raises(TypeError): + lst.arr = [m.NoDefaultCtor(7)] + with pytest.raises(TypeError): + lst.arr = [m.NoDefaultCtor(7), 8] + assert [e.val for e in lst.arr] == [5, 6] + + +@pytest.mark.parametrize( + ("load", "item_type"), + [ + (m.load_array_explicit_default_ctor, m.ExplicitDefaultCtor), + (m.load_array_not_copy_assignable, m.NotCopyAssignable), + ], +) +def test_array_element_types(load, item_type): + assert load([item_type(1), item_type(2)]) == [1, 2] + with pytest.raises(TypeError): + load([item_type(1)]) + with pytest.raises(TypeError): + load([item_type(1), 2]) + + +def test_array_nested(): + assert m.nested_array_cast_sequence([("a", "b"), ["c", "d"]]) == [ + ["a", "b"], + ["c", "d"], + ] + with pytest.raises(TypeError): + m.nested_array_cast_sequence([("a", "b"), ("c",)]) + + +def test_array_large(): + size = 102400 + assert m.sum_large_array(range(size)) == (size - 1) * size // 2 + with pytest.raises(TypeError): + m.sum_large_array(range(size - 1)) + with pytest.raises(TypeError): + m.sum_large_array([0] * (size - 1) + ["x"]) def test_valarray(doc): @@ -201,6 +241,10 @@ def test_valarray(doc): assert lst == [1, 4, 9] assert m.load_valarray(lst) assert m.load_valarray(tuple(lst)) + assert m.valarray_cast_sequence(i * i for i in range(4)) == [0, 1, 4, 9] + assert m.valarray_cast_sequence([]) == [] + with pytest.raises(TypeError): + m.valarray_cast_sequence([1, "x"]) assert doc(m.cast_valarray) == "cast_valarray() -> list[int]" assert ( @@ -585,6 +629,20 @@ def test_array_cast_sequence(): assert m.array_cast_sequence((1, 2, 3)) == [1, 2, 3] +def load_array_no_default_ctor(seq): + holder = m.NoDefaultCtorArray(0) + holder.arr = seq + return [e.val for e in holder.arr] + + +# (load, item_type, size): one entry for each of the element loops in `array_caster`. +ARRAY_CASTER_LOADS = [ + pytest.param(m.array_cast_sequence, int, 3, id="array"), + pytest.param(m.valarray_cast_sequence, int, 3, id="valarray"), + pytest.param(load_array_no_default_ctor, m.NoDefaultCtor, 2, id="array_fallback"), +] + + class SequenceWithChangingLen: """Reports `lens[0]` when first asked for its length, `lens[1]` after that.""" @@ -601,24 +659,45 @@ def __getitem__(self, index): return self.items[index] -ARRAY_CASTER_LOADS = [m.array_cast_sequence, m.valarray_cast_sequence] - - -@pytest.mark.parametrize("load", ARRAY_CASTER_LOADS) +@pytest.mark.parametrize(("load", "item_type", "size"), ARRAY_CASTER_LOADS) @pytest.mark.parametrize("len_change", [-1, 1]) -def test_array_caster_len_changes(load, len_change): +def test_array_caster_len_changes(load, item_type, size, len_change): """The length is determined only once, which rules out out-of-bounds accesses.""" - seq = SequenceWithChangingLen(list(range(4)), [3, 3 + len_change]) - assert load(seq) == [0, 1, 2] - assert seq.indices == [0, 1, 2] + items = [item_type(i) for i in range(size + 1)] + seq = SequenceWithChangingLen(items, [size, size + len_change]) + assert load(seq) == list(range(size)) + assert seq.indices == list(range(size)) -@pytest.mark.parametrize("load", ARRAY_CASTER_LOADS) -def test_array_caster_len_exceeds_items(load): - seq = SequenceWithChangingLen([0, 1], [3]) +@pytest.mark.parametrize(("load", "item_type", "size"), ARRAY_CASTER_LOADS) +def test_array_caster_len_exceeds_items(load, item_type, size): + items = [item_type(i) for i in range(size - 1)] + seq = SequenceWithChangingLen(items, [size]) with pytest.raises(IndexError): load(seq) - assert seq.indices == [0, 1, 2] + assert seq.indices == list(range(size)) + + +@pytest.mark.parametrize( + ("load", "item_type"), + [ + pytest.param( + m.load_array_explicit_default_ctor, m.ExplicitDefaultCtor, id="array" + ), + pytest.param(load_array_no_default_ctor, m.NoDefaultCtor, id="array_fallback"), + ], +) +def test_array_caster_items_created_on_the_fly(load, item_type): + """Each item is kept alive until its value was copied (no use-after-free).""" + + class OnTheFly: + def __len__(self): + return 2 + + def __getitem__(self, index): + return item_type(index + 10) # The caster holds the only reference. + + assert load(OnTheFly()) == [10, 11] def test_issue_1561(): From fbfc3681ac847f56d630f8ce22c23aa32a4040df Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 13 Nov 2026 23:38:40 -0800 Subject: [PATCH 2/9] fix(stl): do not require cast_op_type of cast-only element casters Select the loading path inside a member function instead of in class scope, so that an element caster that only implements `cast()` keeps working for returning a `std::array`, as on master. Assisted-by: ClaudeCode:claude-fable-5-2 --- include/pybind11/stl.h | 29 ++++++++++++++++------------- tests/test_stl.cpp | 22 ++++++++++++++++++++++ tests/test_stl.py | 6 +++++- 3 files changed, 43 insertions(+), 14 deletions(-) diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index ddb5650573..e088b10b97 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -402,9 +402,6 @@ struct array_caster { // On the heap because `ArrayType` is not necessarily default constructible. std::unique_ptr value; - // What `cast_op` yields, e.g. `Value &` for types registered with `py::class_`. - using value_cast_op_type = typename value_conv::template cast_op_type; - // A `std::array` can only be populated in one of two ways: // * Default-construct it, then assign the elements. This is the general path, which // `std::valarray` shares. @@ -412,10 +409,17 @@ struct array_caster { // types that rule out the general path. It needs a temporary `std::vector` and an // `index_sequence` of `Size` elements, which is very slow to compile for large `Size` // (#5765). - using assign_elements - = bool_constant::value - && std::is_assignable::value)>; + bool convert_elements(handle seq, bool convert) { + // What `cast_op` yields, e.g. `Value &` for types registered with `py::class_`. + // Not in class scope: a `value_conv` that is only used for `cast()` does not need to + // have a `cast_op_type`. + using value_cast_op_type = typename value_conv::template cast_op_type; + using assign_elements + = bool_constant::value + && std::is_assignable::value)>; + return convert_elements(seq, convert, assign_elements{}); + } // Returns a default-constructed array for `size` elements, or `nullptr` if `size` is not // acceptable. `ArrayType()` rather than `ArrayType{}`: the latter is ill-formed for `Value` @@ -432,9 +436,9 @@ struct array_caster { return std::unique_ptr(size == Size ? new ArrayType() : nullptr); } - // Both `convert_elements` overloads index `l` up to a size that is determined only once, - // rather than iterating over `l`: the accesses stay in bounds even if the length of `l` - // changes underneath (e.g. if another thread resizes a list in a free-threaded build). + // Both overloads below index `l` up to a size that is determined only once, rather than + // iterating over `l`: the accesses stay in bounds even if the length of `l` changes + // underneath (e.g. if another thread resizes a list in a free-threaded build). bool convert_elements(handle seq, bool convert, std::true_type /*assign_elements*/) { auto l = reinterpret_borrow(seq); @@ -481,7 +485,7 @@ struct array_caster { return false; } if (isinstance(src)) { - return convert_elements(src, convert, assign_elements{}); + return convert_elements(src, convert); } if (!convert) { return false; @@ -491,8 +495,7 @@ struct array_caster { // the generator is not left in an unpredictable (to the caller) partially-consumed // state. assert(isinstance(src)); - return convert_elements( - tuple(reinterpret_borrow(src)), convert, assign_elements{}); + return convert_elements(tuple(reinterpret_borrow(src)), convert); } template diff --git a/tests/test_stl.cpp b/tests/test_stl.cpp index 3bcd9309fa..12df40eaeb 100644 --- a/tests/test_stl.cpp +++ b/tests/test_stl.cpp @@ -65,6 +65,24 @@ struct visit_helper { PYBIND11_MAKE_OPAQUE(std::vector>) +// An element type with a caster that only supports the C++ to Python direction. +struct CastOnlyElement { + int val; +}; + +namespace PYBIND11_NAMESPACE { +namespace detail { +template <> +struct type_caster { + static constexpr auto name = const_name("int"); + + static handle cast(const CastOnlyElement &src, return_value_policy, handle) { + return int_(src.val).release(); + } +}; +} // namespace detail +} // namespace PYBIND11_NAMESPACE + /// Issue #528: templated constructor struct TplCtorClass { template @@ -252,6 +270,10 @@ TEST_SUBMODULE(stl, m) { return std::vector{a[0].val, a[1].val}; }); + m.def("cast_array_cast_only_element", []() { + return std::array{{CastOnlyElement{1}, CastOnlyElement{2}}}; + }); + // test_array_nested m.def("nested_array_cast_sequence", [](std::array, 2> x) { return x; }); diff --git a/tests/test_stl.py b/tests/test_stl.py index 72052bc8df..a2be756daf 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -217,6 +217,10 @@ def test_array_element_types(load, item_type): load([item_type(1), 2]) +def test_array_cast_only_element(): + assert m.cast_array_cast_only_element() == [1, 2] + + def test_array_nested(): assert m.nested_array_cast_sequence([("a", "b"), ["c", "d"]]) == [ ["a", "b"], @@ -695,7 +699,7 @@ def __len__(self): return 2 def __getitem__(self, index): - return item_type(index + 10) # The caster holds the only reference. + return item_type(index + 10) # The sequence does not retain the item. assert load(OnTheFly()) == [10, 11] From 84813049086846a90260e2e67c7b193cab769038 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 14 Nov 2026 10:38:07 -0800 Subject: [PATCH 3/9] fix(stl): assign std::array elements only for trivial element types Default-constructing the elements and then assigning them is not equivalent to copy constructing them for every element type. With `boost::multi_array` elements, whose assignment does not resize, the previous revision loaded empty arrays where master loads them correctly. Assign only if that cannot be told apart from constructing: the element type is trivially default constructible, trivially copyable, and trivially assignable from what its caster yields. All other element types are constructed, exactly as on master. Assisted-by: ClaudeCode:claude-fable-5-2 --- include/pybind11/stl.h | 26 ++++++++++++++---------- tests/test_stl.cpp | 45 +++++++++++++++++++++++++++++++++--------- tests/test_stl.py | 30 ++++++++++++++++++++++------ 3 files changed, 76 insertions(+), 25 deletions(-) diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index e088b10b97..a8c95113e7 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -403,21 +403,27 @@ struct array_caster { std::unique_ptr value; // A `std::array` can only be populated in one of two ways: - // * Default-construct it, then assign the elements. This is the general path, which - // `std::valarray` shares. - // * Aggregate-initialize it from `Size` initializers. This is the fallback for `Value` - // types that rule out the general path. It needs a temporary `std::vector` and an - // `index_sequence` of `Size` elements, which is very slow to compile for large `Size` - // (#5765). + // * Aggregate-initialize it from `Size` initializers, which copy or move constructs the + // elements. This needs a temporary `std::vector` and an `index_sequence` of `Size` + // elements, which is very slow to compile for large `Size` (#5765). + // * Default-construct it, then assign the elements, as for `std::valarray`. In general that + // is not equivalent: a default constructor can have side effects, and an assignment + // operator does not necessarily do what the copy constructor does (`boost::multi_array`, + // for example, does not resize on assignment). It is therefore used only for `Value` + // types for which it cannot be told apart from constructing the elements. bool convert_elements(handle seq, bool convert) { // What `cast_op` yields, e.g. `Value &` for types registered with `py::class_`. // Not in class scope: a `value_conv` that is only used for `cast()` does not need to // have a `cast_op_type`. using value_cast_op_type = typename value_conv::template cast_op_type; - using assign_elements - = bool_constant::value - && std::is_assignable::value)>; + // The conditions are on `Value` rather than `ArrayType`, to keep the selection + // independent of `Size`: a `std::array` with `Size == 0` can be trivially default + // constructible even if `Value` is not. + using assign_elements = bool_constant< + Resizable + || (std::is_trivially_default_constructible::value + && std::is_trivially_copyable::value + && std::is_trivially_assignable::value)>; return convert_elements(seq, convert, assign_elements{}); } diff --git a/tests/test_stl.cpp b/tests/test_stl.cpp index 12df40eaeb..51f929ebc2 100644 --- a/tests/test_stl.cpp +++ b/tests/test_stl.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -190,6 +191,11 @@ struct type_caster> } // namespace detail } // namespace PYBIND11_NAMESPACE +template +std::vector array_element_vals(const std::array &a) { + return {a[0].val, a[1].val}; +} + int pass_std_vector_int(const std::vector &v) { int zum = 100; for (const int i : v) { @@ -244,31 +250,52 @@ TEST_SUBMODULE(stl, m) { .def_readwrite("arr", &NoDefaultCtorArray::arr); // test_array_element_types + // `array_caster` assigns the elements of a `std::array` only if `Value` is trivial enough + // for that to be indistinguishable from copy constructing them. `ExplicitDefaultCtor` is the + // one type here that qualifies. // `std::array{}` is ill-formed, `std::array()` // is not. struct ExplicitDefaultCtor { explicit ExplicitDefaultCtor() = default; explicit ExplicitDefaultCtor(int val) : val{val} {} - int val = 0; + int val; }; - // Default constructible and move assignable, but the elements of an array cannot be - // assigned: the caster for a type registered with `py::class_` yields an lvalue reference. + static_assert(std::is_trivially_default_constructible::value + && std::is_trivially_copyable::value, + "ExplicitDefaultCtor is meant to have its array elements assigned."); + // Move assignable, but the elements of an array cannot be assigned: the caster for a type + // registered with `py::class_` yields an lvalue reference. struct NotCopyAssignable { NotCopyAssignable() = default; explicit NotCopyAssignable(int val) : val{val} {} NotCopyAssignable(const NotCopyAssignable &) = default; NotCopyAssignable &operator=(const NotCopyAssignable &) = delete; NotCopyAssignable &operator=(NotCopyAssignable &&) = default; + int val; + }; + // Assigning is not equivalent to copy constructing, in the manner of `boost::multi_array`, + // which does not resize on assignment. + struct AssignIsNoOp { + AssignIsNoOp() = default; + explicit AssignIsNoOp(int val) : val{val} {} + AssignIsNoOp(const AssignIsNoOp &) = default; + AssignIsNoOp &operator=(const AssignIsNoOp &) { return *this; } + int val; + }; + // Trivially copyable, but the default constructor must not be called. + struct ThrowingDefaultCtor { + ThrowingDefaultCtor() { throw std::runtime_error("ThrowingDefaultCtor"); } + explicit ThrowingDefaultCtor(int val) : val{val} {} int val = 0; }; py::class_(m, "ExplicitDefaultCtor").def(py::init()); py::class_(m, "NotCopyAssignable").def(py::init()); - m.def("load_array_explicit_default_ctor", [](const std::array &a) { - return std::vector{a[0].val, a[1].val}; - }); - m.def("load_array_not_copy_assignable", [](const std::array &a) { - return std::vector{a[0].val, a[1].val}; - }); + py::class_(m, "AssignIsNoOp").def(py::init()); + py::class_(m, "ThrowingDefaultCtor").def(py::init()); + m.def("load_array_explicit_default_ctor", &array_element_vals); + m.def("load_array_not_copy_assignable", &array_element_vals); + m.def("load_array_assign_is_no_op", &array_element_vals); + m.def("load_array_throwing_default_ctor", &array_element_vals); m.def("cast_array_cast_only_element", []() { return std::array{{CastOnlyElement{1}, CastOnlyElement{2}}}; diff --git a/tests/test_stl.py b/tests/test_stl.py index a2be756daf..fa5a3676a6 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -205,8 +205,24 @@ def test_array_no_default_ctor(): @pytest.mark.parametrize( ("load", "item_type"), [ - (m.load_array_explicit_default_ctor, m.ExplicitDefaultCtor), - (m.load_array_not_copy_assignable, m.NotCopyAssignable), + pytest.param( + m.load_array_explicit_default_ctor, + m.ExplicitDefaultCtor, + id="explicit_default_ctor", + ), + pytest.param( + m.load_array_not_copy_assignable, + m.NotCopyAssignable, + id="not_copy_assignable", + ), + pytest.param( + m.load_array_assign_is_no_op, m.AssignIsNoOp, id="assign_is_no_op" + ), + pytest.param( + m.load_array_throwing_default_ctor, + m.ThrowingDefaultCtor, + id="throwing_default_ctor", + ), ], ) def test_array_element_types(load, item_type): @@ -641,9 +657,9 @@ def load_array_no_default_ctor(seq): # (load, item_type, size): one entry for each of the element loops in `array_caster`. ARRAY_CASTER_LOADS = [ - pytest.param(m.array_cast_sequence, int, 3, id="array"), + pytest.param(m.array_cast_sequence, int, 3, id="array_assign"), pytest.param(m.valarray_cast_sequence, int, 3, id="valarray"), - pytest.param(load_array_no_default_ctor, m.NoDefaultCtor, 2, id="array_fallback"), + pytest.param(load_array_no_default_ctor, m.NoDefaultCtor, 2, id="array_construct"), ] @@ -686,9 +702,11 @@ def test_array_caster_len_exceeds_items(load, item_type, size): ("load", "item_type"), [ pytest.param( - m.load_array_explicit_default_ctor, m.ExplicitDefaultCtor, id="array" + m.load_array_explicit_default_ctor, + m.ExplicitDefaultCtor, + id="array_assign", ), - pytest.param(load_array_no_default_ctor, m.NoDefaultCtor, id="array_fallback"), + pytest.param(load_array_no_default_ctor, m.NoDefaultCtor, id="array_construct"), ], ) def test_array_caster_items_created_on_the_fly(load, item_type): From ebef2be244cf97b7297f3c1b94d126199dc577ab Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 14 Nov 2026 15:01:27 -0500 Subject: [PATCH 4/9] refactor(stl): simplify array_caster comments and tests Shorten the array_caster comments, get each item as an owned object directly (one less incref/decref pair per element), and bind array_element_vals for NoDefaultCtor so the tests share one parameter list. Assisted-by: ClaudeCode:claude-fable-5-2 --- include/pybind11/stl.h | 32 ++++++++++++-------------------- tests/test_stl.cpp | 1 + tests/test_stl.py | 40 ++++++++++++++++------------------------ 3 files changed, 29 insertions(+), 44 deletions(-) diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index a8c95113e7..4c1a4e8800 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -402,23 +402,17 @@ struct array_caster { // On the heap because `ArrayType` is not necessarily default constructible. std::unique_ptr value; - // A `std::array` can only be populated in one of two ways: - // * Aggregate-initialize it from `Size` initializers, which copy or move constructs the - // elements. This needs a temporary `std::vector` and an `index_sequence` of `Size` - // elements, which is very slow to compile for large `Size` (#5765). - // * Default-construct it, then assign the elements, as for `std::valarray`. In general that - // is not equivalent: a default constructor can have side effects, and an assignment - // operator does not necessarily do what the copy constructor does (`boost::multi_array`, - // for example, does not resize on assignment). It is therefore used only for `Value` - // types for which it cannot be told apart from constructing the elements. + // A `std::array` is aggregate-initialized from a temporary `std::vector`, which needs an + // `index_sequence` of `Size` elements: very slow to compile for large `Size` (#5765). + // Assigning to the elements of a default-constructed array avoids that, but is used only + // for `Value` types for which it cannot be told apart from constructing the elements + // (`boost::multi_array`, for example, does not resize on assignment). bool convert_elements(handle seq, bool convert) { - // What `cast_op` yields, e.g. `Value &` for types registered with `py::class_`. // Not in class scope: a `value_conv` that is only used for `cast()` does not need to // have a `cast_op_type`. using value_cast_op_type = typename value_conv::template cast_op_type; - // The conditions are on `Value` rather than `ArrayType`, to keep the selection - // independent of `Size`: a `std::array` with `Size == 0` can be trivially default - // constructible even if `Value` is not. + // On `Value` rather than `ArrayType`: a `std::array` with `Size == 0` can be trivially + // default constructible even if `Value` is not. using assign_elements = bool_constant< Resizable || (std::is_trivially_default_constructible::value @@ -432,9 +426,6 @@ struct array_caster { // types with an explicit default constructor. static std::unique_ptr make_array(size_t size, std::true_type /*Resizable*/) { std::unique_ptr array(new ArrayType()); - // For the `resize` to work, `Value` must be default constructible. - // For `std::valarray`, this is a requirement: - // https://en.cppreference.com/w/cpp/named_req/NumericType array->resize(size); return array; } @@ -445,6 +436,7 @@ struct array_caster { // Both overloads below index `l` up to a size that is determined only once, rather than // iterating over `l`: the accesses stay in bounds even if the length of `l` changes // underneath (e.g. if another thread resizes a list in a free-threaded build). + // Each item is owned by `item`, which keeps it alive while `conv` is in use. bool convert_elements(handle seq, bool convert, std::true_type /*assign_elements*/) { auto l = reinterpret_borrow(seq); @@ -454,8 +446,8 @@ struct array_caster { return false; } for (size_t i = 0; i < size; i++) { - value_conv conv; // Constructed before the item is fetched. - const object item = l[i]; // Keeps the item alive while `conv` is in use. + const object item = accessor_policies::sequence_item::get(l, i); + value_conv conv; if (!conv.load(item, convert)) { return false; } @@ -474,8 +466,8 @@ struct array_caster { std::vector temp; temp.reserve(size); for (size_t i = 0; i < size; i++) { - value_conv conv; // Constructed before the item is fetched. - const object item = l[i]; // Keeps the item alive while `conv` is in use. + const object item = accessor_policies::sequence_item::get(l, i); + value_conv conv; if (!conv.load(item, convert)) { return false; } diff --git a/tests/test_stl.cpp b/tests/test_stl.cpp index 51f929ebc2..516bba13f2 100644 --- a/tests/test_stl.cpp +++ b/tests/test_stl.cpp @@ -296,6 +296,7 @@ TEST_SUBMODULE(stl, m) { m.def("load_array_not_copy_assignable", &array_element_vals); m.def("load_array_assign_is_no_op", &array_element_vals); m.def("load_array_throwing_default_ctor", &array_element_vals); + m.def("load_array_no_default_ctor", &array_element_vals); m.def("cast_array_cast_only_element", []() { return std::array{{CastOnlyElement{1}, CastOnlyElement{2}}}; diff --git a/tests/test_stl.py b/tests/test_stl.py index fa5a3676a6..f1306ec0e4 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -195,8 +195,6 @@ def test_array_no_default_ctor(): assert [e.val for e in lst.arr] == [14, 24] lst.arr = (m.NoDefaultCtor(i) for i in (5, 6)) assert [e.val for e in lst.arr] == [5, 6] - with pytest.raises(TypeError): - lst.arr = [m.NoDefaultCtor(7)] with pytest.raises(TypeError): lst.arr = [m.NoDefaultCtor(7), 8] assert [e.val for e in lst.arr] == [5, 6] @@ -223,6 +221,9 @@ def test_array_no_default_ctor(): m.ThrowingDefaultCtor, id="throwing_default_ctor", ), + pytest.param( + m.load_array_no_default_ctor, m.NoDefaultCtor, id="no_default_ctor" + ), ], ) def test_array_element_types(load, item_type): @@ -649,17 +650,18 @@ def test_array_cast_sequence(): assert m.array_cast_sequence((1, 2, 3)) == [1, 2, 3] -def load_array_no_default_ctor(seq): - holder = m.NoDefaultCtorArray(0) - holder.arr = seq - return [e.val for e in holder.arr] - - # (load, item_type, size): one entry for each of the element loops in `array_caster`. ARRAY_CASTER_LOADS = [ - pytest.param(m.array_cast_sequence, int, 3, id="array_assign"), + pytest.param( + m.load_array_explicit_default_ctor, + m.ExplicitDefaultCtor, + 2, + id="array_assign", + ), pytest.param(m.valarray_cast_sequence, int, 3, id="valarray"), - pytest.param(load_array_no_default_ctor, m.NoDefaultCtor, 2, id="array_construct"), + pytest.param( + m.load_array_no_default_ctor, m.NoDefaultCtor, 2, id="array_construct" + ), ] @@ -698,28 +700,18 @@ def test_array_caster_len_exceeds_items(load, item_type, size): assert seq.indices == list(range(size)) -@pytest.mark.parametrize( - ("load", "item_type"), - [ - pytest.param( - m.load_array_explicit_default_ctor, - m.ExplicitDefaultCtor, - id="array_assign", - ), - pytest.param(load_array_no_default_ctor, m.NoDefaultCtor, id="array_construct"), - ], -) -def test_array_caster_items_created_on_the_fly(load, item_type): +@pytest.mark.parametrize(("load", "item_type", "size"), ARRAY_CASTER_LOADS) +def test_array_caster_items_created_on_the_fly(load, item_type, size): """Each item is kept alive until its value was copied (no use-after-free).""" class OnTheFly: def __len__(self): - return 2 + return size def __getitem__(self, index): return item_type(index + 10) # The sequence does not retain the item. - assert load(OnTheFly()) == [10, 11] + assert load(OnTheFly()) == list(range(10, 10 + size)) def test_issue_1561(): From a1aac6a6ede4dc7a92c72b00bdd7822cb4c82a4a Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 14 Nov 2026 12:06:05 -0800 Subject: [PATCH 5/9] fix(stl): assign std::array elements for scalar and numeric types Assign the elements of a `std::array`, instead of constructing them, for the element types for which that cannot make a difference: scalar types, the numeric types (`is_fmt_numeric`), and `std::array`s of those. The caster has to yield the element type itself, and the assignment has to be possible. All other class types are constructed, as on master. The numeric types are what makes this more than "trivial types": they include `std::complex` (with pybind11/complex.h), whose default constructor is not trivial, although all it does is zero-initialize. A large `std::array` of `std::complex` now compiles as quickly as one of `double`, and numeric types added to `is_fmt_numeric` later are covered without further changes. Trivial class types are no longer assigned. Identifying the ones for which assigning is equivalent to constructing is not feasible with type traits: review found a constructor template that takes over from the copy constructor, and an explicit move constructor, which direct-initialization uses but the copy-initialization of the array elements does not. In both cases master runs a user-defined constructor that assigning would skip. Assisted-by: ClaudeCode:claude-fable-5-2 --- include/pybind11/detail/common.h | 2 + include/pybind11/stl.h | 29 ++++++++++---- tests/test_stl.cpp | 66 ++++++++++++++++++++++++-------- tests/test_stl.py | 48 +++++++++++++++-------- 4 files changed, 107 insertions(+), 38 deletions(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index b35147add7..a1b7155e6a 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -1363,6 +1363,8 @@ PYBIND11_NAMESPACE_BEGIN(detail) // complex float,double,long double. Note that the long double types only participate when long // double is actually longer than double (it isn't under MSVC). // NB: not only the string below but also complex.h and numpy.h rely on this order. +// `array_caster` in stl.h relies on `value`: it assumes that a numeric type is a plain value, for +// which default-constructing, copying and assigning have no observable effect beyond the value. template struct is_fmt_numeric { static constexpr bool value = false; diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index 4c1a4e8800..8d2b856cb4 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -394,6 +394,20 @@ ArrayType vector_to_array(V &&v) { return vector_to_array_impl(std::forward(v), make_index_sequence{}); } +// The element types for which `array_caster` assigns the elements of a `std::array`, instead of +// constructing them: types for which that makes no difference. +// * Scalar types, which have no constructors, assignment operators or destructors. +// * The numeric types (`is_fmt_numeric`), which are assumed to be plain values. They include +// `std::complex` (with pybind11/complex.h). +// * `std::array`s of such types. +// Other class types are always constructed. Type traits cannot identify the ones for which +// assigning would be equivalent: they miss constructor templates that take over from the copy +// and move constructors, and the difference between direct- and copy-initialization. +template +struct is_plain_array_element : any_of, is_fmt_numeric> {}; +template +struct is_plain_array_element> : is_plain_array_element {}; + template struct array_caster { using value_conv = make_caster; @@ -405,19 +419,20 @@ struct array_caster { // A `std::array` is aggregate-initialized from a temporary `std::vector`, which needs an // `index_sequence` of `Size` elements: very slow to compile for large `Size` (#5765). // Assigning to the elements of a default-constructed array avoids that, but is used only - // for `Value` types for which it cannot be told apart from constructing the elements - // (`boost::multi_array`, for example, does not resize on assignment). + // for the `Value` types for which it makes no difference: `is_plain_array_element`. + // (In general it does: `boost::multi_array`, for example, does not resize on assignment.) bool convert_elements(handle seq, bool convert) { // Not in class scope: a `value_conv` that is only used for `cast()` does not need to // have a `cast_op_type`. using value_cast_op_type = typename value_conv::template cast_op_type; - // On `Value` rather than `ArrayType`: a `std::array` with `Size == 0` can be trivially - // default constructible even if `Value` is not. + // The caster also has to yield a `Value`, rather than something that converts to it, and + // the assignment has to be possible: it is not for a `std::array` of `const` elements. using assign_elements = bool_constant< Resizable - || (std::is_trivially_default_constructible::value - && std::is_trivially_copyable::value - && std::is_trivially_assignable::value)>; + || (is_plain_array_element::value + && std::is_same>, + remove_cv_t>::value + && std::is_assignable::value)>; return convert_elements(seq, convert, assign_elements{}); } diff --git a/tests/test_stl.cpp b/tests/test_stl.cpp index 516bba13f2..e359fa7a97 100644 --- a/tests/test_stl.cpp +++ b/tests/test_stl.cpp @@ -7,6 +7,7 @@ BSD-style license that can be found in the LICENSE file. */ +#include #include #include "constructor_stats.h" @@ -19,6 +20,7 @@ #include #include +#include #include #include #include @@ -66,6 +68,11 @@ struct visit_helper { PYBIND11_MAKE_OPAQUE(std::vector>) +// test_array_nested: an array with `const` elements cannot be assigned. As an element of another +// array it has to be constructed. +using OpaqueConstIntArray = std::array; +PYBIND11_MAKE_OPAQUE(OpaqueConstIntArray) + // An element type with a caster that only supports the C++ to Python direction. struct CastOnlyElement { int val; @@ -191,6 +198,18 @@ struct type_caster> } // namespace detail } // namespace PYBIND11_NAMESPACE +// test_array_element_types (at namespace scope because local classes cannot have member templates) +// Trivially copyable, but a constructor template takes over the construction from a non-const +// lvalue, which is what the caster for a registered type yields. It adds 100, idempotently. +struct CtorTemplateLvalue { + CtorTemplateLvalue() = default; + explicit CtorTemplateLvalue(int val) : val{val} {} + template + // NOLINTNEXTLINE(bugprone-forwarding-reference-overload, google-explicit-constructor) + CtorTemplateLvalue(U &&other) : val{other.val % 100 + 100} {} + int val; +}; + template std::vector array_element_vals(const std::array &a) { return {a[0].val, a[1].val}; @@ -250,19 +269,21 @@ TEST_SUBMODULE(stl, m) { .def_readwrite("arr", &NoDefaultCtorArray::arr); // test_array_element_types - // `array_caster` assigns the elements of a `std::array` only if `Value` is trivial enough - // for that to be indistinguishable from copy constructing them. `ExplicitDefaultCtor` is the - // one type here that qualifies. - // `std::array{}` is ill-formed, `std::array()` - // is not. - struct ExplicitDefaultCtor { - explicit ExplicitDefaultCtor() = default; - explicit ExplicitDefaultCtor(int val) : val{val} {} - int val; - }; - static_assert(std::is_trivially_default_constructible::value - && std::is_trivially_copyable::value, - "ExplicitDefaultCtor is meant to have its array elements assigned."); + // `array_caster` constructs the elements of a `std::array`, except for the element types + // for which assigning them instead cannot make a difference: `is_plain_array_element`. + // The class types below must all be constructed. They are trivially copyable, though, apart + // from `AssignIsNoOp`. + enum class ArrayElementEnum {}; + static_assert(py::detail::is_plain_array_element::value, ""); + static_assert(py::detail::is_plain_array_element::value, ""); + static_assert(py::detail::is_plain_array_element::value, ""); + static_assert(py::detail::is_plain_array_element>::value, ""); + static_assert(py::detail::is_plain_array_element< + std::array, 2>, 3>>::value, + ""); + static_assert(!py::detail::is_plain_array_element::value, ""); + static_assert(!py::detail::is_plain_array_element::value, ""); + static_assert(!py::detail::is_plain_array_element>::value, ""); // Move assignable, but the elements of an array cannot be assigned: the caster for a type // registered with `py::class_` yields an lvalue reference. struct NotCopyAssignable { @@ -288,21 +309,31 @@ TEST_SUBMODULE(stl, m) { explicit ThrowingDefaultCtor(int val) : val{val} {} int val = 0; }; - py::class_(m, "ExplicitDefaultCtor").def(py::init()); + py::enum_(m, "ArrayElementEnum"); py::class_(m, "NotCopyAssignable").def(py::init()); py::class_(m, "AssignIsNoOp").def(py::init()); py::class_(m, "ThrowingDefaultCtor").def(py::init()); - m.def("load_array_explicit_default_ctor", &array_element_vals); + py::class_(m, "CtorTemplateLvalue").def(py::init()); + m.def("load_array_enum", [](const std::array &a) { + return std::vector{static_cast(a[0]), static_cast(a[1])}; + }); m.def("load_array_not_copy_assignable", &array_element_vals); m.def("load_array_assign_is_no_op", &array_element_vals); m.def("load_array_throwing_default_ctor", &array_element_vals); m.def("load_array_no_default_ctor", &array_element_vals); + m.def("load_array_ctor_template_lvalue", &array_element_vals); m.def("cast_array_cast_only_element", []() { return std::array{{CastOnlyElement{1}, CastOnlyElement{2}}}; }); // test_array_nested + py::class_(m, "OpaqueConstIntArray"); + m.def("make_opaque_const_int_array", []() { return OpaqueConstIntArray{{3, 4}}; }); + m.def("load_array_opaque_const_int_array", + [](const std::array &a) { return a[0][0] + a[0][1]; }); + m.def("nested_numeric_array_cast_sequence", + [](std::array, 2>, 2> x) { return x; }); m.def("nested_array_cast_sequence", [](std::array, 2> x) { return x; }); @@ -312,6 +343,11 @@ TEST_SUBMODULE(stl, m) { m.def("sum_large_array", [](const std::array &a) { return std::accumulate(a.begin(), a.end(), 0LL); }); + // The elements of all numeric types (`is_fmt_numeric`) are assigned. `std::complex` is the + // one numeric type that is not trivially default constructible. + m.def("sum_large_complex_array", [](const std::array, 102400> &a) { + return std::accumulate(a.begin(), a.end(), std::complex()); + }); // test_valarray m.def("cast_valarray", []() { return std::valarray{1, 4, 9}; }); diff --git a/tests/test_stl.py b/tests/test_stl.py index f1306ec0e4..459dab3bf7 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -201,33 +201,37 @@ def test_array_no_default_ctor(): @pytest.mark.parametrize( - ("load", "item_type"), + ("load", "item_type", "offset"), [ - pytest.param( - m.load_array_explicit_default_ctor, - m.ExplicitDefaultCtor, - id="explicit_default_ctor", - ), pytest.param( m.load_array_not_copy_assignable, m.NotCopyAssignable, + 0, id="not_copy_assignable", ), pytest.param( - m.load_array_assign_is_no_op, m.AssignIsNoOp, id="assign_is_no_op" + m.load_array_assign_is_no_op, m.AssignIsNoOp, 0, id="assign_is_no_op" ), pytest.param( m.load_array_throwing_default_ctor, m.ThrowingDefaultCtor, + 0, id="throwing_default_ctor", ), pytest.param( - m.load_array_no_default_ctor, m.NoDefaultCtor, id="no_default_ctor" + m.load_array_no_default_ctor, m.NoDefaultCtor, 0, id="no_default_ctor" + ), + pytest.param( + m.load_array_ctor_template_lvalue, + m.CtorTemplateLvalue, + 100, + id="ctor_template_lvalue", ), ], ) -def test_array_element_types(load, item_type): - assert load([item_type(1), item_type(2)]) == [1, 2] +def test_array_element_types(load, item_type, offset): + # `offset` is what the constructor template adds when the elements are constructed. + assert load([item_type(1), item_type(2)]) == [1 + offset, 2 + offset] with pytest.raises(TypeError): load([item_type(1)]) with pytest.raises(TypeError): @@ -245,6 +249,13 @@ def test_array_nested(): ] with pytest.raises(TypeError): m.nested_array_cast_sequence([("a", "b"), ("c",)]) + assert m.nested_numeric_array_cast_sequence([(1, 2j), [3.5, 4]]) == [ + [1, 2j], + [3.5, 4], + ] + with pytest.raises(TypeError): + m.nested_numeric_array_cast_sequence([(1, 2j), [3.5]]) + assert m.load_array_opaque_const_int_array([m.make_opaque_const_int_array()]) == 7 def test_array_large(): @@ -255,6 +266,16 @@ def test_array_large(): with pytest.raises(TypeError): m.sum_large_array([0] * (size - 1) + ["x"]) + total = (size - 1) * size // 2 + assert m.sum_large_complex_array([complex(i, -i) for i in range(size)]) == complex( + total, -total + ) + assert m.sum_large_complex_array(range(size)) == total + with pytest.raises(TypeError): + m.sum_large_complex_array(range(size - 1)) + with pytest.raises(TypeError): + m.sum_large_complex_array([0j] * (size - 1) + ["x"]) + def test_valarray(doc): """std::valarray <-> list""" @@ -652,12 +673,7 @@ def test_array_cast_sequence(): # (load, item_type, size): one entry for each of the element loops in `array_caster`. ARRAY_CASTER_LOADS = [ - pytest.param( - m.load_array_explicit_default_ctor, - m.ExplicitDefaultCtor, - 2, - id="array_assign", - ), + pytest.param(m.load_array_enum, m.ArrayElementEnum, 2, id="array_assign"), pytest.param(m.valarray_cast_sequence, int, 3, id="valarray"), pytest.param( m.load_array_no_default_ctor, m.NoDefaultCtor, 2, id="array_construct" From a0d031a3481653a6deec3473e68d109993ef434b Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 14 Nov 2026 12:53:17 -0800 Subject: [PATCH 6/9] fix(stl): construct each element caster before its item is fetched Master iterates over the sequence with a lazy accessor, so that an item is fetched when its caster loads it, after the caster was constructed. The indexed loops fetched the item first. A caster whose constructor has side effects that `__getitem__` can observe saw the difference, also for a sequence of constant length. Go back to an accessor, which fetches the item when it is first used, and which keeps it alive while the caster is in use. Pass `item.ptr()` to the caster instead of the accessor itself: that keeps the saving of the cleanup commit, which avoided the temporary `object` that master creates for each item. Assisted-by: ClaudeCode:claude-fable-5-2 --- include/pybind11/stl.h | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index 8d2b856cb4..884696b7d9 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -451,7 +451,9 @@ struct array_caster { // Both overloads below index `l` up to a size that is determined only once, rather than // iterating over `l`: the accesses stay in bounds even if the length of `l` changes // underneath (e.g. if another thread resizes a list in a free-threaded build). - // Each item is owned by `item`, which keeps it alive while `conv` is in use. + // `item` is an accessor: the item is fetched by `item.ptr()`, that is after `conv` is + // constructed, as when iterating over `l`. It keeps the item alive while `conv` is in use. + // Passing `item.ptr()` rather than `item` avoids a temporary `object`. bool convert_elements(handle seq, bool convert, std::true_type /*assign_elements*/) { auto l = reinterpret_borrow(seq); @@ -461,9 +463,9 @@ struct array_caster { return false; } for (size_t i = 0; i < size; i++) { - const object item = accessor_policies::sequence_item::get(l, i); + const auto item = l[i]; value_conv conv; - if (!conv.load(item, convert)) { + if (!conv.load(item.ptr(), convert)) { return false; } (*result)[i] = cast_op(std::move(conv)); @@ -481,9 +483,9 @@ struct array_caster { std::vector temp; temp.reserve(size); for (size_t i = 0; i < size; i++) { - const object item = accessor_policies::sequence_item::get(l, i); + const auto item = l[i]; value_conv conv; - if (!conv.load(item, convert)) { + if (!conv.load(item.ptr(), convert)) { return false; } temp.emplace_back(cast_op(std::move(conv))); From af0d8092c5f02abb80874354be97d1b07582326f Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 14 Nov 2026 12:53:18 -0800 Subject: [PATCH 7/9] docs: caster headers must be visible consistently Add a note to the documentation of the STL container conversions. The general rule in the note already applies on master, but was only documented for custom type casters: the optional caster headers (stl.h, complex.h, ...) must be visible consistently wherever the casters or traits they provide affect a conversion. If one translation unit of an extension module converts a type with such a header taking effect, every translation unit that converts that type has to include it, or the ODR is violated. The last sentence of the note is new with this PR. complex.h now also matters for a `std::array` of `std::complex` if `std::complex` is opaque or has a custom caster, because `array_caster` selects how it loads the elements with `is_fmt_numeric`, a trait that complex.h specializes. Assisted-by: ClaudeCode:claude-fable-5-2 --- docs/advanced/cast/stl.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/advanced/cast/stl.rst b/docs/advanced/cast/stl.rst index 6abe8b467d..271dc8b29f 100644 --- a/docs/advanced/cast/stl.rst +++ b/docs/advanced/cast/stl.rst @@ -25,6 +25,19 @@ In particular, containers of C++ string views have additional Arbitrary nesting of any of these types is possible. +.. note:: + + The optional headers that provide type casters (:file:`pybind11/stl.h`, + :file:`pybind11/complex.h`, ...) must be visible consistently wherever the + casters or traits they provide affect a conversion. If such a header + affects the conversion of a type in one translation unit of an extension + module, it must be included in every translation unit that converts that + type. Otherwise the C++ One Definition Rule (ODR) is violated. This applies + to :file:`pybind11/complex.h` for a ``std::array<>`` of ``std::complex<>`` + even if ``std::complex<>`` is :ref:`opaque ` or has a custom type + caster, because the conversion of such an array depends on a trait that + the header specializes. + .. seealso:: The file :file:`tests/test_stl.cpp` contains a complete From 90683bf435ee5f18f451e193e3742bb4a5dd1860 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 14 Nov 2026 21:17:11 -0500 Subject: [PATCH 8/9] fix(stl): limit the array_caster assign path to std::array * Assign the elements only if `ArrayType` is `std::array`. Other fixed-size array types do not need a default constructor or a mutable `operator[]`, as before. * Match `std::complex` directly in `is_plain_array_element`, not through `is_fmt_numeric`. The loop that `array_caster` instantiates no longer depends on whether pybind11/complex.h is included, so the docs note about that is removed. * Use one element loop for both paths. It constructs the caster before it fetches the item, by statement order. * Remove the large `std::complex` array from the tests. Assisted-by: ClaudeCode:claude-fable-5-2 --- docs/advanced/cast/stl.rst | 13 ------- include/pybind11/detail/common.h | 2 -- include/pybind11/stl.h | 58 +++++++++++++++++++------------- tests/test_stl.cpp | 26 +++++++++++--- tests/test_stl.py | 13 +++---- 5 files changed, 60 insertions(+), 52 deletions(-) diff --git a/docs/advanced/cast/stl.rst b/docs/advanced/cast/stl.rst index 271dc8b29f..6abe8b467d 100644 --- a/docs/advanced/cast/stl.rst +++ b/docs/advanced/cast/stl.rst @@ -25,19 +25,6 @@ In particular, containers of C++ string views have additional Arbitrary nesting of any of these types is possible. -.. note:: - - The optional headers that provide type casters (:file:`pybind11/stl.h`, - :file:`pybind11/complex.h`, ...) must be visible consistently wherever the - casters or traits they provide affect a conversion. If such a header - affects the conversion of a type in one translation unit of an extension - module, it must be included in every translation unit that converts that - type. Otherwise the C++ One Definition Rule (ODR) is violated. This applies - to :file:`pybind11/complex.h` for a ``std::array<>`` of ``std::complex<>`` - even if ``std::complex<>`` is :ref:`opaque ` or has a custom type - caster, because the conversion of such an array depends on a trait that - the header specializes. - .. seealso:: The file :file:`tests/test_stl.cpp` contains a complete diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index a1b7155e6a..b35147add7 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -1363,8 +1363,6 @@ PYBIND11_NAMESPACE_BEGIN(detail) // complex float,double,long double. Note that the long double types only participate when long // double is actually longer than double (it isn't under MSVC). // NB: not only the string below but also complex.h and numpy.h rely on this order. -// `array_caster` in stl.h relies on `value`: it assumes that a numeric type is a plain value, for -// which default-constructing, copying and assigning have no observable effect beyond the value. template struct is_fmt_numeric { static constexpr bool value = false; diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index 884696b7d9..e7da514696 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -14,6 +14,7 @@ #include "detail/descr.h" #include "detail/type_caster_base.h" +#include #include #include #include @@ -397,14 +398,16 @@ ArrayType vector_to_array(V &&v) { // The element types for which `array_caster` assigns the elements of a `std::array`, instead of // constructing them: types for which that makes no difference. // * Scalar types, which have no constructors, assignment operators or destructors. -// * The numeric types (`is_fmt_numeric`), which are assumed to be plain values. They include -// `std::complex` (with pybind11/complex.h). +// * `std::complex` of an arithmetic type. It is matched here, not through a trait that +// pybind11/complex.h specializes: the result must not depend on the headers included. // * `std::array`s of such types. // Other class types are always constructed. Type traits cannot identify the ones for which // assigning would be equivalent: they miss constructor templates that take over from the copy // and move constructors, and the difference between direct- and copy-initialization. template -struct is_plain_array_element : any_of, is_fmt_numeric> {}; +struct is_plain_array_element : std::is_scalar {}; +template +struct is_plain_array_element> : std::is_arithmetic {}; template struct is_plain_array_element> : is_plain_array_element {}; @@ -421,6 +424,8 @@ struct array_caster { // Assigning to the elements of a default-constructed array avoids that, but is used only // for the `Value` types for which it makes no difference: `is_plain_array_element`. // (In general it does: `boost::multi_array`, for example, does not resize on assignment.) + // It is also used only for `std::array` itself: another `ArrayType` does not need to have a + // default constructor or a mutable `operator[]`. bool convert_elements(handle seq, bool convert) { // Not in class scope: a `value_conv` that is only used for `cast()` does not need to // have a `cast_op_type`. @@ -429,7 +434,8 @@ struct array_caster { // the assignment has to be possible: it is not for a `std::array` of `const` elements. using assign_elements = bool_constant< Resizable - || (is_plain_array_element::value + || (std::is_same>::value + && is_plain_array_element::value && std::is_same>, remove_cv_t>::value && std::is_assignable::value)>; @@ -448,12 +454,22 @@ struct array_caster { return std::unique_ptr(size == Size ? new ArrayType() : nullptr); } - // Both overloads below index `l` up to a size that is determined only once, rather than - // iterating over `l`: the accesses stay in bounds even if the length of `l` changes - // underneath (e.g. if another thread resizes a list in a free-threaded build). - // `item` is an accessor: the item is fetched by `item.ptr()`, that is after `conv` is - // constructed, as when iterating over `l`. It keeps the item alive while `conv` is in use. - // Passing `item.ptr()` rather than `item` avoids a temporary `object`. + // Loads the items of `l` one by one and passes each loaded caster to `sink(i, conv)`. + // Indexes `l` up to `size`, which the caller determines only once, rather than iterating + // over `l`: the accesses stay in bounds even if the length of `l` changes underneath (e.g. + // if another thread resizes a list in a free-threaded build). + template + static bool load_elements(const sequence &l, size_t size, bool convert, Sink &&sink) { + for (size_t i = 0; i < size; i++) { + value_conv conv; // Before the item is fetched, as when iterating over `l`. + const object item = l[i]; // Keeps the item alive while `conv` is in use. + if (!conv.load(item, convert)) { + return false; + } + sink(i, std::move(conv)); + } + return true; + } bool convert_elements(handle seq, bool convert, std::true_type /*assign_elements*/) { auto l = reinterpret_borrow(seq); @@ -462,13 +478,10 @@ struct array_caster { if (!result) { return false; } - for (size_t i = 0; i < size; i++) { - const auto item = l[i]; - value_conv conv; - if (!conv.load(item.ptr(), convert)) { - return false; - } - (*result)[i] = cast_op(std::move(conv)); + if (!load_elements(l, size, convert, [&](size_t i, value_conv &&conv) { + (*result)[i] = cast_op(std::move(conv)); + })) { + return false; } value = std::move(result); return true; @@ -482,13 +495,10 @@ struct array_caster { } std::vector temp; temp.reserve(size); - for (size_t i = 0; i < size; i++) { - const auto item = l[i]; - value_conv conv; - if (!conv.load(item.ptr(), convert)) { - return false; - } - temp.emplace_back(cast_op(std::move(conv))); + if (!load_elements(l, size, convert, [&](size_t, value_conv &&conv) { + temp.emplace_back(cast_op(std::move(conv))); + })) { + return false; } value.reset(new ArrayType(vector_to_array(std::move(temp)))); return true; diff --git a/tests/test_stl.cpp b/tests/test_stl.cpp index e359fa7a97..6b5c080294 100644 --- a/tests/test_stl.cpp +++ b/tests/test_stl.cpp @@ -91,6 +91,24 @@ struct type_caster { } // namespace detail } // namespace PYBIND11_NAMESPACE +// test_array_custom_type: `array_caster` can only construct this from a braced list. It has no +// default constructor and no mutable `operator[]`. +struct FixedArray3 { + // NOLINTNEXTLINE(google-explicit-constructor) + FixedArray3(const std::array &arr) : arr(arr) {} + const double *begin() const { return arr.data(); } + const double *end() const { return arr.data() + arr.size(); } + size_t size() const { return arr.size(); } + std::array arr; +}; + +namespace PYBIND11_NAMESPACE { +namespace detail { +template <> +struct type_caster : array_caster {}; +} // namespace detail +} // namespace PYBIND11_NAMESPACE + /// Issue #528: templated constructor struct TplCtorClass { template @@ -343,11 +361,9 @@ TEST_SUBMODULE(stl, m) { m.def("sum_large_array", [](const std::array &a) { return std::accumulate(a.begin(), a.end(), 0LL); }); - // The elements of all numeric types (`is_fmt_numeric`) are assigned. `std::complex` is the - // one numeric type that is not trivially default constructible. - m.def("sum_large_complex_array", [](const std::array, 102400> &a) { - return std::accumulate(a.begin(), a.end(), std::complex()); - }); + + // test_array_custom_type + m.def("fixed_array3_cast_sequence", [](FixedArray3 x) { return x; }); // test_valarray m.def("cast_valarray", []() { return std::valarray{1, 4, 9}; }); diff --git a/tests/test_stl.py b/tests/test_stl.py index 459dab3bf7..e180da0912 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -266,15 +266,12 @@ def test_array_large(): with pytest.raises(TypeError): m.sum_large_array([0] * (size - 1) + ["x"]) - total = (size - 1) * size // 2 - assert m.sum_large_complex_array([complex(i, -i) for i in range(size)]) == complex( - total, -total - ) - assert m.sum_large_complex_array(range(size)) == total - with pytest.raises(TypeError): - m.sum_large_complex_array(range(size - 1)) + +def test_array_custom_type(): + """A fixed-size `ArrayType` other than `std::array` only has to be constructible.""" + assert m.fixed_array3_cast_sequence([1, 2, 3]) == [1.0, 2.0, 3.0] with pytest.raises(TypeError): - m.sum_large_complex_array([0j] * (size - 1) + ["x"]) + m.fixed_array3_cast_sequence([1, 2]) def test_valarray(doc): From 73130f514fd97a9b5708169413a956d959be214f Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 16 Nov 2026 00:52:43 -0500 Subject: [PATCH 9/9] test(stl): make FixedArray3 an aggregate Clang 5 rejects the braced initialization through the converting constructor with -Wmissing-braces. Assisted-by: ClaudeCode:claude-fable-5-2 --- tests/test_stl.cpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/test_stl.cpp b/tests/test_stl.cpp index 6b5c080294..eb550f1bd2 100644 --- a/tests/test_stl.cpp +++ b/tests/test_stl.cpp @@ -91,15 +91,13 @@ struct type_caster { } // namespace detail } // namespace PYBIND11_NAMESPACE -// test_array_custom_type: `array_caster` can only construct this from a braced list. It has no -// default constructor and no mutable `operator[]`. +// test_array_custom_type: `array_caster` can only construct this from a braced list. The `const` +// member rules out a default constructor, and there is no `operator[]`. struct FixedArray3 { - // NOLINTNEXTLINE(google-explicit-constructor) - FixedArray3(const std::array &arr) : arr(arr) {} - const double *begin() const { return arr.data(); } - const double *end() const { return arr.data() + arr.size(); } - size_t size() const { return arr.size(); } - std::array arr; + const double *begin() const { return arr; } + const double *end() const { return arr + 3; } + size_t size() const { return 3; } + const double arr[3]; }; namespace PYBIND11_NAMESPACE {