diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index b73d0e95b2..e7da514696 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -14,6 +14,7 @@ #include "detail/descr.h" #include "detail/type_caster_base.h" +#include #include #include #include @@ -394,59 +395,110 @@ ArrayType vector_to_array(V &&v) { return vector_to_array_impl(std::forward(v), make_index_sequence{}); } +// The element types for which `array_caster` assigns the elements of a `std::array`, instead of +// constructing them: types for which that makes no difference. +// * Scalar types, which have no constructors, assignment operators or destructors. +// * `std::complex` of an arithmetic type. It is matched here, not through a trait that +// pybind11/complex.h specializes: the result must not depend on the headers included. +// * `std::array`s of such types. +// Other class types are always constructed. Type traits cannot identify the ones for which +// assigning would be equivalent: they miss constructor templates that take over from the copy +// and move constructors, and the difference between direct- and copy-initialization. +template +struct is_plain_array_element : std::is_scalar {}; +template +struct is_plain_array_element> : std::is_arithmetic {}; +template +struct is_plain_array_element> : is_plain_array_element {}; + template struct array_caster { using value_conv = make_caster; private: + // On the heap because `ArrayType` is not necessarily default constructible. std::unique_ptr value; - // Both `convert_elements` overloads index `l` up to a size that is determined only once, - // rather than iterating over `l`: the accesses stay in bounds even if the length of `l` - // changes underneath (e.g. if another thread resizes a list in a free-threaded build). - - template = 0> + // A `std::array` is aggregate-initialized from a temporary `std::vector`, which needs an + // `index_sequence` of `Size` elements: very slow to compile for large `Size` (#5765). + // Assigning to the elements of a default-constructed array avoids that, but is used only + // for the `Value` types for which it makes no difference: `is_plain_array_element`. + // (In general it does: `boost::multi_array`, for example, does not resize on assignment.) + // It is also used only for `std::array` itself: another `ArrayType` does not need to have a + // default constructor or a mutable `operator[]`. bool convert_elements(handle seq, bool convert) { - auto l = reinterpret_borrow(seq); - const size_t size = l.size(); - value.reset(new ArrayType{}); - // Using `resize` to preserve the behavior exactly as it was before PR #5305 - // For the `resize` to work, `Value` must be default constructible. - // For `std::valarray`, this is a requirement: - // https://en.cppreference.com/w/cpp/named_req/NumericType - value->resize(size); + // Not in class scope: a `value_conv` that is only used for `cast()` does not need to + // have a `cast_op_type`. + using value_cast_op_type = typename value_conv::template cast_op_type; + // The caster also has to yield a `Value`, rather than something that converts to it, and + // the assignment has to be possible: it is not for a `std::array` of `const` elements. + using assign_elements = bool_constant< + Resizable + || (std::is_same>::value + && is_plain_array_element::value + && std::is_same>, + remove_cv_t>::value + && std::is_assignable::value)>; + return convert_elements(seq, convert, assign_elements{}); + } + + // Returns a default-constructed array for `size` elements, or `nullptr` if `size` is not + // acceptable. `ArrayType()` rather than `ArrayType{}`: the latter is ill-formed for `Value` + // types with an explicit default constructor. + static std::unique_ptr make_array(size_t size, std::true_type /*Resizable*/) { + std::unique_ptr array(new ArrayType()); + array->resize(size); + return array; + } + static std::unique_ptr make_array(size_t size, std::false_type /*Resizable*/) { + return std::unique_ptr(size == Size ? new ArrayType() : nullptr); + } + + // Loads the items of `l` one by one and passes each loaded caster to `sink(i, conv)`. + // Indexes `l` up to `size`, which the caller determines only once, rather than iterating + // over `l`: the accesses stay in bounds even if the length of `l` changes underneath (e.g. + // if another thread resizes a list in a free-threaded build). + template + static bool load_elements(const sequence &l, size_t size, bool convert, Sink &&sink) { for (size_t i = 0; i < size; i++) { - value_conv conv; // Constructed before the item is fetched. + value_conv conv; // Before the item is fetched, as when iterating over `l`. const object item = l[i]; // Keeps the item alive while `conv` is in use. if (!conv.load(item, convert)) { return false; } - (*value)[i] = cast_op(std::move(conv)); + sink(i, std::move(conv)); } return true; } - template = 0> - bool convert_elements(handle seq, bool convert) { + bool convert_elements(handle seq, bool convert, std::true_type /*assign_elements*/) { + auto l = reinterpret_borrow(seq); + const size_t size = l.size(); + auto result = make_array(size, bool_constant{}); + if (!result) { + return false; + } + if (!load_elements(l, size, convert, [&](size_t i, value_conv &&conv) { + (*result)[i] = cast_op(std::move(conv)); + })) { + return false; + } + value = std::move(result); + return true; + } + + bool convert_elements(handle seq, bool convert, std::false_type /*assign_elements*/) { auto l = reinterpret_borrow(seq); const size_t size = l.size(); if (size != Size) { return false; } - // The `temp` storage is needed to support `Value` types that are not - // default-constructible. - // Deliberate choice: no template specializations, for simplicity, and - // because the compile time overhead for the specializations is deemed - // more significant than the runtime overhead for the `temp` storage. std::vector temp; temp.reserve(size); - for (size_t i = 0; i < size; i++) { - value_conv conv; // Constructed before the item is fetched. - const object item = l[i]; // Keeps the item alive while `conv` is in use. - if (!conv.load(item, convert)) { - return false; - } - temp.emplace_back(cast_op(std::move(conv))); + if (!load_elements(l, size, convert, [&](size_t, value_conv &&conv) { + temp.emplace_back(cast_op(std::move(conv))); + })) { + return false; } value.reset(new ArrayType(vector_to_array(std::move(temp)))); return true; diff --git a/tests/test_stl.cpp b/tests/test_stl.cpp index 9306b20670..eb550f1bd2 100644 --- a/tests/test_stl.cpp +++ b/tests/test_stl.cpp @@ -7,6 +7,7 @@ BSD-style license that can be found in the LICENSE file. */ +#include #include #include "constructor_stats.h" @@ -18,7 +19,12 @@ #include +#include +#include +#include +#include #include +#include #include #if defined(PYBIND11_TEST_BOOST) @@ -62,6 +68,45 @@ struct visit_helper { PYBIND11_MAKE_OPAQUE(std::vector>) +// test_array_nested: an array with `const` elements cannot be assigned. As an element of another +// array it has to be constructed. +using OpaqueConstIntArray = std::array; +PYBIND11_MAKE_OPAQUE(OpaqueConstIntArray) + +// An element type with a caster that only supports the C++ to Python direction. +struct CastOnlyElement { + int val; +}; + +namespace PYBIND11_NAMESPACE { +namespace detail { +template <> +struct type_caster { + static constexpr auto name = const_name("int"); + + static handle cast(const CastOnlyElement &src, return_value_policy, handle) { + return int_(src.val).release(); + } +}; +} // namespace detail +} // namespace PYBIND11_NAMESPACE + +// test_array_custom_type: `array_caster` can only construct this from a braced list. The `const` +// member rules out a default constructor, and there is no `operator[]`. +struct FixedArray3 { + const double *begin() const { return arr; } + const double *end() const { return arr + 3; } + size_t size() const { return 3; } + const double arr[3]; +}; + +namespace PYBIND11_NAMESPACE { +namespace detail { +template <> +struct type_caster : array_caster {}; +} // namespace detail +} // namespace PYBIND11_NAMESPACE + /// Issue #528: templated constructor struct TplCtorClass { template @@ -169,6 +214,23 @@ struct type_caster> } // namespace detail } // namespace PYBIND11_NAMESPACE +// test_array_element_types (at namespace scope because local classes cannot have member templates) +// Trivially copyable, but a constructor template takes over the construction from a non-const +// lvalue, which is what the caster for a registered type yields. It adds 100, idempotently. +struct CtorTemplateLvalue { + CtorTemplateLvalue() = default; + explicit CtorTemplateLvalue(int val) : val{val} {} + template + // NOLINTNEXTLINE(bugprone-forwarding-reference-overload, google-explicit-constructor) + CtorTemplateLvalue(U &&other) : val{other.val % 100 + 100} {} + int val; +}; + +template +std::vector array_element_vals(const std::array &a) { + return {a[0].val, a[1].val}; +} + int pass_std_vector_int(const std::vector &v) { int zum = 100; for (const int i : v) { @@ -215,11 +277,92 @@ TEST_SUBMODULE(stl, m) { }; // test_array_no_default_ctor - py::class_(m, "NoDefaultCtor").def_readonly("val", &NoDefaultCtor::val); + py::class_(m, "NoDefaultCtor") + .def(py::init()) + .def_readonly("val", &NoDefaultCtor::val); py::class_(m, "NoDefaultCtorArray") .def(py::init()) .def_readwrite("arr", &NoDefaultCtorArray::arr); + // test_array_element_types + // `array_caster` constructs the elements of a `std::array`, except for the element types + // for which assigning them instead cannot make a difference: `is_plain_array_element`. + // The class types below must all be constructed. They are trivially copyable, though, apart + // from `AssignIsNoOp`. + enum class ArrayElementEnum {}; + static_assert(py::detail::is_plain_array_element::value, ""); + static_assert(py::detail::is_plain_array_element::value, ""); + static_assert(py::detail::is_plain_array_element::value, ""); + static_assert(py::detail::is_plain_array_element>::value, ""); + static_assert(py::detail::is_plain_array_element< + std::array, 2>, 3>>::value, + ""); + static_assert(!py::detail::is_plain_array_element::value, ""); + static_assert(!py::detail::is_plain_array_element::value, ""); + static_assert(!py::detail::is_plain_array_element>::value, ""); + // Move assignable, but the elements of an array cannot be assigned: the caster for a type + // registered with `py::class_` yields an lvalue reference. + struct NotCopyAssignable { + NotCopyAssignable() = default; + explicit NotCopyAssignable(int val) : val{val} {} + NotCopyAssignable(const NotCopyAssignable &) = default; + NotCopyAssignable &operator=(const NotCopyAssignable &) = delete; + NotCopyAssignable &operator=(NotCopyAssignable &&) = default; + int val; + }; + // Assigning is not equivalent to copy constructing, in the manner of `boost::multi_array`, + // which does not resize on assignment. + struct AssignIsNoOp { + AssignIsNoOp() = default; + explicit AssignIsNoOp(int val) : val{val} {} + AssignIsNoOp(const AssignIsNoOp &) = default; + AssignIsNoOp &operator=(const AssignIsNoOp &) { return *this; } + int val; + }; + // Trivially copyable, but the default constructor must not be called. + struct ThrowingDefaultCtor { + ThrowingDefaultCtor() { throw std::runtime_error("ThrowingDefaultCtor"); } + explicit ThrowingDefaultCtor(int val) : val{val} {} + int val = 0; + }; + py::enum_(m, "ArrayElementEnum"); + py::class_(m, "NotCopyAssignable").def(py::init()); + py::class_(m, "AssignIsNoOp").def(py::init()); + py::class_(m, "ThrowingDefaultCtor").def(py::init()); + py::class_(m, "CtorTemplateLvalue").def(py::init()); + m.def("load_array_enum", [](const std::array &a) { + return std::vector{static_cast(a[0]), static_cast(a[1])}; + }); + m.def("load_array_not_copy_assignable", &array_element_vals); + m.def("load_array_assign_is_no_op", &array_element_vals); + m.def("load_array_throwing_default_ctor", &array_element_vals); + m.def("load_array_no_default_ctor", &array_element_vals); + m.def("load_array_ctor_template_lvalue", &array_element_vals); + + m.def("cast_array_cast_only_element", []() { + return std::array{{CastOnlyElement{1}, CastOnlyElement{2}}}; + }); + + // test_array_nested + py::class_(m, "OpaqueConstIntArray"); + m.def("make_opaque_const_int_array", []() { return OpaqueConstIntArray{{3, 4}}; }); + m.def("load_array_opaque_const_int_array", + [](const std::array &a) { return a[0][0] + a[0][1]; }); + m.def("nested_numeric_array_cast_sequence", + [](std::array, 2>, 2> x) { return x; }); + m.def("nested_array_cast_sequence", + [](std::array, 2> x) { return x; }); + + // test_array_large + // #5765: this was very slow to compile when `array_caster` needed an `index_sequence` of + // `Size` elements for all element types. + m.def("sum_large_array", [](const std::array &a) { + return std::accumulate(a.begin(), a.end(), 0LL); + }); + + // test_array_custom_type + m.def("fixed_array3_cast_sequence", [](FixedArray3 x) { return x; }); + // test_valarray m.def("cast_valarray", []() { return std::valarray{1, 4, 9}; }); m.def("load_valarray", [](const std::valarray &v) { diff --git a/tests/test_stl.py b/tests/test_stl.py index 7c1f76daa6..e180da0912 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -193,6 +193,85 @@ def test_array_no_default_ctor(): assert [e.val for e in lst.arr] == [13, 23] lst.arr = m.NoDefaultCtorArray(4).arr assert [e.val for e in lst.arr] == [14, 24] + lst.arr = (m.NoDefaultCtor(i) for i in (5, 6)) + assert [e.val for e in lst.arr] == [5, 6] + with pytest.raises(TypeError): + lst.arr = [m.NoDefaultCtor(7), 8] + assert [e.val for e in lst.arr] == [5, 6] + + +@pytest.mark.parametrize( + ("load", "item_type", "offset"), + [ + pytest.param( + m.load_array_not_copy_assignable, + m.NotCopyAssignable, + 0, + id="not_copy_assignable", + ), + pytest.param( + m.load_array_assign_is_no_op, m.AssignIsNoOp, 0, id="assign_is_no_op" + ), + pytest.param( + m.load_array_throwing_default_ctor, + m.ThrowingDefaultCtor, + 0, + id="throwing_default_ctor", + ), + pytest.param( + m.load_array_no_default_ctor, m.NoDefaultCtor, 0, id="no_default_ctor" + ), + pytest.param( + m.load_array_ctor_template_lvalue, + m.CtorTemplateLvalue, + 100, + id="ctor_template_lvalue", + ), + ], +) +def test_array_element_types(load, item_type, offset): + # `offset` is what the constructor template adds when the elements are constructed. + assert load([item_type(1), item_type(2)]) == [1 + offset, 2 + offset] + with pytest.raises(TypeError): + load([item_type(1)]) + with pytest.raises(TypeError): + load([item_type(1), 2]) + + +def test_array_cast_only_element(): + assert m.cast_array_cast_only_element() == [1, 2] + + +def test_array_nested(): + assert m.nested_array_cast_sequence([("a", "b"), ["c", "d"]]) == [ + ["a", "b"], + ["c", "d"], + ] + with pytest.raises(TypeError): + m.nested_array_cast_sequence([("a", "b"), ("c",)]) + assert m.nested_numeric_array_cast_sequence([(1, 2j), [3.5, 4]]) == [ + [1, 2j], + [3.5, 4], + ] + with pytest.raises(TypeError): + m.nested_numeric_array_cast_sequence([(1, 2j), [3.5]]) + assert m.load_array_opaque_const_int_array([m.make_opaque_const_int_array()]) == 7 + + +def test_array_large(): + size = 102400 + assert m.sum_large_array(range(size)) == (size - 1) * size // 2 + with pytest.raises(TypeError): + m.sum_large_array(range(size - 1)) + with pytest.raises(TypeError): + m.sum_large_array([0] * (size - 1) + ["x"]) + + +def test_array_custom_type(): + """A fixed-size `ArrayType` other than `std::array` only has to be constructible.""" + assert m.fixed_array3_cast_sequence([1, 2, 3]) == [1.0, 2.0, 3.0] + with pytest.raises(TypeError): + m.fixed_array3_cast_sequence([1, 2]) def test_valarray(doc): @@ -201,6 +280,10 @@ def test_valarray(doc): assert lst == [1, 4, 9] assert m.load_valarray(lst) assert m.load_valarray(tuple(lst)) + assert m.valarray_cast_sequence(i * i for i in range(4)) == [0, 1, 4, 9] + assert m.valarray_cast_sequence([]) == [] + with pytest.raises(TypeError): + m.valarray_cast_sequence([1, "x"]) assert doc(m.cast_valarray) == "cast_valarray() -> list[int]" assert ( @@ -585,6 +668,16 @@ def test_array_cast_sequence(): assert m.array_cast_sequence((1, 2, 3)) == [1, 2, 3] +# (load, item_type, size): one entry for each of the element loops in `array_caster`. +ARRAY_CASTER_LOADS = [ + pytest.param(m.load_array_enum, m.ArrayElementEnum, 2, id="array_assign"), + pytest.param(m.valarray_cast_sequence, int, 3, id="valarray"), + pytest.param( + m.load_array_no_default_ctor, m.NoDefaultCtor, 2, id="array_construct" + ), +] + + class SequenceWithChangingLen: """Reports `lens[0]` when first asked for its length, `lens[1]` after that.""" @@ -601,24 +694,37 @@ def __getitem__(self, index): return self.items[index] -ARRAY_CASTER_LOADS = [m.array_cast_sequence, m.valarray_cast_sequence] - - -@pytest.mark.parametrize("load", ARRAY_CASTER_LOADS) +@pytest.mark.parametrize(("load", "item_type", "size"), ARRAY_CASTER_LOADS) @pytest.mark.parametrize("len_change", [-1, 1]) -def test_array_caster_len_changes(load, len_change): +def test_array_caster_len_changes(load, item_type, size, len_change): """The length is determined only once, which rules out out-of-bounds accesses.""" - seq = SequenceWithChangingLen(list(range(4)), [3, 3 + len_change]) - assert load(seq) == [0, 1, 2] - assert seq.indices == [0, 1, 2] + items = [item_type(i) for i in range(size + 1)] + seq = SequenceWithChangingLen(items, [size, size + len_change]) + assert load(seq) == list(range(size)) + assert seq.indices == list(range(size)) -@pytest.mark.parametrize("load", ARRAY_CASTER_LOADS) -def test_array_caster_len_exceeds_items(load): - seq = SequenceWithChangingLen([0, 1], [3]) +@pytest.mark.parametrize(("load", "item_type", "size"), ARRAY_CASTER_LOADS) +def test_array_caster_len_exceeds_items(load, item_type, size): + items = [item_type(i) for i in range(size - 1)] + seq = SequenceWithChangingLen(items, [size]) with pytest.raises(IndexError): load(seq) - assert seq.indices == [0, 1, 2] + assert seq.indices == list(range(size)) + + +@pytest.mark.parametrize(("load", "item_type", "size"), ARRAY_CASTER_LOADS) +def test_array_caster_items_created_on_the_fly(load, item_type, size): + """Each item is kept alive until its value was copied (no use-after-free).""" + + class OnTheFly: + def __len__(self): + return size + + def __getitem__(self, index): + return item_type(index + 10) # The sequence does not retain the item. + + assert load(OnTheFly()) == list(range(10, 10 + size)) def test_issue_1561():