Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions docs/advanced/cast/custom.rst
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,15 @@ type is explicitly allowed.
intended only for type-caster implementations. See
:ref:`string_view_lifetime` for a concrete example.

.. note::

If a custom ``load()`` instead produces a non-owning view of storage that
the type caster itself owns (for example, a span over a ``std::vector``
member), add ``static constexpr bool refers_to_caster = true;`` to the type
caster. The STL container casters then keep the type caster of each element
alive, so that a ``std::vector`` of such views stays valid for the duration
of a bound-function call.

.. warning::

When using custom type casters, it's important to declare them consistently
Expand Down
6 changes: 3 additions & 3 deletions docs/limitations.rst
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,10 @@ Known limitations
These are issues that are probably solvable, but have not been fixed yet. A
clean, well written patch would likely be accepted to solve them.

- Type casters are not kept alive recursively.
- ``py::cast<T>()`` does not keep type casters alive. A ``T`` that refers to
data that its type caster owns, such as a container of ``char *``, is valid
only as an argument of a bound function.
`#2527 <https://github.com/pybind/pybind11/issues/2527>`_
One consequence is that containers of ``char *`` are currently not supported.
`#2245 <https://github.com/pybind/pybind11/issues/2245>`_

Python 3.9.0 warning
^^^^^^^^^^^^^^^^^^^^
Expand Down
16 changes: 16 additions & 0 deletions include/pybind11/cast.h
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,10 @@ bool isinstance_native_enum(handle obj, const std::type_info &tp) {
return isinstance_native_enum_impl<intrinsic_t<T>>(obj, tp);
}

// Defined below, after `cast_is_temporary_value_reference`.
template <typename T, typename SFINAE = void>
struct cast_refers_to_caster;

template <typename type>
class type_caster<std::reference_wrapper<type>> {
private:
Expand All @@ -176,6 +180,7 @@ class type_caster<std::reference_wrapper<type>> {
public:
bool load(handle src, bool convert) { return subcaster.load(src, convert); }
static constexpr auto name = caster_t::name;
static constexpr bool refers_to_caster = cast_refers_to_caster<reference_t>::value;
static handle
cast(const std::reference_wrapper<type> &src, return_value_policy policy, handle parent) {
// It is definitely wrong to take ownership of this pointer, so mask that rvp
Expand Down Expand Up @@ -800,6 +805,7 @@ class tuple_caster {
static constexpr auto name = const_name("tuple[")
+ ::pybind11::detail::concat(make_caster<Ts>::name...)
+ const_name("]");
static constexpr bool refers_to_caster = any_of<cast_refers_to_caster<Ts>...>::value;

template <typename T>
using cast_op_type = type;
Expand Down Expand Up @@ -1637,6 +1643,16 @@ using cast_is_temporary_value_reference
&& !std::is_base_of<type_caster_generic, make_caster<type>>::value
&& !std::is_same<intrinsic_t<type>, void>::value>;

// Detect whether the result of a cast to `T` refers to data that the type_caster owns. The
// type_caster must then stay alive for as long as the result is in use. This is the case for
// `cast_is_temporary_value_reference`, and for a type_caster that has a
// `static constexpr bool refers_to_caster = true` member. A type_caster needs that member if
// it returns, by value, a type that does not own its data (a container of pointers, a span).
template <typename T, typename SFINAE>
struct cast_refers_to_caster : cast_is_temporary_value_reference<T> {};
template <typename T>
struct cast_refers_to_caster<T, enable_if_t<make_caster<T>::refers_to_caster>> : std::true_type {};

// When a value returned from a C++ function is being cast back to Python, we almost always want to
// force `policy = move`, regardless of the return value policy the function/method was declared
// with.
Expand Down
101 changes: 90 additions & 11 deletions include/pybind11/stl.h
Original file line number Diff line number Diff line change
Expand Up @@ -167,10 +167,47 @@ constexpr forwarded_type<T, U> forward_like(U &&u) {
template <typename C>
using has_reserve_method = std::is_same<decltype(std::declval<C>().reserve(0)), void>;

// Provides the type_caster for each element that a container type_caster loads. The element
// type_casters stay alive with this object only if the elements refer to them, for example the
// elements of `std::vector<const char *>`.
template <typename T, bool Keep = cast_refers_to_caster<T>::value>
class element_casters {
public:
struct slot {
// Value-initialized, as the kept type_casters are.
explicit slot(element_casters &) : caster() {}
make_caster<T> caster;
};
void clear() {}
};

template <typename T>
class element_casters<T, true> {
public:
struct slot {
explicit slot(element_casters &owner) : caster(owner.add()) {}
make_caster<T> &caster;
};
void clear() { kept.clear(); }

element_casters() = default;
// Not copyable: the loaded elements refer to the type_casters of the original.
element_casters(element_casters &&) = default;
element_casters &operator=(element_casters &&) = default;

private:
make_caster<T> &add() {
kept.emplace_back();
return kept.back();
}
std::list<make_caster<T>> kept; // The elements refer to the nodes, which do not move.
};

template <typename Type, typename Key>
struct set_caster {
using type = Type;
using key_conv = make_caster<Key>;
static constexpr bool refers_to_caster = cast_refers_to_caster<Key>::value;

private:
template <typename T = Type, enable_if_t<has_reserve_method<T>::value, int> = 0>
Expand All @@ -179,9 +216,13 @@ struct set_caster {
}
void reserve_maybe(const anyset &, void *) {}

element_casters<Key> key_casters;

bool convert_iterable(const iterable &itbl, bool convert) {
key_casters.clear();
for (const auto &it : itbl) {
key_conv conv;
typename element_casters<Key>::slot slot(key_casters);
key_conv &conv = slot.caster;
if (!conv.load(it, convert)) {
return false;
}
Expand Down Expand Up @@ -238,8 +279,13 @@ template <typename Type, typename Key, typename Value>
struct map_caster {
using key_conv = make_caster<Key>;
using value_conv = make_caster<Value>;
static constexpr bool refers_to_caster
= any_of<cast_refers_to_caster<Key>, cast_refers_to_caster<Value>>::value;

private:
element_casters<Key> key_casters;
element_casters<Value> value_casters;

template <typename T = Type, enable_if_t<has_reserve_method<T>::value, int> = 0>
void reserve_maybe(const dict &d, Type *) {
value.reserve(d.size());
Expand All @@ -248,10 +294,14 @@ struct map_caster {

bool convert_elements(const dict &d, bool convert) {
value.clear();
key_casters.clear();
value_casters.clear();
reserve_maybe(d, &value);
for (const auto &it : d) {
key_conv kconv;
value_conv vconv;
typename element_casters<Key>::slot kslot(key_casters);
typename element_casters<Value>::slot vslot(value_casters);
key_conv &kconv = kslot.caster;
value_conv &vconv = vslot.caster;
if (!kconv.load(it.first.ptr(), convert) || !vconv.load(it.second.ptr(), convert)) {
return false;
}
Expand Down Expand Up @@ -310,6 +360,7 @@ struct map_caster {
template <typename Type, typename Value>
struct list_caster {
using value_conv = make_caster<Value>;
static constexpr bool refers_to_caster = cast_refers_to_caster<Value>::value;

bool load(handle src, bool convert) {
if (!object_is_convertible_to_std_vector(src)) {
Expand All @@ -330,6 +381,8 @@ struct list_caster {
}

private:
element_casters<Value> value_casters;

template <typename T = Type, enable_if_t<has_reserve_method<T>::value, int> = 0>
void reserve_maybe(const sequence &s, Type *) {
value.reserve(s.size());
Expand All @@ -339,9 +392,11 @@ struct list_caster {
bool convert_elements(handle seq, bool convert) {
auto s = reinterpret_borrow<sequence>(seq);
value.clear();
value_casters.clear();
reserve_maybe(s, &value);
for (const auto &it : seq) {
value_conv conv;
typename element_casters<Value>::slot slot(value_casters);
value_conv &conv = slot.caster;
if (!conv.load(it, convert)) {
return false;
}
Expand Down Expand Up @@ -397,9 +452,11 @@ ArrayType vector_to_array(V &&v) {
template <typename ArrayType, typename Value, bool Resizable, size_t Size = 0>
struct array_caster {
using value_conv = make_caster<Value>;
static constexpr bool refers_to_caster = cast_refers_to_caster<Value>::value;

private:
std::unique_ptr<ArrayType> value;
element_casters<Value> value_casters;

// Both `convert_elements` overloads index `l` up to a size that is determined only once,
// rather than iterating over `l`: the accesses stay in bounds even if the length of `l`
Expand All @@ -415,8 +472,11 @@ struct array_caster {
// For `std::valarray`, this is a requirement:
// https://en.cppreference.com/w/cpp/named_req/NumericType
value->resize(size);
value_casters.clear();
for (size_t i = 0; i < size; i++) {
value_conv conv; // Constructed before the item is fetched.
// Constructed before the item is fetched.
typename element_casters<Value>::slot slot(value_casters);
value_conv &conv = slot.caster;
const object item = l[i]; // Keeps the item alive while `conv` is in use.
if (!conv.load(item, convert)) {
return false;
Expand All @@ -440,8 +500,11 @@ struct array_caster {
// more significant than the runtime overhead for the `temp` storage.
std::vector<Value> temp;
temp.reserve(size);
value_casters.clear();
for (size_t i = 0; i < size; i++) {
value_conv conv; // Constructed before the item is fetched.
// Constructed before the item is fetched.
typename element_casters<Value>::slot slot(value_casters);
value_conv &conv = slot.caster;
const object item = l[i]; // Keeps the item alive while `conv` is in use.
if (!conv.load(item, convert)) {
return false;
Expand Down Expand Up @@ -547,6 +610,7 @@ struct type_caster<std::unordered_map<Key, Value, Hash, Equal, Alloc>>
template <typename Type, typename Value = typename Type::value_type>
struct optional_caster {
using value_conv = make_caster<Value>;
static constexpr bool refers_to_caster = cast_refers_to_caster<Value>::value;

template <typename T>
static handle cast(T &&src, return_value_policy policy, handle parent) {
Expand All @@ -567,7 +631,9 @@ struct optional_caster {
if (src.is_none()) {
return true; // default-constructed value is already empty
}
value_conv inner_caster;
value_casters.clear();
typename element_casters<Value>::slot slot(value_casters);
value_conv &inner_caster = slot.caster;
if (!inner_caster.load(src, convert)) {
return false;
}
Expand All @@ -577,6 +643,9 @@ struct optional_caster {
}

PYBIND11_TYPE_CASTER(Type, value_conv::name | make_caster<none>::name);

private:
element_casters<Value> value_casters;
};

#if defined(PYBIND11_HAS_OPTIONAL)
Expand Down Expand Up @@ -629,14 +698,21 @@ struct variant_caster;
template <template <typename...> class V, typename... Ts>
struct variant_caster<V<Ts...>> {
static_assert(sizeof...(Ts) > 0, "Variant must consist of at least one alternative.");
static constexpr bool refers_to_caster = any_of<cast_refers_to_caster<Ts>...>::value;

template <typename U, typename... Us>
bool load_alternative(handle src, bool convert, type_list<U, Us...>) {
auto caster = make_caster<U>();
if (caster.load(src, convert)) {
value = cast_op<U>(std::move(caster));
return true;
auto &casters = std::get<sizeof...(Ts) - sizeof...(Us) - 1>(alternative_casters);
casters.clear();
{
typename element_casters<U>::slot slot(casters);
make_caster<U> &caster = slot.caster;
if (caster.load(src, convert)) {
value = cast_op<U>(std::move(caster));
return true;
}
}
casters.clear(); // Do not keep the type_caster of a rejected alternative.
return load_alternative(src, convert, type_list<Us...>{});
}

Expand All @@ -661,6 +737,9 @@ struct variant_caster<V<Ts...>> {

using Type = V<Ts...>;
PYBIND11_TYPE_CASTER(Type, ::pybind11::detail::union_concat(make_caster<Ts>::name...));

private:
std::tuple<element_casters<Ts>...> alternative_casters;
};

#if defined(PYBIND11_HAS_VARIANT)
Expand Down
Loading