diff --git a/compiler/rustc_mir_transform/src/capture_mut_vec.rs b/compiler/rustc_mir_transform/src/capture_mut_vec.rs new file mode 100644 index 0000000000000..e6791cc3de58d --- /dev/null +++ b/compiler/rustc_mir_transform/src/capture_mut_vec.rs @@ -0,0 +1,224 @@ +use rustc_data_structures::thin_vec::ThinVec; +use rustc_middle::mir::visit::{PlaceContext, Visitor}; +use rustc_middle::mir::*; +use rustc_middle::ty::{self, TyCtxt, TypeVisitableExt}; +use rustc_session::Session; +use rustc_span::sym; + +use crate::{MirPass, PassPolicy}; + +/// Experimental copy-in/copy-out promotion for a narrowly constrained `&mut Vec` argument. +/// +/// This deliberately only accepts arguments whose uses are direct `Vec` method receiver operands, +/// including at least one `Vec::push` in a loop. Moving a `Vec` header changes its address, so +/// allowing arbitrary uses could make a raw pointer to that header observably different. The pass +/// runs after borrow checking and drop elaboration, and explicitly restores the header on normal +/// and unwind exits. +pub(super) struct CaptureMutVec; + +impl<'tcx> MirPass<'tcx> for CaptureMutVec { + fn policy(&self, sess: &Session) -> PassPolicy { + PassPolicy::optimization(sess.mir_opt_level() >= 3) + } + + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let Some(vec_push) = tcx.get_diagnostic_item(sym::vec_push) else { return }; + let typing_env = body.typing_env(tcx); + + // Keep the first experiment simple: promote at most one concrete, small-element Vec. + let candidate = (1..=body.arg_count).map(Local::from_usize).find(|&arg| { + let ty::Ref(_, vec_ty, Mutability::Mut) = *body.local_decls[arg].ty.kind() else { + return false; + }; + let ty::Adt(def, args) = *vec_ty.kind() else { return false }; + if !tcx.is_diagnostic_item(sym::Vec, def.did()) { + return false; + } + let elem = args.type_at(0); + let Ok(layout) = tcx.layout_of(typing_env.as_query_input(elem)) else { return false }; + layout.size.bytes() <= 16 && only_used_by_vec_methods(tcx, body, arg, vec_push) + }); + let Some(arg) = candidate else { return }; + + let span = body.span; + let source_info = SourceInfo::outermost(span); + let ref_ty = body.local_decls[arg].ty; + let ty::Ref(_, vec_ty, Mutability::Mut) = *ref_ty.kind() else { unreachable!() }; + let owned = body.local_decls.push(LocalDecl::new(vec_ty, span)); + let local_ref = body.local_decls.push(LocalDecl::new(ref_ty, span)); + let deref_arg = Place::from(arg).project_deeper(&[ProjectionElem::Deref], tcx); + + // Rewrite the receiver reborrows before adding the prologue, which itself uses `arg`. + for data in body.basic_blocks.as_mut().iter_mut() { + for statement in &mut data.statements { + if let StatementKind::Assign(assign) = &mut statement.kind + && let Rvalue::Ref(_, _, place) = &mut assign.1 + && place.local == arg + { + place.local = local_ref; + } + } + } + + let start = &mut body.basic_blocks.as_mut()[START_BLOCK].statements; + start.insert( + 0, + Statement::new( + source_info, + StatementKind::Assign(Box::new(( + Place::from(owned), + Rvalue::Use(Operand::Move(deref_arg), WithRetag::Yes), + ))), + ), + ); + start.insert( + 1, + Statement::new( + source_info, + StatementKind::Assign(Box::new(( + Place::from(local_ref), + Rvalue::Ref( + tcx.lifetimes.re_erased, + BorrowKind::Mut { kind: MutBorrowKind::Default }, + Place::from(owned), + ), + ))), + ), + ); + + let restore = || { + Statement::new( + source_info, + StatementKind::Assign(Box::new(( + deref_arg, + Rvalue::Use(Operand::Move(Place::from(owned)), WithRetag::Yes), + ))), + ) + }; + + let original_blocks = body.basic_blocks.len(); + // A single cleanup restores the header before propagating any unwind. Do not create an + // unwind block for aborting targets, where all unwind actions have already been lowered. + let needs_cleanup = body + .basic_blocks + .iter() + .any(|data| matches!(data.terminator().unwind(), Some(UnwindAction::Continue))); + let cleanup = needs_cleanup.then(|| { + let mut cleanup_data = BasicBlockData::new( + Some(Terminator { + source_info, + kind: TerminatorKind::UnwindResume, + attributes: ThinVec::new(), + }), + true, + ); + cleanup_data.statements.push(restore()); + body.basic_blocks_mut().push(cleanup_data) + }); + + for data in body.basic_blocks.as_mut().iter_mut().take(original_blocks) { + if matches!( + data.terminator().kind, + TerminatorKind::Return | TerminatorKind::UnwindResume + ) { + data.statements.push(restore()); + } + if let Some(cleanup) = cleanup + && let Some(unwind @ UnwindAction::Continue) = data.terminator_mut().unwind_mut() + { + *unwind = UnwindAction::Cleanup(cleanup); + } + } + } +} + +fn only_used_by_vec_methods( + tcx: TyCtxt<'_>, + body: &Body<'_>, + arg: Local, + vec_push: rustc_hir::def_id::DefId, +) -> bool { + struct Uses { + arg: Local, + count: usize, + } + impl<'tcx> Visitor<'tcx> for Uses { + fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, location: Location) { + if place.local == self.arg && !matches!(context, PlaceContext::NonUse(_)) { + self.count += 1; + } + self.super_place(place, context, location); + } + } + + let mut uses = Uses { arg, count: 0 }; + uses.visit_body(body); + let mut receivers = Vec::new(); + for data in body.basic_blocks.iter() { + for statement in &data.statements { + if let StatementKind::Assign(assign) = &statement.kind + && assign.0.projection.is_empty() + && let Rvalue::Ref(_, _, place) = &assign.1 + && place.local == arg + { + receivers.push(assign.0.local); + } + } + } + let mut method_calls = 0; + let mut push_in_loop = false; + for (block, data) in body.basic_blocks.iter_enumerated() { + let TerminatorKind::Call { func, args, destination, .. } = &data.terminator().kind else { + continue; + }; + let Some(Operand::Move(receiver) | Operand::Copy(receiver)) = + args.first().map(|arg| &arg.node) + else { + continue; + }; + if !receiver.projection.is_empty() || !receivers.contains(&receiver.local) { + continue; + } + let Some((did, _)) = func.const_fn_def() else { continue }; + let Some(item) = tcx.opt_associated_item(did) else { continue }; + if !item.is_method() || !matches!(item.container, ty::AssocContainer::InherentImpl) { + continue; + } + // A method returning a value tied to the receiver may retain the address of the local + // header (for example, `Vec::drain`). Proving that such a value does not escape requires + // more dataflow than this initial pass performs. + if body.local_decls[destination.local].ty.has_regions() { + continue; + } + + method_calls += 1; + push_in_loop |= did == vec_push && block_is_in_cycle(body, block); + } + if !push_in_loop || uses.count != receivers.len() || method_calls != receivers.len() { + return false; + } + + // Each receiver temporary must occur exactly once as the assignment destination above and + // exactly once as the receiver of a direct `Vec` method. In particular, reject any extra + // escape of it. + receivers.into_iter().all(|receiver| { + let mut uses = Uses { arg: receiver, count: 0 }; + uses.visit_body(body); + uses.count == 2 + }) +} + +fn block_is_in_cycle(body: &Body<'_>, start: BasicBlock) -> bool { + let mut visited = vec![false; body.basic_blocks.len()]; + let mut pending: Vec<_> = body.basic_blocks[start].terminator().successors().collect(); + while let Some(block) = pending.pop() { + if block == start { + return true; + } + if !visited[block.index()] { + visited[block.index()] = true; + pending.extend(body.basic_blocks[block].terminator().successors()); + } + } + false +} diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index d2dd77c986318..5f8ff5dba4ac4 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -131,6 +131,7 @@ declare_passes! { mod check_const_item_mutation : CheckConstItemMutation; mod check_null : CheckNull; mod check_packed_ref : CheckPackedRef; + mod capture_mut_vec : CaptureMutVec; mod check_mut_restriction : CheckMutRestriction; // This pass is public to allow external drivers to perform MIR cleanup pub mod cleanup_post_borrowck : CleanupPostBorrowck; @@ -698,6 +699,7 @@ pub(crate) fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<' &check_alignment::CheckAlignment, &check_null::CheckNull, &check_enums::CheckEnums, + &capture_mut_vec::CaptureMutVec, // Before inlining: trim down MIR with passes to reduce inlining work. // Has to be done before inlining, otherwise actual call will be almost always inlined. diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 6376fe032c64e..fd03ef3cd6dd0 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -2333,6 +2333,7 @@ symbols! { variant_non_exhaustive, variants, vec, + vec_push, vector, verbatim, version, diff --git a/library/alloc/src/raw_vec/mod.rs b/library/alloc/src/raw_vec/mod.rs index 5d4ad3ac4bf98..65dd0bfc281e7 100644 --- a/library/alloc/src/raw_vec/mod.rs +++ b/library/alloc/src/raw_vec/mod.rs @@ -71,18 +71,21 @@ const unsafe fn new_cap(cap: usize) -> Cap { /// `Box<[T]>`, since `capacity()` won't yield the length. #[allow(missing_debug_implementations)] pub(crate) struct RawVec { - inner: RawVecInner, + inner: RawVecInner, + alloc: A, _marker: PhantomData, } -/// Like a `RawVec`, but only generic over the allocator, not the type. +/// Like a `RawVec`, but not generic over the allocator. /// -/// As such, all the methods need the layout passed-in as a parameter. +/// As such, all the methods need the layout passed-in as a parameter, +/// and methods that need an allocator receive it separately. /// /// Having this separation reduces the amount of code we need to monomorphize, /// as most operations don't need the actual type, just its layout. #[allow(missing_debug_implementations)] -struct RawVecInner { +#[derive(Clone, Copy)] +struct RawVecInner { ptr: Unique, /// Never used for ZSTs; it's `capacity()`'s responsibility to return usize::MAX in that case. /// @@ -90,7 +93,6 @@ struct RawVecInner { /// /// `cap` must be in the `0..=isize::MAX` range. cap: Cap, - alloc: A, } impl RawVec { @@ -123,7 +125,7 @@ impl RawVec { #[must_use] #[inline] pub(crate) fn with_capacity(capacity: usize) -> Self { - Self { inner: RawVecInner::with_capacity(capacity, T::LAYOUT), _marker: PhantomData } + Self::with_capacity_in(capacity, Global) } /// Like `with_capacity`, but guarantees the buffer is zeroed. @@ -131,22 +133,7 @@ impl RawVec { #[must_use] #[inline] pub(crate) fn with_capacity_zeroed(capacity: usize) -> Self { - Self { - inner: RawVecInner::with_capacity_zeroed_in(capacity, Global, T::LAYOUT), - _marker: PhantomData, - } - } -} - -impl RawVecInner { - #[cfg(not(any(no_global_oom_handling, test)))] - #[must_use] - #[inline] - fn with_capacity(capacity: usize, elem_layout: Layout) -> Self { - match Self::try_allocate_in(capacity, AllocInit::Uninitialized, Global, elem_layout) { - Ok(res) => res, - Err(err) => handle_error(err), - } + Self::with_capacity_zeroed_in(capacity, Global) } } @@ -174,7 +161,8 @@ const impl RawVec { #[inline] pub(crate) fn with_capacity_in(capacity: usize, alloc: A) -> Self { Self { - inner: RawVecInner::with_capacity_in(capacity, alloc, T::LAYOUT), + inner: RawVecInner::with_capacity_in(capacity, &alloc, T::LAYOUT), + alloc, _marker: PhantomData, } } @@ -182,10 +170,69 @@ const impl RawVec { /// A specialized version of `self.reserve(len, 1)` which requires the /// caller to ensure `len == self.capacity()`. #[cfg(not(no_global_oom_handling))] - #[inline(never)] + #[inline(always)] pub(crate) fn grow_one(&mut self) { - // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout - unsafe { self.inner.grow_one(T::LAYOUT) } + // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout. + self.inner = if A::IS_ZST { + RawVecInner::grow_one_zst_allocator::(self.inner, T::LAYOUT) + } else { + // Keep the allocator local so the containing `Vec` does not escape through `&A`. + // The guard restores it on normal return and unwind. + let local_alloc = CaptureLocally::new(&mut self.alloc); + let inner = unsafe { + RawVecInner::grow_one_outlined(self.inner, local_alloc.get(), T::LAYOUT) + }; + local_alloc.restore(); + inner + }; + } +} + +#[cfg(not(no_global_oom_handling))] +struct CaptureLocally<'a, T> { + value: ManuallyDrop, + old: *mut T, + _marker: PhantomData<&'a mut T>, +} + +#[rustc_const_unstable(feature = "const_heap", issue = "79597")] +#[cfg(not(no_global_oom_handling))] +const impl<'a, T: [const] Destruct> Drop for CaptureLocally<'a, T> { + fn drop(&mut self) { + // SAFETY: We have sole ownership of `self.old` during the lifetime of + // this struct, so writing back to it is safe. + unsafe { + let value = ptr::read((&raw const self.value).cast::()); + ptr::write(self.old, value); + } + } +} + +#[cfg(not(no_global_oom_handling))] +impl<'a, T> CaptureLocally<'a, T> { + const fn new(old: &'a mut T) -> Self { + // SAFETY: We are taking ownership of the value at `old`, given that we + // store the mut reference, we have sole ownership of it for the duration of + // this struct's lifetime. The Drop impl will write it back. + Self { value: ManuallyDrop::new(unsafe { ptr::read(old) }), old, _marker: PhantomData } + } + + const fn get(&self) -> &T { + // SAFETY: `ManuallyDrop` has the same layout and validity as `T`, + // and the guard owns the value until it is restored. + unsafe { &*((&raw const self.value).cast::()) } + } + + const fn restore(self) { + // Suppress `Drop` on the normal path so codegen can see the restoration + // directly. If the allocator operation unwinds before this call, the + // guard's `Drop` implementation performs the same restoration. + let this = ManuallyDrop::new(self); + let this = (&raw const this).cast::(); + unsafe { + let value = ptr::read((&raw const (*this).value).cast::()); + ptr::write((*this).old, value); + } } } @@ -199,15 +246,15 @@ impl RawVec { pub(crate) const fn new_in(alloc: A) -> Self { // Check assumption made in `current_memory` const { assert!(T::LAYOUT.size() % T::LAYOUT.align() == 0) }; - Self { inner: RawVecInner::new_in(alloc, Alignment::of::()), _marker: PhantomData } + Self { inner: RawVecInner::new(Alignment::of::()), alloc, _marker: PhantomData } } /// Like `try_with_capacity`, but parameterized over the choice of /// allocator for the returned `RawVec`. #[inline] pub(crate) fn try_with_capacity_in(capacity: usize, alloc: A) -> Result { - match RawVecInner::try_with_capacity_in(capacity, alloc, T::LAYOUT) { - Ok(inner) => Ok(Self { inner, _marker: PhantomData }), + match RawVecInner::try_allocate_in(capacity, AllocInit::Uninitialized, &alloc, T::LAYOUT) { + Ok(inner) => Ok(Self { inner, alloc, _marker: PhantomData }), Err(e) => Err(e), } } @@ -217,9 +264,9 @@ impl RawVec { #[cfg(not(no_global_oom_handling))] #[inline] pub(crate) fn with_capacity_zeroed_in(capacity: usize, alloc: A) -> Self { - Self { - inner: RawVecInner::with_capacity_zeroed_in(capacity, alloc, T::LAYOUT), - _marker: PhantomData, + match RawVecInner::try_allocate_in(capacity, AllocInit::Zeroed, &alloc, T::LAYOUT) { + Ok(inner) => Self { inner, alloc, _marker: PhantomData }, + Err(err) => handle_error(err), } } @@ -245,7 +292,7 @@ impl RawVec { let me = ManuallyDrop::new(self); unsafe { let slice = me.ptr().cast::>().cast_slice(len); - Box::from_raw_in(slice, ptr::read(&me.inner.alloc)) + Box::from_raw_in(slice, ptr::read(&me.alloc)) } } @@ -265,10 +312,7 @@ impl RawVec { unsafe { let ptr = ptr.cast(); let capacity = new_cap::(capacity); - Self { - inner: RawVecInner::from_raw_parts_in(ptr, capacity, alloc), - _marker: PhantomData, - } + Self { inner: RawVecInner::from_raw_parts(ptr, capacity), alloc, _marker: PhantomData } } } @@ -284,7 +328,7 @@ impl RawVec { unsafe { let ptr = ptr.cast(); let capacity = new_cap::(capacity); - Self { inner: RawVecInner::from_nonnull_in(ptr, capacity, alloc), _marker: PhantomData } + Self { inner: RawVecInner::from_nonnull(ptr, capacity), alloc, _marker: PhantomData } } } @@ -312,7 +356,7 @@ impl RawVec { /// Returns a shared reference to the allocator backing this `RawVec`. #[inline] pub(crate) const fn allocator(&self) -> &A { - self.inner.allocator() + &self.alloc } /// Ensures that the buffer contains at least enough space to hold `len + @@ -338,17 +382,18 @@ impl RawVec { #[inline] pub(crate) fn reserve(&mut self, len: usize, additional: usize) { // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout - unsafe { self.inner.reserve(len, additional, T::LAYOUT) } + unsafe { self.inner.reserve(len, additional, T::LAYOUT, &self.alloc) } } /// The same as `reserve`, but returns on errors instead of panicking or aborting. + #[inline] pub(crate) fn try_reserve( &mut self, len: usize, additional: usize, ) -> Result<(), TryReserveError> { // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout - unsafe { self.inner.try_reserve(len, additional, T::LAYOUT) } + unsafe { self.inner.try_reserve(len, additional, T::LAYOUT, &self.alloc) } } /// Ensures that the buffer contains at least enough space to hold `len + @@ -369,19 +414,21 @@ impl RawVec { /// /// Aborts on OOM. #[cfg(not(no_global_oom_handling))] + #[inline] pub(crate) fn reserve_exact(&mut self, len: usize, additional: usize) { // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout - unsafe { self.inner.reserve_exact(len, additional, T::LAYOUT) } + unsafe { self.inner.reserve_exact(len, additional, T::LAYOUT, &self.alloc) } } /// The same as `reserve_exact`, but returns on errors instead of panicking or aborting. + #[inline] pub(crate) fn try_reserve_exact( &mut self, len: usize, additional: usize, ) -> Result<(), TryReserveError> { // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout - unsafe { self.inner.try_reserve_exact(len, additional, T::LAYOUT) } + unsafe { self.inner.try_reserve_exact(len, additional, T::LAYOUT, &self.alloc) } } /// Shrinks the buffer down to the specified capacity. If the given amount @@ -398,7 +445,7 @@ impl RawVec { #[inline] pub(crate) fn shrink_to_fit(&mut self, cap: usize) { // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout - unsafe { self.inner.shrink_to_fit(cap, T::LAYOUT) } + unsafe { self.inner.shrink_to_fit(cap, T::LAYOUT, &self.alloc) } } /// Shrinks the buffer down to the specified capacity. If the given amount @@ -413,41 +460,45 @@ impl RawVec { /// Panics if the given amount is *larger* than the current capacity. #[inline] pub(crate) fn try_shrink_to_fit(&mut self, cap: usize) -> Result<(), TryReserveError> { - unsafe { self.inner.try_shrink_to_fit(cap, T::LAYOUT) } + unsafe { self.inner.try_shrink_to_fit(cap, T::LAYOUT, &self.alloc) } } } #[rustc_const_unstable(feature = "const_heap", issue = "79597")] const unsafe impl<#[may_dangle] T, A: [const] Allocator + [const] Destruct> Drop for RawVec { /// Frees the memory owned by the `RawVec` *without* trying to drop its contents. + #[inline(always)] fn drop(&mut self) { - // SAFETY: We are in a Drop impl, self.inner will not be used again. - unsafe { self.inner.deallocate(T::LAYOUT) } + // SAFETY: We are in a Drop impl, self.inner and self.alloc will not be used again. + // Copy to temporaries to prevent &self from escaping. + let inner = self.inner; + unsafe { inner.deallocate(T::LAYOUT, &self.alloc) } } } #[rustc_const_unstable(feature = "const_heap", issue = "79597")] #[rustfmt::skip] // FIXME(fee1-dead): temporary measure before rustfmt is bumped -const impl RawVecInner { +const impl RawVecInner { #[cfg(not(no_global_oom_handling))] #[inline] - fn with_capacity_in(capacity: usize, alloc: A, elem_layout: Layout) -> Self { - match Self::try_allocate_in(capacity, AllocInit::Uninitialized, alloc, elem_layout) { - Ok(this) => { + pub(crate) fn with_capacity_in(capacity: usize, alloc: &A, elem_layout: Layout) -> Self { + match RawVecInner::try_allocate_in(capacity, AllocInit::Uninitialized, alloc, elem_layout) { + Ok(inner) => { unsafe { // Make it more obvious that a subsequent Vec::reserve(capacity) will not allocate. - hint::assert_unchecked(!this.needs_to_grow(0, capacity, elem_layout)); + hint::assert_unchecked(!inner.needs_to_grow(0, capacity, elem_layout)); } - this + inner } Err(err) => handle_error(err), } } - fn try_allocate_in( + #[inline] + fn try_allocate_in( capacity: usize, init: AllocInit, - alloc: A, + alloc: &A, elem_layout: Layout, ) -> Result { // We avoid `unwrap_or_else` here because it bloats the amount of @@ -459,7 +510,7 @@ const impl RawVecInner { // Don't allocate here because `Drop` will not deallocate when `capacity` is 0. if layout.size() == 0 { - return Ok(Self::new_in(alloc, elem_layout.alignment())); + return Ok(Self::new(elem_layout.alignment())); } let result = match init { @@ -478,20 +529,39 @@ const impl RawVecInner { Ok(Self { ptr: Unique::from(ptr.cast()), cap: unsafe { Cap::new_unchecked(capacity) }, - alloc, }) } + #[cfg(not(no_global_oom_handling))] + #[inline(never)] + fn grow_one_zst_allocator(self, elem_layout: Layout) -> Self { + debug_assert!(A::IS_ZST); + let alloc = unsafe { NonNull::::dangling().as_ref() }; + unsafe { self.grow_one(alloc, elem_layout) } + } + /// # Safety - /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to - /// initially construct `self` - /// - `elem_layout`'s size must be a multiple of its alignment + /// `elem_layout` must be the layout used to create this allocation. #[cfg(not(no_global_oom_handling))] - #[inline] - unsafe fn grow_one(&mut self, elem_layout: Layout) { - // SAFETY: Precondition passed to caller - if let Err(err) = unsafe { self.grow_amortized(self.cap.as_inner(), 1, elem_layout) } { - handle_error(err); + #[inline(never)] + unsafe fn grow_one_outlined( + self, + alloc: &A, + elem_layout: Layout, + ) -> Self { + unsafe { self.grow_one(alloc, elem_layout) } + } + + #[cfg(not(no_global_oom_handling))] + #[inline(always)] + unsafe fn grow_one( + self, + alloc: &A, + elem_layout: Layout, + ) -> Self { + match unsafe { self.grow_amortized(self.cap.as_inner(), 1, elem_layout, alloc) } { + Ok(inner) => inner, + Err(err) => handle_error(err), } } @@ -500,12 +570,13 @@ const impl RawVecInner { /// initially construct `self` /// - `elem_layout`'s size must be a multiple of its alignment /// - The sum of `len` and `additional` must be greater than the current capacity - unsafe fn grow_amortized( - &mut self, + unsafe fn grow_amortized( + mut self, len: usize, additional: usize, elem_layout: Layout, - ) -> Result<(), TryReserveError> { + alloc: &A, + ) -> Result { // This is ensured by the calling contexts. debug_assert!(additional > 0); @@ -526,11 +597,11 @@ const impl RawVecInner { // SAFETY: // - cap >= len + additional // - other preconditions passed to caller - let ptr = unsafe { self.finish_grow(cap, elem_layout)? }; + let ptr = unsafe { self.finish_grow(cap, elem_layout, alloc)? }; // SAFETY: `finish_grow` would have failed if `cap > isize::MAX` unsafe { self.set_ptr_and_cap(ptr, cap) }; - Ok(()) + Ok(self) } /// # Safety @@ -541,10 +612,11 @@ const impl RawVecInner { // not marked inline(never) since we want optimizers to be able to observe the specifics of this // function, see tests/codegen-llvm/vec-reserve-extend.rs. #[cold] - unsafe fn finish_grow( + unsafe fn finish_grow( &self, cap: usize, elem_layout: Layout, + alloc: &A, ) -> Result, TryReserveError> { let new_layout = layout_array(cap, elem_layout)?; @@ -554,51 +626,48 @@ const impl RawVecInner { unsafe { // The allocator checks for alignment equality hint::assert_unchecked(old_layout.align() == new_layout.align()); - self.alloc.grow(ptr, old_layout, new_layout) + alloc.grow(ptr, old_layout, new_layout) } } else { - self.alloc.allocate(new_layout) + alloc.allocate(new_layout) }; - memory.map_err(const |_| AllocError { layout: new_layout, non_exhaustive: () }.into()) - } -} - -impl RawVecInner { - #[inline] - const fn new_in(alloc: A, align: Alignment) -> Self { - let ptr = Unique::from_non_null(NonNull::without_provenance(align.as_nonzero_usize())); - // `cap: 0` means "unallocated". zero-sized types are ignored. - Self { ptr, cap: ZERO_CAP, alloc } + match memory { + Ok(memory) => Ok(memory), + Err(_) => Err(AllocError { layout: new_layout, non_exhaustive: () }.into()), + } } + /// # Safety + /// + /// This should only be called once for a given `RawVecInner`. After this function any copies + /// of this `RawVecInner` are invalidated. #[inline] - fn try_with_capacity_in( - capacity: usize, - alloc: A, - elem_layout: Layout, - ) -> Result { - Self::try_allocate_in(capacity, AllocInit::Uninitialized, alloc, elem_layout) + unsafe fn deallocate(self, elem_layout: Layout, alloc: &A) { + // SAFETY: Precondition passed to caller. + if let Some((ptr, layout)) = unsafe { self.current_memory(elem_layout) } { + unsafe { alloc.deallocate(ptr, layout) }; + } } +} - #[cfg(not(no_global_oom_handling))] +impl RawVecInner { #[inline] - fn with_capacity_zeroed_in(capacity: usize, alloc: A, elem_layout: Layout) -> Self { - match Self::try_allocate_in(capacity, AllocInit::Zeroed, alloc, elem_layout) { - Ok(res) => res, - Err(err) => handle_error(err), - } + const fn new(align: Alignment) -> Self { + let ptr = Unique::from_non_null(NonNull::without_provenance(align.as_nonzero_usize())); + // `cap: 0` means "unallocated". zero-sized types are ignored. + Self { ptr, cap: ZERO_CAP } } #[inline] - const unsafe fn from_raw_parts_in(ptr: *mut u8, cap: Cap, alloc: A) -> Self { - Self { ptr: unsafe { Unique::new_unchecked(ptr) }, cap, alloc } + const unsafe fn from_raw_parts(ptr: *mut u8, cap: Cap) -> Self { + Self { ptr: unsafe { Unique::new_unchecked(ptr) }, cap } } #[inline] #[rustc_const_unstable(feature = "const_heap", issue = "79597")] - const unsafe fn from_nonnull_in(ptr: NonNull, cap: Cap, alloc: A) -> Self { - Self { ptr: Unique::from(ptr), cap, alloc } + const unsafe fn from_nonnull(ptr: NonNull, cap: Cap) -> Self { + Self { ptr: Unique::from(ptr), cap } } #[inline] @@ -616,11 +685,6 @@ impl RawVecInner { if elem_size == 0 { usize::MAX } else { self.cap.as_inner() } } - #[inline] - const fn allocator(&self) -> &A { - &self.alloc - } - /// # Safety /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to /// initially construct `self` @@ -649,27 +713,35 @@ impl RawVecInner { /// - `elem_layout`'s size must be a multiple of its alignment #[cfg(not(no_global_oom_handling))] #[inline] - unsafe fn reserve(&mut self, len: usize, additional: usize, elem_layout: Layout) { + unsafe fn reserve( + &mut self, + len: usize, + additional: usize, + elem_layout: Layout, + alloc: &A, + ) { // Callers expect this function to be very cheap when there is already sufficient capacity. // Therefore, we move all the resizing and error-handling logic from grow_amortized and // handle_reserve behind a call, while making sure that this function is likely to be // inlined as just a comparison and a call if the comparison fails. #[cold] - unsafe fn do_reserve_and_handle( - slf: &mut RawVecInner, + unsafe fn do_reserve_and_handle( + slf: &mut RawVecInner, len: usize, additional: usize, elem_layout: Layout, + alloc: &A, ) { // SAFETY: Precondition passed to caller - if let Err(err) = unsafe { slf.grow_amortized(len, additional, elem_layout) } { - handle_error(err); + match unsafe { slf.grow_amortized(len, additional, elem_layout, alloc) } { + Ok(inner) => *slf = inner, + Err(err) => handle_error(err), } } if self.needs_to_grow(len, additional, elem_layout) { unsafe { - do_reserve_and_handle(self, len, additional, elem_layout); + do_reserve_and_handle(self, len, additional, elem_layout, alloc); } } } @@ -678,20 +750,21 @@ impl RawVecInner { /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to /// initially construct `self` /// - `elem_layout`'s size must be a multiple of its alignment - unsafe fn try_reserve( + #[inline] + unsafe fn try_reserve( &mut self, len: usize, additional: usize, elem_layout: Layout, + alloc: &A, ) -> Result<(), TryReserveError> { if self.needs_to_grow(len, additional, elem_layout) { // SAFETY: Precondition passed to caller - unsafe { - self.grow_amortized(len, additional, elem_layout)?; - } + let inner = unsafe { self.grow_amortized(len, additional, elem_layout, alloc)? }; + *self = inner; } unsafe { - // Inform the optimizer that the reservation has succeeded or wasn't needed + // Inform the optimizer that the reservation has succeeded or wasn't needed. hint::assert_unchecked(!self.needs_to_grow(len, additional, elem_layout)); } Ok(()) @@ -702,9 +775,15 @@ impl RawVecInner { /// initially construct `self` /// - `elem_layout`'s size must be a multiple of its alignment #[cfg(not(no_global_oom_handling))] - unsafe fn reserve_exact(&mut self, len: usize, additional: usize, elem_layout: Layout) { + unsafe fn reserve_exact( + &mut self, + len: usize, + additional: usize, + elem_layout: Layout, + alloc: &A, + ) { // SAFETY: Precondition passed to caller - if let Err(err) = unsafe { self.try_reserve_exact(len, additional, elem_layout) } { + if let Err(err) = unsafe { self.try_reserve_exact(len, additional, elem_layout, alloc) } { handle_error(err); } } @@ -713,20 +792,22 @@ impl RawVecInner { /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to /// initially construct `self` /// - `elem_layout`'s size must be a multiple of its alignment - unsafe fn try_reserve_exact( + #[inline] + unsafe fn try_reserve_exact( &mut self, len: usize, additional: usize, elem_layout: Layout, + alloc: &A, ) -> Result<(), TryReserveError> { if self.needs_to_grow(len, additional, elem_layout) { // SAFETY: Precondition passed to caller unsafe { - self.grow_exact(len, additional, elem_layout)?; + self.grow_exact(len, additional, elem_layout, alloc)?; } } unsafe { - // Inform the optimizer that the reservation has succeeded or wasn't needed + // Inform the optimizer that the reservation has succeeded or wasn't needed. hint::assert_unchecked(!self.needs_to_grow(len, additional, elem_layout)); } Ok(()) @@ -739,8 +820,8 @@ impl RawVecInner { /// - `cap` must be less than or equal to `self.capacity(elem_layout.size())` #[cfg(not(no_global_oom_handling))] #[inline] - unsafe fn shrink_to_fit(&mut self, cap: usize, elem_layout: Layout) { - if let Err(err) = unsafe { self.shrink(cap, elem_layout) } { + unsafe fn shrink_to_fit(&mut self, cap: usize, elem_layout: Layout, alloc: &A) { + if let Err(err) = unsafe { self.shrink(cap, elem_layout, alloc) } { handle_error(err); } } @@ -751,12 +832,13 @@ impl RawVecInner { /// initially construct `self` /// - `elem_layout`'s size must be a multiple of its alignment /// - `cap` must be less than or equal to `self.capacity(elem_layout.size())` - unsafe fn try_shrink_to_fit( + unsafe fn try_shrink_to_fit( &mut self, cap: usize, elem_layout: Layout, + alloc: &A, ) -> Result<(), TryReserveError> { - unsafe { self.shrink(cap, elem_layout) } + unsafe { self.shrink(cap, elem_layout, alloc) } } #[inline] @@ -779,11 +861,12 @@ impl RawVecInner { /// initially construct `self` /// - `elem_layout`'s size must be a multiple of its alignment /// - The sum of `len` and `additional` must be greater than the current capacity - unsafe fn grow_exact( + unsafe fn grow_exact( &mut self, len: usize, additional: usize, elem_layout: Layout, + alloc: &A, ) -> Result<(), TryReserveError> { if elem_layout.size() == 0 { // Since we return a capacity of `usize::MAX` when the type size is @@ -794,7 +877,7 @@ impl RawVecInner { let cap = len.checked_add(additional).ok_or(CapacityOverflow)?; // SAFETY: preconditions passed to caller - let ptr = unsafe { self.finish_grow(cap, elem_layout)? }; + let ptr = unsafe { self.finish_grow(cap, elem_layout, alloc)? }; // SAFETY: `finish_grow` would have failed if `cap > isize::MAX` unsafe { self.set_ptr_and_cap(ptr, cap) }; @@ -807,10 +890,15 @@ impl RawVecInner { /// - `elem_layout`'s size must be a multiple of its alignment /// - `cap` must be less than or equal to `self.capacity(elem_layout.size())` #[inline] - unsafe fn shrink(&mut self, cap: usize, elem_layout: Layout) -> Result<(), TryReserveError> { + unsafe fn shrink( + &mut self, + cap: usize, + elem_layout: Layout, + alloc: &A, + ) -> Result<(), TryReserveError> { assert!(cap <= self.capacity(elem_layout.size()), "Tried to shrink to a larger capacity"); // SAFETY: Just checked this isn't trying to grow - unsafe { self.shrink_unchecked(cap, elem_layout) } + unsafe { self.shrink_unchecked(cap, elem_layout, alloc) } } /// `shrink`, but without the capacity check. @@ -823,10 +911,11 @@ impl RawVecInner { /// /// # Safety /// `cap <= self.capacity()` - unsafe fn shrink_unchecked( + unsafe fn shrink_unchecked( &mut self, cap: usize, elem_layout: Layout, + alloc: &A, ) -> Result<(), TryReserveError> { // SAFETY: Precondition passed to caller let Some((ptr, layout)) = (unsafe { self.current_memory(elem_layout) }) else { @@ -837,7 +926,7 @@ impl RawVecInner { // for the T::IS_ZST case since current_memory() will have returned // None. if cap == 0 { - unsafe { self.alloc.deallocate(ptr, layout) }; + unsafe { alloc.deallocate(ptr, layout) }; self.ptr = unsafe { Unique::new_unchecked(ptr::without_provenance_mut(elem_layout.align())) }; self.cap = ZERO_CAP; @@ -847,7 +936,7 @@ impl RawVecInner { // overflowed earlier when capacity was larger. let new_size = elem_layout.size().unchecked_mul(cap); let new_layout = Layout::from_size_align_unchecked(new_size, layout.align()); - self.alloc + alloc .shrink(ptr, layout, new_layout) .map_err(|_| AllocError { layout: new_layout, non_exhaustive: () })? }; @@ -860,25 +949,6 @@ impl RawVecInner { } } -#[rustc_const_unstable(feature = "const_heap", issue = "79597")] -const impl RawVecInner { - /// # Safety - /// - /// This function deallocates the owned allocation, but does not update `ptr` or `cap` to - /// prevent double-free or use-after-free. Essentially, do not do anything with the caller - /// after this function returns. - /// Ideally this function would take `self` by move, but it cannot because it exists to be - /// called from a `Drop` impl. - unsafe fn deallocate(&mut self, elem_layout: Layout) { - // SAFETY: Precondition passed to caller - if let Some((ptr, layout)) = unsafe { self.current_memory(elem_layout) } { - unsafe { - self.alloc.deallocate(ptr, layout); - } - } - } -} - // Central function for reserve error handling. #[cfg(not(no_global_oom_handling))] #[cold] diff --git a/library/alloc/src/raw_vec/tests.rs b/library/alloc/src/raw_vec/tests.rs index 15f48c03dc54c..fb9a2acd97e0f 100644 --- a/library/alloc/src/raw_vec/tests.rs +++ b/library/alloc/src/raw_vec/tests.rs @@ -42,9 +42,9 @@ fn allocator_param() { let a = BoundedAlloc { fuel: Cell::new(500) }; let mut v: RawVec = RawVec::with_capacity_in(50, a); - assert_eq!(v.inner.alloc.fuel.get(), 450); + assert_eq!(v.allocator().fuel.get(), 450); v.reserve(50, 150); // (causes a realloc, thus using 50 + 150 = 200 units of fuel) - assert_eq!(v.inner.alloc.fuel.get(), 250); + assert_eq!(v.allocator().fuel.get(), 250); } #[test] @@ -126,12 +126,24 @@ fn zst() { assert_eq!(v.try_reserve_exact(101, usize::MAX - 100), cap_err); zst_sanity(&v); - assert_eq!(unsafe { v.inner.grow_amortized(100, usize::MAX - 100, ZST::LAYOUT) }, cap_err); - assert_eq!(unsafe { v.inner.grow_amortized(101, usize::MAX - 100, ZST::LAYOUT) }, cap_err); + assert_eq!( + unsafe { v.inner.grow_plan(100, usize::MAX - 100, ZST::LAYOUT, false).map(|_| ()) }, + cap_err + ); + assert_eq!( + unsafe { v.inner.grow_plan(101, usize::MAX - 100, ZST::LAYOUT, false).map(|_| ()) }, + cap_err + ); zst_sanity(&v); - assert_eq!(unsafe { v.inner.grow_exact(100, usize::MAX - 100, ZST::LAYOUT) }, cap_err); - assert_eq!(unsafe { v.inner.grow_exact(101, usize::MAX - 100, ZST::LAYOUT) }, cap_err); + assert_eq!( + unsafe { v.inner.grow_plan(100, usize::MAX - 100, ZST::LAYOUT, true).map(|_| ()) }, + cap_err + ); + assert_eq!( + unsafe { v.inner.grow_plan(101, usize::MAX - 100, ZST::LAYOUT, true).map(|_| ()) }, + cap_err + ); zst_sanity(&v); } diff --git a/library/alloc/src/vec/mod.rs b/library/alloc/src/vec/mod.rs index 7965a10c98289..27c209d9889bd 100644 --- a/library/alloc/src/vec/mod.rs +++ b/library/alloc/src/vec/mod.rs @@ -992,7 +992,8 @@ const impl Vec { /// capacity after the push, *O*(*capacity*) time is taken to copy the /// vector's elements to a larger allocation. This expensive operation is /// offset by the *capacity* *O*(1) insertions it allows. - #[inline] + #[inline(always)] + #[rustc_diagnostic_item = "vec_push"] #[stable(feature = "rust1", since = "1.0.0")] #[rustc_confusables("push_back", "put", "append")] pub fn push(&mut self, value: T) { @@ -1024,7 +1025,7 @@ const impl Vec { /// capacity after the push, *O*(*capacity*) time is taken to copy the /// vector's elements to a larger allocation. This expensive operation is /// offset by the *capacity* *O*(1) insertions it allows. - #[inline] + #[inline(always)] #[stable(feature = "push_mut", since = "1.95.0")] #[must_use = "if you don't need a reference to the value, use `Vec::push` instead"] pub fn push_mut(&mut self, value: T) -> &mut T { @@ -1528,7 +1529,14 @@ impl Vec { /// ``` #[stable(feature = "try_reserve", since = "1.57.0")] pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { - self.buf.try_reserve(self.len, additional) + let len = self.len; + let result = self.buf.try_reserve(len, additional); + unsafe { + // The buffer and its allocator cannot change the vector's length. + // Keep that fact visible when allocator operations are type-erased. + hint::assert_unchecked(self.len == len); + } + result } /// Tries to reserve the minimum capacity for at least `additional` @@ -1571,7 +1579,12 @@ impl Vec { /// ``` #[stable(feature = "try_reserve", since = "1.57.0")] pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), TryReserveError> { - self.buf.try_reserve_exact(self.len, additional) + let len = self.len; + let result = self.buf.try_reserve_exact(len, additional); + unsafe { + hint::assert_unchecked(self.len == len); + } + result } /// Shrinks the capacity of the vector as much as possible. diff --git a/library/alloctests/benches/vec.rs b/library/alloctests/benches/vec.rs index 656164da72084..920c51d1fd66e 100644 --- a/library/alloctests/benches/vec.rs +++ b/library/alloctests/benches/vec.rs @@ -1,8 +1,172 @@ use std::iter::repeat; +use std::marker::PhantomData; +use std::mem::ManuallyDrop; +use std::ptr; use rand::RngCore; use test::{Bencher, black_box}; +#[inline(never)] +fn push_grow(n: usize) -> Vec { + let mut v = Vec::new(); + for i in 0..n { + v.push(i); + } + v +} + +fn do_bench_push(b: &mut Bencher, n: usize) { + b.iter(|| push_grow(n)); +} + +#[bench] +fn bench_push_0100(b: &mut Bencher) { + do_bench_push(b, 100); +} + +#[bench] +fn bench_push_1000(b: &mut Bencher) { + do_bench_push(b, 1000); +} + +#[bench] +fn bench_push_10000(b: &mut Bencher) { + do_bench_push(b, 10000); +} + +#[inline(never)] +fn push_preallocated(n: usize) -> Vec { + let mut v = Vec::with_capacity(n); + for i in 0..n { + v.push(i); + } + v +} + +fn do_bench_push_preallocated(b: &mut Bencher, n: usize) { + b.iter(|| push_preallocated(n)); +} + +struct CaptureLocally<'a, T> { + value: ManuallyDrop, + old: *mut T, + _marker: PhantomData<&'a mut T>, +} + +impl Drop for CaptureLocally<'_, T> { + fn drop(&mut self) { + unsafe { + let value = ptr::read((&raw const self.value).cast::()); + ptr::write(self.old, value); + } + } +} + +impl<'a, T> CaptureLocally<'a, T> { + #[inline(always)] + fn new(old: &'a mut T) -> Self { + Self { value: ManuallyDrop::new(unsafe { ptr::read(old) }), old, _marker: PhantomData } + } + + #[inline(always)] + fn get_mut(&mut self) -> &mut T { + unsafe { &mut *((&raw mut self.value).cast::()) } + } + + #[inline(always)] + fn restore(self) { + let this = ManuallyDrop::new(self); + unsafe { + let value = ptr::read((&raw const this.value).cast::()); + ptr::write(this.old, value); + } + } +} + +#[inline(never)] +fn push_borrowed(v: &mut Vec, n: usize) { + for i in 0..n { + v.push(i); + } +} + +#[inline(never)] +fn push_borrowed_captured(v: &mut Vec, n: usize) { + let mut local = CaptureLocally::new(v); + { + let v = local.get_mut(); + for i in 0..n { + v.push(i); + } + } + local.restore(); +} + +#[inline(never)] +fn push_owned(mut v: Vec, n: usize) -> Vec { + for i in 0..n { + v.push(i); + } + v +} + +fn do_bench_push_borrowed(b: &mut Bencher, n: usize, captured: bool) { + b.iter(|| { + let mut v = Vec::new(); + if captured { + push_borrowed_captured(&mut v, n); + } else { + push_borrowed(&mut v, n); + } + v + }); +} + +#[bench] +fn bench_push_borrowed_1000(b: &mut Bencher) { + do_bench_push_borrowed(b, 1000, false); +} + +#[bench] +fn bench_push_borrowed_captured_1000(b: &mut Bencher) { + do_bench_push_borrowed(b, 1000, true); +} + +#[bench] +fn bench_push_borrowed_10000(b: &mut Bencher) { + do_bench_push_borrowed(b, 10000, false); +} + +#[bench] +fn bench_push_borrowed_captured_10000(b: &mut Bencher) { + do_bench_push_borrowed(b, 10000, true); +} + +#[bench] +fn bench_push_owned_1000(b: &mut Bencher) { + b.iter(|| push_owned(Vec::new(), 1000)); +} + +#[bench] +fn bench_push_owned_10000(b: &mut Bencher) { + b.iter(|| push_owned(Vec::new(), 10000)); +} + +#[bench] +fn bench_push_preallocated_0100(b: &mut Bencher) { + do_bench_push_preallocated(b, 100); +} + +#[bench] +fn bench_push_preallocated_1000(b: &mut Bencher) { + do_bench_push_preallocated(b, 1000); +} + +#[bench] +fn bench_push_preallocated_10000(b: &mut Bencher) { + do_bench_push_preallocated(b, 10000); +} + #[bench] fn bench_new(b: &mut Bencher) { b.iter(|| Vec::::new()) diff --git a/library/alloctests/lib.rs b/library/alloctests/lib.rs index eca8444812521..1c2022ad51bed 100644 --- a/library/alloctests/lib.rs +++ b/library/alloctests/lib.rs @@ -52,6 +52,7 @@ #![feature(trusted_random_access)] #![feature(try_reserve_kind)] #![feature(try_trait_v2)] +#![feature(tuple_trait)] #![feature(unwrap_infallible)] #![feature(wtf8_internals)] // tidy-alphabetical-end @@ -61,12 +62,14 @@ #![feature(const_closures)] #![feature(const_trait_impl)] #![feature(dropck_eyepatch)] +#![feature(intrinsics)] #![feature(min_specialization)] #![feature(optimize_attribute)] #![feature(prelude_import)] #![feature(rustc_attrs)] #![feature(staged_api)] #![feature(test)] +#![feature(unboxed_closures)] #![rustc_preserve_ub_checks] // tidy-alphabetical-end diff --git a/tests/debuginfo/strings-and-strs.rs b/tests/debuginfo/strings-and-strs.rs index a860aa6106d07..faa7d1ed750d9 100644 --- a/tests/debuginfo/strings-and-strs.rs +++ b/tests/debuginfo/strings-and-strs.rs @@ -8,7 +8,7 @@ //@ gdb-command:run //@ gdb-command:print plain_string -//@ gdb-check:$1 = alloc::string::String {vec: alloc::vec::Vec {buf: alloc::raw_vec::RawVec {inner: alloc::raw_vec::RawVecInner {ptr: core::ptr::unique::Unique {pointer: core::ptr::non_null::NonNull {pointer: 0x[...]}, _marker: core::marker::PhantomData}, cap: core::num::niche_types::UsizeNoHighBit (5), alloc: alloc::alloc::Global}, _marker: core::marker::PhantomData}, len: 5}} +//@ gdb-check:$1 = alloc::string::String {vec: alloc::vec::Vec {buf: alloc::raw_vec::RawVec {inner: alloc::raw_vec::RawVecInner {ptr: core::ptr::unique::Unique {pointer: core::ptr::non_null::NonNull {pointer: 0x[...]}, _marker: core::marker::PhantomData}, cap: core::num::niche_types::UsizeNoHighBit (5)}, alloc: alloc::alloc::Global, _marker: core::marker::PhantomData}, len: 5}} //@ gdb-command:print plain_str //@ gdb-check:$2 = "Hello" diff --git a/tests/mir-opt/capture_mut_vec.push.CaptureMutVec.diff b/tests/mir-opt/capture_mut_vec.push.CaptureMutVec.diff new file mode 100644 index 0000000000000..518d743ea7fdc --- /dev/null +++ b/tests/mir-opt/capture_mut_vec.push.CaptureMutVec.diff @@ -0,0 +1,169 @@ +- // MIR for `push` before CaptureMutVec ++ // MIR for `push` after CaptureMutVec + + fn push(_1: &mut Vec, _2: usize) -> () { + debug vec => _1; + debug count => _2; + let mut _0: (); + let _3: (); + let mut _4: bool; + let mut _5: &std::vec::Vec; + let _6: (); + let mut _7: &mut std::vec::Vec; + let _8: (); + let mut _9: std::ops::Range; + let mut _10: std::ops::Range; + let mut _11: usize; + let mut _12: std::ops::Range; + let mut _13: (); + let _14: (); + let mut _15: std::option::Option; + let mut _16: &mut std::ops::Range; + let mut _17: &mut std::ops::Range; + let mut _18: isize; + let mut _19: !; + let _21: (); + let mut _22: &mut std::vec::Vec; + let mut _23: usize; + let _24: (); + let mut _25: &mut std::vec::Vec; + let mut _26: usize; ++ let mut _27: std::vec::Vec; ++ let mut _28: &mut std::vec::Vec; + scope 1 { + debug iter => _12; + let _20: usize; + scope 2 { + debug value => _20; + } + } + + bb0: { ++ _27 = move (*_1); ++ _28 = &mut _27; + StorageLive(_3); + StorageLive(_4); + StorageLive(_5); +- _5 = &(*_1); ++ _5 = &(*_28); + _4 = Vec::::is_empty(move _5) -> [return: bb1, unwind unreachable]; + } + + bb1: { + switchInt(move _4) -> [0: bb3, otherwise: bb2]; + } + + bb2: { + StorageDead(_5); + _3 = const (); + goto -> bb5; + } + + bb3: { + StorageDead(_5); + StorageLive(_6); + StorageLive(_7); +- _7 = &mut (*_1); ++ _7 = &mut (*_28); + _6 = Vec::::clear(move _7) -> [return: bb4, unwind unreachable]; + } + + bb4: { + StorageDead(_7); + StorageDead(_6); + _3 = const (); + goto -> bb5; + } + + bb5: { + StorageDead(_4); + StorageDead(_3); + StorageLive(_8); + StorageLive(_9); + StorageLive(_10); + StorageLive(_11); + _11 = copy _2; + _10 = std::ops::Range:: { start: const 0_usize, end: move _11 }; + StorageDead(_11); + _9 = as IntoIterator>::into_iter(move _10) -> [return: bb6, unwind unreachable]; + } + + bb6: { + StorageDead(_10); + StorageLive(_12); + _12 = move _9; + goto -> bb7; + } + + bb7: { + StorageLive(_14); + StorageLive(_15); + StorageLive(_16); + StorageLive(_17); + _17 = &mut _12; + _16 = &mut (*_17); + _15 = as Iterator>::next(move _16) -> [return: bb8, unwind unreachable]; + } + + bb8: { + StorageDead(_16); + _18 = discriminant(_15); + switchInt(move _18) -> [0: bb11, 1: bb10, otherwise: bb9]; + } + + bb9: { + unreachable; + } + + bb10: { + StorageLive(_20); + _20 = copy ((_15 as Some).0: usize); + StorageLive(_21); + StorageLive(_22); +- _22 = &mut (*_1); ++ _22 = &mut (*_28); + StorageLive(_23); + _23 = copy _20; + _21 = Vec::::push(move _22, move _23) -> [return: bb12, unwind unreachable]; + } + + bb11: { + _8 = const (); + StorageDead(_17); + StorageDead(_15); + StorageDead(_14); + StorageDead(_12); + StorageDead(_9); + StorageDead(_8); + StorageLive(_24); + StorageLive(_25); +- _25 = &mut (*_1); ++ _25 = &mut (*_28); + StorageLive(_26); + _26 = copy _2; + _24 = Vec::::truncate(move _25, move _26) -> [return: bb13, unwind unreachable]; + } + + bb12: { + StorageDead(_23); + StorageDead(_22); + StorageDead(_21); + _14 = const (); + StorageDead(_20); + StorageDead(_17); + StorageDead(_15); + StorageDead(_14); + _13 = const (); + goto -> bb7; + } + + bb13: { + StorageDead(_26); + StorageDead(_25); + StorageDead(_24); + _0 = const (); ++ (*_1) = move _27; + return; + } + } + diff --git a/tests/mir-opt/capture_mut_vec.rs b/tests/mir-opt/capture_mut_vec.rs new file mode 100644 index 0000000000000..85cddc0e5b759 --- /dev/null +++ b/tests/mir-opt/capture_mut_vec.rs @@ -0,0 +1,17 @@ +//@ compile-flags: -Zmir-opt-level=3 -Zmir-enable-passes=+CaptureMutVec +//@ test-mir-pass: CaptureMutVec +//@ skip-filecheck + +// EMIT_MIR capture_mut_vec.push.CaptureMutVec.diff +#[inline(never)] +pub fn push(vec: &mut Vec, count: usize) { + if !vec.is_empty() { + vec.clear(); + } + for value in 0..count { + vec.push(value); + } + vec.truncate(count); +} + +fn main() {} diff --git a/tests/mir-opt/pre-codegen/loops.vec_move.runtime-optimized.after.mir b/tests/mir-opt/pre-codegen/loops.vec_move.runtime-optimized.after.mir index a49688ae891de..0016ae5e12d7e 100644 --- a/tests/mir-opt/pre-codegen/loops.vec_move.runtime-optimized.after.mir +++ b/tests/mir-opt/pre-codegen/loops.vec_move.runtime-optimized.after.mir @@ -50,103 +50,103 @@ fn vec_move(_1: Vec) -> () { scope 9 { debug cap => _20; } - scope 45 (inlined > as Deref>::deref) { + scope 44 (inlined > as Deref>::deref) { debug self => _38; - scope 46 (inlined MaybeDangling::>::as_ref) { + scope 45 (inlined MaybeDangling::>::as_ref) { } } - scope 47 (inlined alloc::raw_vec::RawVec::::capacity) { + scope 46 (inlined alloc::raw_vec::RawVec::::capacity) { debug self => _37; let mut _39: &alloc::raw_vec::RawVecInner; - scope 48 (inlined std::mem::size_of::) { + scope 47 (inlined std::mem::size_of::) { } - scope 49 (inlined alloc::raw_vec::RawVecInner::capacity) { + scope 48 (inlined alloc::raw_vec::RawVecInner::capacity) { debug self => _39; debug elem_size => const ::SIZE; let mut _21: core::num::niche_types::UsizeNoHighBit; - scope 50 (inlined core::num::niche_types::UsizeNoHighBit::as_inner) { + scope 49 (inlined core::num::niche_types::UsizeNoHighBit::as_inner) { debug self => _21; } } } } - scope 29 (inlined > as Deref>::deref) { + scope 28 (inlined > as Deref>::deref) { debug self => _34; - scope 30 (inlined MaybeDangling::>::as_ref) { + scope 29 (inlined MaybeDangling::>::as_ref) { } } - scope 31 (inlined Vec::::len) { + scope 30 (inlined Vec::::len) { debug self => _33; let mut _13: bool; - scope 32 { + scope 31 { } } - scope 33 (inlined std::ptr::mut_ptr::::wrapping_byte_add) { + scope 32 (inlined std::ptr::mut_ptr::::wrapping_byte_add) { debug self => _7; debug count => _12; let mut _14: *mut u8; let mut _18: *mut u8; let mut _19: *const impl Sized; - scope 34 (inlined std::ptr::mut_ptr::::cast::) { + scope 33 (inlined std::ptr::mut_ptr::::cast::) { debug self => _7; } - scope 35 (inlined std::ptr::mut_ptr::::wrapping_add) { + scope 34 (inlined std::ptr::mut_ptr::::wrapping_add) { debug self => _14; debug count => _12; let mut _15: isize; - scope 36 (inlined std::ptr::mut_ptr::::wrapping_offset) { + scope 35 (inlined std::ptr::mut_ptr::::wrapping_offset) { debug self => _14; debug count => _15; let mut _16: *const u8; let mut _17: *const u8; } } - scope 37 (inlined std::ptr::mut_ptr::::with_metadata_of::) { + scope 36 (inlined std::ptr::mut_ptr::::with_metadata_of::) { debug self => _18; debug meta => _19; - scope 38 (inlined std::ptr::metadata::) { + scope 37 (inlined std::ptr::metadata::) { debug ptr => _19; } - scope 39 (inlined std::ptr::from_raw_parts_mut::) { + scope 38 (inlined std::ptr::from_raw_parts_mut::) { } } } - scope 40 (inlined > as Deref>::deref) { + scope 39 (inlined > as Deref>::deref) { debug self => _36; - scope 41 (inlined MaybeDangling::>::as_ref) { + scope 40 (inlined MaybeDangling::>::as_ref) { } } - scope 42 (inlined Vec::::len) { + scope 41 (inlined Vec::::len) { debug self => _35; let mut _9: bool; - scope 43 { + scope 42 { } } - scope 44 (inlined #[track_caller] std::ptr::mut_ptr::::add) { + scope 43 (inlined #[track_caller] std::ptr::mut_ptr::::add) { debug self => _7; debug count => _8; } } - scope 28 (inlined NonNull::::as_ptr) { + scope 27 (inlined NonNull::::as_ptr) { debug self => _6; } } - scope 20 (inlined > as Deref>::deref) { + scope 19 (inlined > as Deref>::deref) { debug self => _32; - scope 21 (inlined MaybeDangling::>::as_ref) { + scope 20 (inlined MaybeDangling::>::as_ref) { } } - scope 22 (inlined alloc::raw_vec::RawVec::::non_null) { + scope 21 (inlined alloc::raw_vec::RawVec::::non_null) { debug self => _31; - scope 23 (inlined alloc::raw_vec::RawVecInner::non_null::) { + scope 22 (inlined alloc::raw_vec::RawVecInner::non_null::) { let mut _5: std::ptr::NonNull; - scope 24 (inlined std::ptr::Unique::::cast::) { - scope 25 (inlined NonNull::::cast::) { - scope 26 (inlined NonNull::::as_ptr) { + scope 23 (inlined std::ptr::Unique::::cast::) { + scope 24 (inlined NonNull::::cast::) { + scope 25 (inlined NonNull::::as_ptr) { } } } - scope 27 (inlined std::ptr::Unique::::as_non_null_ptr) { + scope 26 (inlined std::ptr::Unique::::as_non_null_ptr) { } } } @@ -159,16 +159,14 @@ fn vec_move(_1: Vec) -> () { scope 14 (inlined Vec::::allocator) { debug self => _29; scope 15 (inlined alloc::raw_vec::RawVec::::allocator) { - scope 16 (inlined alloc::raw_vec::RawVecInner::allocator) { - } } } - scope 17 (inlined #[track_caller] std::ptr::read::) { + scope 16 (inlined #[track_caller] std::ptr::read::) { debug src => _4; } - scope 18 (inlined ManuallyDrop::::new) { + scope 17 (inlined ManuallyDrop::::new) { debug value => const std::alloc::Global; - scope 19 (inlined MaybeDangling::::new) { + scope 18 (inlined MaybeDangling::::new) { } } } @@ -194,7 +192,7 @@ fn vec_move(_1: Vec) -> () { StorageLive(_4); // DBG: _30 = &_3; // DBG: _29 = &((_3.0: std::mem::MaybeDangling>).0: std::vec::Vec); - _4 = &raw const (((((_3.0: std::mem::MaybeDangling>).0: std::vec::Vec).0: alloc::raw_vec::RawVec).0: alloc::raw_vec::RawVecInner).2: std::alloc::Global); + _4 = &raw const ((((_3.0: std::mem::MaybeDangling>).0: std::vec::Vec).0: alloc::raw_vec::RawVec).1: std::alloc::Global); StorageDead(_4); StorageLive(_6); // DBG: _32 = &_3;