diff --git a/Cargo.lock b/Cargo.lock index cc011d59..7d30af58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c880a97d28a3681c0267bd29cff89621202715b065127cd445fa0f0fe0aa2880" +dependencies = [ + "serde_core", +] + [[package]] name = "anes" version = "0.1.6" @@ -500,6 +509,7 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" name = "smallvec" version = "2.0.0-alpha.12" dependencies = [ + "allocator-api2", "bytes", "criterion", "malloc_size_of", diff --git a/Cargo.toml b/Cargo.toml index 4dbbc234..8b559d28 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,16 +14,19 @@ documentation = "https://docs.rs/smallvec/" exclude = [".gitignore", "tests/", "scripts/", "fuzz/", "benches/", ".github/"] [features] -std = [] +default = ["allocator-api2"] +std = ["allocator-api2?/std"] specialization = [] may_dangle = [] -serde = ["dep:serde_core"] +serde = ["dep:serde_core", "allocator-api2?/serde"] internals = [] +allocator-api2 = ["dep:allocator-api2"] [dependencies] bytes = { version = "1", optional = true, default-features = false } serde_core = { version = "1.0.221", optional = true, default-features = false } malloc_size_of = { version = "0.1.1", optional = true, default-features = false } +allocator-api2 = { version = "0.4.0", optional = true, default-features = false, features = ["alloc"] } [dev-dependencies] serde_test = "1.0" diff --git a/fuzz/fuzz_targets/smallvec_ops.rs b/fuzz/fuzz_targets/smallvec_ops.rs index 991793dc..f8f70bc4 100644 --- a/fuzz/fuzz_targets/smallvec_ops.rs +++ b/fuzz/fuzz_targets/smallvec_ops.rs @@ -58,7 +58,7 @@ fn do_test(data: &[u8]) -> SmallVec { 5 => { v.pop(); } - 6 => v.grow(next_usize!(bytes) + v.len()), + 6 => v.reserve_exact(next_usize!(bytes)), 7 => { if v.len() < CAP_GROWTH { v.reserve(next_usize!(bytes)) diff --git a/src/lib.rs b/src/lib.rs index af120445..41fe07f9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -58,6 +58,7 @@ #![cfg_attr(feature = "specialization", allow(incomplete_features))] #![cfg_attr(feature = "specialization", feature(specialization, trusted_len))] #![cfg_attr(feature = "may_dangle", feature(dropck_eyepatch))] +#![cfg_attr(not(feature = "allocator-api2"), feature(allocator_api))] #[doc(hidden)] pub extern crate alloc; @@ -69,10 +70,16 @@ mod rawsmallvec; #[cfg(test)] mod tests; -use alloc::alloc::Layout; +#[cfg(not(feature = "allocator-api2"))] +use alloc::alloc::{AllocError, Allocator, Global, Layout}; use alloc::boxed::Box; use alloc::vec; use alloc::vec::Vec; +#[cfg(feature = "allocator-api2")] +use allocator_api2::{ + alloc::{AllocError, Allocator, Global, Layout}, + SliceExt, +}; #[cfg(feature = "bytes")] use bytes::{buf::UninitSlice, BufMut}; use core::borrow::Borrow; @@ -90,9 +97,9 @@ use core::ptr::NonNull; #[cfg(feature = "malloc_size_of")] use malloc_size_of::{MallocShallowSizeOf, MallocSizeOf, MallocSizeOfOps}; #[cfg(feature = "internals")] -pub use rawsmallvec::RawSmallVec; +pub use rawsmallvec::{RawSmallVec, RawSmallVecUnion}; #[cfg(not(feature = "internals"))] -use rawsmallvec::RawSmallVec; +use rawsmallvec::{RawSmallVec, RawSmallVecUnion}; #[cfg(feature = "serde")] use serde_core::{ de::{Deserialize, Deserializer, SeqAccess, Visitor}, @@ -129,12 +136,39 @@ fn infallible(result: Result) -> T { } } +/// Creates a [`Layout`] values for arrays of length `n` +/// for a given type without checking preconditions. +/// +/// # Safety +/// +/// The caller must ensure that an array of length `n` results +/// in a valid layout. +#[inline(always)] +const unsafe fn array_layout_unchecked(n: usize) -> Layout { + // SAFETY: The caller ensures that the an array of length `n` is possible + // which means that the multiplication can't overflow. + // The value returned by `align_of` will fulfill the safety conditions for + // `Layout::from_size_align_unchecked`. + unsafe { Layout::from_size_align_unchecked(size_of::().unchecked_mul(n), align_of::()) } +} + /// Helper function to check if a type is a ZST. #[inline] const fn is_zst() -> bool { const { size_of::() == 0 } } +#[inline(always)] +const fn inline_size() -> usize { + const { + if is_zst::() { + usize::MAX + } else { + N + } + } +} + #[inline] /// A local copy of [`core::slice::range`]. The latter function is unstable /// and thus cannot be used yet. @@ -170,19 +204,14 @@ where core::ops::Range { start, end } } -impl RawSmallVec { - const IS_ZST: bool = is_zst::(); - - #[inline] - const fn new() -> Self { - Self::new_inline(MaybeUninit::uninit()) - } +impl RawSmallVecUnion { #[inline] const fn new_inline(inline: MaybeUninit<[T; N]>) -> Self { Self { inline: ManuallyDrop::new(inline), } } + #[inline] const fn new_heap(ptr: NonNull, capacity: usize) -> Self { Self { @@ -206,73 +235,328 @@ impl RawSmallVec { /// # Safety /// - /// The vector must be on the heap + /// The vector must be on the heap. #[inline] const unsafe fn as_ptr_heap(&self) -> *const T { - self.heap.0.as_ptr() + // SAFETY: Safety conditions are identical. + unsafe { self.heap.0.as_ptr() } } /// # Safety /// - /// The vector must be on the heap + /// The vector must be on the heap. #[inline] const unsafe fn as_mut_ptr_heap(&mut self) -> *mut T { - self.heap.0.as_ptr() + // SAFETY: Safety conditions are identical. + unsafe { self.heap.0.as_ptr() } + } +} + +impl RawSmallVec { + #[inline] + pub const fn new() -> Self { + Self::new_in(Global) + } + + #[inline] + pub const fn new_inline(inline: MaybeUninit<[T; N]>) -> Self { + Self::new_inline_in(inline, Global) + } + + #[inline] + pub const fn new_heap(ptr: NonNull, capacity: usize) -> Self { + Self::new_heap_in(ptr, capacity, Global) + } + + #[inline] + pub fn with_capacity(capacity: usize) -> Self { + Self::with_capacity_in(capacity, Global) + } +} + +impl RawSmallVec { + const IS_ZST: bool = is_zst::(); + + /// Turn a generic allocation error returned by a parametric allocator into + /// a [`CollectionAllocErr`]. + #[inline(always)] + fn handle_alloc_error( + r: Result, + layout: Layout, + ) -> Result { + r.map_err(|_| CollectionAllocErr::AllocErr { layout }) + } + + #[inline] + const fn new_in(allocator: A) -> Self { + Self::new_inline_in(MaybeUninit::uninit(), allocator) + } + + #[inline] + const fn new_inline_in(inline: MaybeUninit<[T; N]>, allocator: A) -> Self { + Self { + inner: RawSmallVecUnion::new_inline(inline), + allocator, + } } + #[inline] + const fn new_heap_in(ptr: NonNull, capacity: usize, allocator: A) -> Self { + Self { + inner: RawSmallVecUnion::new_heap(ptr, capacity), + allocator, + } + } + + #[inline] + fn with_capacity_in(capacity: usize, allocator: A) -> Self { + infallible(Self::try_with_capacity_in(capacity, allocator)) + } + + #[inline] + fn try_with_capacity_in(capacity: usize, allocator: A) -> Result { + if capacity <= const { inline_size::() } { + Ok(Self::new_inline_in(MaybeUninit::uninit(), allocator)) + } else { + let layout = + Layout::array::(capacity).map_err(|_| CollectionAllocErr::CapacityOverflow)?; + let ptr = Self::handle_alloc_error(allocator.allocate(layout), layout)?; + let inner = RawSmallVecUnion { + heap: (ptr.cast(), capacity), + }; + Ok(Self { inner, allocator }) + } + } + + /// Gets a pointer to the contents of the vector, under the assumption + /// that the content is stored inline. + #[inline] + pub const fn as_ptr_inline(&self) -> *const T { + self.inner.as_ptr_inline() + } + + /// Gets a pointer to the contents of the vector, under the assumption + /// that the content is stored inline. + #[inline] + pub const fn as_mut_ptr_inline(&mut self) -> *mut T { + self.inner.as_mut_ptr_inline() + } + + /// Gets a pointer to the contents of the vector, under the assumption + /// that the content is stored on the heap. + /// /// # Safety /// - /// `new_capacity` must be non zero, and greater or equal to the length. - /// T must not be a ZST. - unsafe fn try_grow_raw( + /// The vector must be on the heap. + #[inline] + pub const unsafe fn as_ptr_heap(&self) -> *const T { + // SAFETY: The safety requirements are identical. + unsafe { self.inner.as_ptr_heap() } + } + + /// Gets a pointer to the contents of the vector, under the assumption + /// that the content is stored on the heap. + /// + /// # Safety + /// + /// The vector must be on the heap. + #[inline] + pub const unsafe fn as_mut_ptr_heap(&mut self) -> *mut T { + // SAFETY: The safety requirements are identical. + unsafe { self.inner.as_mut_ptr_heap() } + } + + /// Returns `true` if the elements are stored on the heap, and `false` + /// otherwise. + /// + /// # Safety + /// + /// The way elements are stored in `self` must correspond to the tag in + /// `len`. + unsafe fn try_reserve( &mut self, len: TaggedLen, - new_capacity: usize, - ) -> Result<(), CollectionAllocErr> { - use alloc::alloc::{alloc, realloc}; + additional: usize, + ) -> Result { debug_assert!(!Self::IS_ZST); - debug_assert!(new_capacity > 0); - debug_assert!(new_capacity >= len.value()); - let was_on_heap = len.on_heap(); - let ptr = if was_on_heap { - self.as_mut_ptr_heap() + let on_heap = len.on_heap(); + let len = len.value(); + + if additional == 0 { + return Ok(on_heap); + } + + let new_capacity = len + .checked_add(additional) + .ok_or(CollectionAllocErr::CapacityOverflow)?; + + if on_heap { + // SAFETY: The caller ensures that the tag corresponds to the + // way in which data is stored. + let (old_ptr, old_capacity) = unsafe { self.inner.heap }; + + // Nothing needs to be done if the capacity is already sufficient. + if old_capacity >= new_capacity { + return Ok(true); + } + + // Ensure capacity growth is exponential. + let new_capacity = new_capacity.max(2 * old_capacity); + + // SAFETY: The stored capacity always corresponds to a valid layout. + let old_layout = unsafe { array_layout_unchecked::(old_capacity) }; + + let new_layout = Layout::array::(new_capacity) + .map_err(|_| CollectionAllocErr::CapacityOverflow)?; + let ptr = Self::handle_alloc_error( + self.allocator.grow(old_ptr.cast(), old_layout, new_layout), + new_layout, + )?; + + self.inner = RawSmallVecUnion::new_heap(ptr.cast(), new_capacity); + Ok(true) + } else if new_capacity > const { inline_size::() } { + // Ensure capacity growth is exponential. + let new_capacity = (2 * N).max(new_capacity); + + let layout = Layout::array::(new_capacity) + .map_err(|_| CollectionAllocErr::CapacityOverflow)?; + let ptr = Self::handle_alloc_error(self.allocator.allocate(layout), layout)?; + + // SAFETY: The pointer returned by `allocate` is valid and its own memory + // region. + unsafe { + copy_nonoverlapping(self.as_mut_ptr_inline(), ptr.cast().as_ptr(), len); + } + + self.inner = RawSmallVecUnion::new_heap(ptr.cast(), new_capacity); + Ok(true) } else { - self.as_mut_ptr_inline() - }; + Ok(on_heap) + } + } + + /// Returns `true` if the elements are stored on the heap, and `false` + /// otherwise. + /// + /// # Safety + /// + /// The way elements are stored in `self` must correspond to the tag in + /// `len`. + unsafe fn try_reserve_exact( + &mut self, + len: TaggedLen, + additional: usize, + ) -> Result { + debug_assert!(!is_zst::()); + + let on_heap = len.on_heap(); let len = len.value(); - let new_layout = - Layout::array::(new_capacity).map_err(|_| CollectionAllocErr::CapacityOverflow)?; - if new_layout.size() > isize::MAX as usize { - return Err(CollectionAllocErr::CapacityOverflow); + if additional == 0 { + return Ok(on_heap); } - let new_ptr = if !was_on_heap { - // get a fresh allocation - let new_ptr = alloc(new_layout) as *mut T; // `new_layout` has nonzero size. - let new_ptr = - NonNull::new(new_ptr).ok_or(CollectionAllocErr::AllocErr { layout: new_layout })?; - copy_nonoverlapping(ptr, new_ptr.as_ptr(), len); - new_ptr + let new_capacity = len + .checked_add(additional) + .ok_or(CollectionAllocErr::CapacityOverflow)?; + + if on_heap { + // SAFETY: The caller ensures that the tag corresponds to the + // way in which data is stored. + let (old_ptr, old_capacity) = unsafe { self.inner.heap }; + + // Nothing needs to be done if the capacity is already sufficient. + if old_capacity >= new_capacity { + return Ok(true); + } + + // SAFETY: The stored capacity always corresponds to a valid layout. + let old_layout = unsafe { array_layout_unchecked::(old_capacity) }; + + let new_layout = Layout::array::(new_capacity) + .map_err(|_| CollectionAllocErr::CapacityOverflow)?; + let ptr = Self::handle_alloc_error( + self.allocator.grow(old_ptr.cast(), old_layout, new_layout), + new_layout, + )?; + + self.inner = RawSmallVecUnion::new_heap(ptr.cast(), new_capacity); + + Ok(true) + } else if new_capacity > const { inline_size::() } { + let layout = Layout::array::(new_capacity) + .map_err(|_| CollectionAllocErr::CapacityOverflow)?; + let ptr = Self::handle_alloc_error(self.allocator.allocate(layout), layout)?; + + // SAFETY: The pointer returned by `allocate` is valid and its own memory + // region. + unsafe { + copy_nonoverlapping(self.as_mut_ptr_inline(), ptr.cast().as_ptr(), len); + } + + self.inner = RawSmallVecUnion::new_heap(ptr.cast(), new_capacity); + + Ok(true) } else { - // use realloc - - // this can't overflow since we already constructed an equivalent layout during - // the previous allocation - let old_layout = - Layout::from_size_align_unchecked(self.heap.1 * size_of::(), align_of::()); - - // SAFETY: ptr was allocated with this allocator - // old_layout is the same as the layout used to allocate the previous memory - // block new_layout.size() is greater than zero - // does not overflow when rounded up to alignment. since it was constructed - // with Layout::array - let new_ptr = realloc(ptr as *mut u8, old_layout, new_layout.size()) as *mut T; - NonNull::new(new_ptr).ok_or(CollectionAllocErr::AllocErr { layout: new_layout })? - }; - *self = Self::new_heap(new_ptr, new_capacity); - Ok(()) + Ok(on_heap) + } + } + + /// Returns `true` if the elements are still stored on the heap, and `false` + /// otherwise. + /// + /// # Safety + /// + /// The way elements are stored in `self` must correspond to `on_heap`. + unsafe fn shrink_to_fit( + &mut self, + on_heap: bool, + cap: usize, + ) -> Result { + debug_assert!(!is_zst::()); + + if on_heap { + // SAFETY: The caller ensures that the tag corresponds to the + // way in which data is stored. + let (old_ptr, old_capacity) = unsafe { self.inner.heap }; + + // SAFETY: The stored capacity corresponds always to a valid layout. + let layout = unsafe { array_layout_unchecked::(old_capacity) }; + + if cap <= N { + self.inner = RawSmallVecUnion::new_inline(MaybeUninit::uninit()); + + // SAFETY: The memory regions don't overlap because one pointer is recently + // created inline storage. By taking the minimum value of both + // capabilities, the copying will only touch valid memory. + unsafe { + let count = cap.min(old_capacity); + copy_nonoverlapping(old_ptr.cast().as_ptr(), self.as_mut_ptr_inline(), count); + } + + self.allocator.deallocate(old_ptr.cast(), layout); + + Ok(false) + } else if cap < old_capacity { + // SAFETY: The new capacity is smaller than the old capacity, + // and it is already possible to construct a valid layout with the old capacity. + let new_layout = unsafe { array_layout_unchecked::(cap) }; + + let ptr = Self::handle_alloc_error( + self.allocator.shrink(old_ptr.cast(), layout, new_layout), + new_layout, + )?; + self.inner = RawSmallVecUnion::new_heap(ptr.cast(), cap); + + Ok(true) + } else { + Ok(true) + } + } else { + Ok(on_heap) + } } } @@ -338,9 +622,9 @@ impl TaggedLen { } #[repr(C)] -pub struct SmallVec { +pub struct SmallVec { len: TaggedLen, - raw: RawSmallVec, + raw: RawSmallVec, _marker: PhantomData, } @@ -360,7 +644,7 @@ impl Default for SmallVec { /// Returned from [`SmallVec::drain`][1]. /// /// [1]: struct.SmallVec.html#method.drain -pub struct Drain<'a, T: 'a, const N: usize> { +pub struct Drain<'a, T: 'a, const N: usize, A: Allocator = Global> { // `vec` points to a valid object within its lifetime. // This is ensured by the fact that we're holding an iterator to its items. // @@ -371,10 +655,10 @@ pub struct Drain<'a, T: 'a, const N: usize> { tail_start: usize, tail_len: usize, iter: core::slice::Iter<'a, T>, - vec: core::ptr::NonNull>, + vec: core::ptr::NonNull>, } -impl<'a, T: 'a, const N: usize> Iterator for Drain<'a, T, N> { +impl<'a, T: 'a, const N: usize, A: Allocator> Iterator for Drain<'a, T, N, A> { type Item = T; #[inline] @@ -392,7 +676,7 @@ impl<'a, T: 'a, const N: usize> Iterator for Drain<'a, T, N> { } } -impl<'a, T: 'a, const N: usize> DoubleEndedIterator for Drain<'a, T, N> { +impl<'a, T: 'a, const N: usize, A: Allocator> DoubleEndedIterator for Drain<'a, T, N, A> { #[inline] fn next_back(&mut self) -> Option { // SAFETY: see above @@ -402,21 +686,21 @@ impl<'a, T: 'a, const N: usize> DoubleEndedIterator for Drain<'a, T, N> { } } -impl ExactSizeIterator for Drain<'_, T, N> { +impl ExactSizeIterator for Drain<'_, T, N, A> { #[inline] fn len(&self) -> usize { self.iter.len() } } -impl core::iter::FusedIterator for Drain<'_, T, N> {} +impl core::iter::FusedIterator for Drain<'_, T, N, A> {} -impl<'a, T: 'a, const N: usize> Drop for Drain<'a, T, N> { +impl<'a, T: 'a, const N: usize, A: Allocator> Drop for Drain<'a, T, N, A> { fn drop(&mut self) { /// Moves back the un-`Drain`ed elements to restore the original `Vec`. - struct DropGuard<'r, 'a, T, const N: usize>(&'r mut Drain<'a, T, N>); + struct DropGuard<'r, 'a, T, const N: usize, A: Allocator>(&'r mut Drain<'a, T, N, A>); - impl<'r, 'a, T, const N: usize> Drop for DropGuard<'r, 'a, T, N> { + impl<'r, 'a, T, const N: usize, A: Allocator> Drop for DropGuard<'r, 'a, T, N, A> { fn drop(&mut self) { if self.0.tail_len > 0 { unsafe { @@ -441,7 +725,7 @@ impl<'a, T: 'a, const N: usize> Drop for Drain<'a, T, N> { let mut vec = self.vec; - if SmallVec::::IS_ZST { + if SmallVec::::IS_ZST { // ZSTs have no identity, so we don't need to move them around, we only need to // drop the correct amount. this can be achieved by manipulating the // Vec length instead of moving values out from `iter`. @@ -485,7 +769,7 @@ impl<'a, T: 'a, const N: usize> Drop for Drain<'a, T, N> { } } -impl Drain<'_, T, N> { +impl Drain<'_, T, N, A> { #[must_use] pub fn as_slice(&self) -> &[T] { self.iter.as_slice() @@ -546,11 +830,11 @@ impl Drain<'_, T, N> { /// Returned from [`SmallVec::extract_if`][1]. /// /// [1]: struct.SmallVec.html#method.extract_if -pub struct ExtractIf<'a, T, const N: usize, F> +pub struct ExtractIf<'a, T, const N: usize, F, A: Allocator = Global> where F: FnMut(&mut T) -> bool, { - vec: &'a mut SmallVec, + vec: &'a mut SmallVec, /// The index of the item that will be inspected by the next call to `next`. idx: usize, /// Elements at and beyond this point will be retained. Must be equal or @@ -564,7 +848,7 @@ where pred: F, } -impl core::fmt::Debug for ExtractIf<'_, T, N, F> +impl core::fmt::Debug for ExtractIf<'_, T, N, F, A> where F: FnMut(&mut T) -> bool, T: core::fmt::Debug, @@ -576,7 +860,7 @@ where } } -impl Iterator for ExtractIf<'_, T, N, F> +impl Iterator for ExtractIf<'_, T, N, F, A> where F: FnMut(&mut T) -> bool, { @@ -611,7 +895,7 @@ where } } -impl Drop for ExtractIf<'_, T, N, F> +impl Drop for ExtractIf<'_, T, N, F, A> where F: FnMut(&mut T) -> bool, { @@ -635,12 +919,12 @@ where } } -pub struct Splice<'a, I: Iterator + 'a, const N: usize> { - drain: Drain<'a, I::Item, N>, +pub struct Splice<'a, I: Iterator + 'a, const N: usize, A: Allocator = Global> { + drain: Drain<'a, I::Item, N, A>, replace_with: I, } -impl<'a, I, const N: usize> core::fmt::Debug for Splice<'a, I, N> +impl<'a, I, const N: usize, A: Allocator> core::fmt::Debug for Splice<'a, I, N, A> where I: Debug + Iterator + 'a, ::Item: Debug, @@ -650,7 +934,7 @@ where } } -impl Iterator for Splice<'_, I, N> { +impl Iterator for Splice<'_, I, N, A> { type Item = I::Item; fn next(&mut self) -> Option { @@ -662,15 +946,15 @@ impl Iterator for Splice<'_, I, N> { } } -impl DoubleEndedIterator for Splice<'_, I, N> { +impl DoubleEndedIterator for Splice<'_, I, N, A> { fn next_back(&mut self) -> Option { self.drain.next_back() } } -impl ExactSizeIterator for Splice<'_, I, N> {} +impl ExactSizeIterator for Splice<'_, I, N, A> {} -impl Drop for Splice<'_, I, N> { +impl Drop for Splice<'_, I, N, A> { fn drop(&mut self) { self.drain.by_ref().for_each(drop); // At this point draining is done and the only remaining tasks are splicing @@ -725,13 +1009,13 @@ impl Drop for Splice<'_, I, N> { /// Returned from [`SmallVec::into_iter`][1]. /// /// [1]: struct.SmallVec.html#method.into_iter -pub struct IntoIter { +pub struct IntoIter { // # Safety // // `end` decides whether the data lives on the heap or not // // The members from begin..end are initialized - raw: RawSmallVec, + raw: RawSmallVec, begin: usize, end: TaggedLen, _marker: PhantomData, @@ -739,10 +1023,10 @@ pub struct IntoIter { // SAFETY: IntoIter has unique ownership of its contents. Sending (or sharing) // an `IntoIter` is equivalent to sending (or sharing) a `SmallVec`. -unsafe impl Send for IntoIter where T: Send {} -unsafe impl Sync for IntoIter where T: Sync {} +unsafe impl Send for IntoIter where T: Send {} +unsafe impl Sync for IntoIter where T: Sync {} -impl IntoIter { +impl IntoIter { #[inline] const fn as_ptr(&self) -> *const T { let on_heap = self.end.on_heap(); @@ -785,7 +1069,7 @@ impl IntoIter { } } -impl Iterator for IntoIter { +impl Iterator for IntoIter { type Item = T; #[inline] @@ -829,8 +1113,8 @@ impl DoubleEndedIterator for IntoIter { } } } -impl ExactSizeIterator for IntoIter {} -impl core::iter::FusedIterator for IntoIter {} +impl ExactSizeIterator for IntoIter {} +impl core::iter::FusedIterator for IntoIter {} impl SmallVec { #[inline] @@ -844,15 +1128,190 @@ impl SmallVec { #[inline] pub fn with_capacity(capacity: usize) -> Self { - let mut this = Self::new(); - if capacity > Self::inline_size() { - this.grow(capacity); + let on_heap = capacity > const { inline_size::() }; + Self { + len: TaggedLen::new(0, on_heap), + raw: RawSmallVec::with_capacity(capacity), + _marker: PhantomData, } - this + } + + /// Creates a `SmallVec` directly from the raw components of another + /// `SmallVec`. + /// + /// # Safety + /// + /// This is highly unsafe, due to the number of invariants that aren’t + /// checked: + /// + /// - `ptr` needs to have been previously allocated via `SmallVec` from its + /// spilled storage (at least, it’s highly likely to be incorrect if it + /// wasn’t). + /// - `ptr`’s `A::Item` type needs to be the same size and alignment that it + /// was allocated with + /// - `length` needs to be less than or equal to `capacity`. + /// - `capacity` needs to be the capacity that the pointer was allocated + /// with. + /// + /// Violating these may cause problems like corrupting the allocator’s + /// internal data structures. + /// + /// Additionally, `capacity` must be greater than the amount of inline + /// storage `A` has; that is, the new `SmallVec` must need to spill over + /// into heap allocated storage. This condition is asserted against. + /// + /// The ownership of `ptr` is effectively transferred to the `SmallVec` + /// which may then deallocate, reallocate or change the contents of memory + /// pointed to by the pointer at will. Ensure that nothing else uses the + /// pointer after calling this function. + /// + /// # Examples + /// + /// ``` + /// use smallvec::{smallvec, SmallVec}; + /// + /// let mut v: SmallVec<_, 1> = smallvec![1, 2, 3]; + /// + /// // Pull out the important parts of `v`. + /// let p = v.as_mut_ptr(); + /// let len = v.len(); + /// let cap = v.capacity(); + /// let spilled = v.spilled(); + /// + /// unsafe { + /// // Forget all about `v`. The heap allocation that stored the + /// // three values won't be deallocated. + /// std::mem::forget(v); + /// + /// // Overwrite memory with [4, 5, 6]. + /// // + /// // This is only safe if `spilled` is true! Otherwise, we are + /// // writing into the old `SmallVec`'s inline storage on the + /// // stack. + /// assert!(spilled); + /// for i in 0..len { + /// std::ptr::write(p.add(i), 4 + i); + /// } + /// + /// // Put everything back together into a SmallVec with a different + /// // amount of inline storage, but which is still less than `cap`. + /// let rebuilt = SmallVec::<_, 2>::from_raw_parts(p, len, cap); + /// assert_eq!(&*rebuilt, &[4, 5, 6]); + /// } + /// ``` + #[inline] + pub unsafe fn from_raw_parts(ptr: *mut T, length: usize, capacity: usize) -> Self { + assert!(!Self::IS_ZST); + + // SAFETY: We require caller to provide same ptr as we alloc + // and we never alloc null pointer. + let ptr = unsafe { + debug_assert!(!ptr.is_null(), "Called `from_raw_parts` with null pointer."); + NonNull::new_unchecked(ptr) + }; + + SmallVec { + len: TaggedLen::new(length, true), + raw: RawSmallVec::new_heap(ptr, capacity), + _marker: PhantomData, + } + } + + #[inline] + pub fn into_raw_parts(self) -> (*mut T, usize, usize) { + let mut me = ManuallyDrop::new(self); + (me.as_mut_ptr(), me.len(), me.capacity()) } #[inline] pub const fn from_buf(elements: [T; S]) -> Self { + Self::from_buf_in::(elements, Global) + } + + #[inline] + pub fn from_buf_and_len(buf: [T; N], len: usize) -> Self { + Self::from_buf_and_len_in(buf, len, Global) + } + + /// Constructs a new `SmallVec` on the stack from an A without copying + /// elements. Also sets the length. The user is responsible for ensuring + /// that `len <= A::size()`. + /// + /// # Examples + /// + /// ``` + /// use smallvec::SmallVec; + /// use std::mem::MaybeUninit; + /// + /// let buf = [1, 2, 3, 4, 5, 0, 0, 0]; + /// let small_vec = unsafe { SmallVec::from_buf_and_len_unchecked(MaybeUninit::new(buf), 5) }; + /// + /// assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); + /// ``` + /// + /// # Safety + /// + /// `len <= N`, and all the elements in `buf[..len]` must be initialized + #[inline] + pub const unsafe fn from_buf_and_len_unchecked(buf: MaybeUninit<[T; N]>, len: usize) -> Self { + debug_assert!(len <= N); + Self { + len: TaggedLen::new(len, false), + raw: RawSmallVec::new_inline(buf), + _marker: PhantomData, + } + } +} + +impl SmallVec { + const IS_ZST: bool = is_zst::(); + + #[inline] + pub const fn new_in(alloc: A) -> Self { + Self { + len: TaggedLen::new(0, false), + raw: RawSmallVec::new_in(alloc), + _marker: PhantomData, + } + } + + #[inline] + pub fn with_capacity_in(capacity: usize, alloc: A) -> Self { + let on_heap = capacity > const { inline_size::() }; + Self { + len: TaggedLen::new(0, on_heap), + raw: RawSmallVec::with_capacity_in(capacity, alloc), + _marker: PhantomData, + } + } + + #[inline] + pub unsafe fn from_raw_parts_in(ptr: *mut T, length: usize, capacity: usize, alloc: A) -> Self { + assert!(!Self::IS_ZST); + + // SAFETY: We require caller to provide same ptr as we alloc + // and we never alloc null pointer. + let ptr = unsafe { + debug_assert!(!ptr.is_null(), "Called `from_raw_parts` with null pointer."); + NonNull::new_unchecked(ptr) + }; + + SmallVec { + len: TaggedLen::new(length, true), + raw: RawSmallVec::new_heap_in(ptr, capacity, alloc), + _marker: PhantomData, + } + } + + #[inline] + pub fn into_raw_parts_with_allocator(self) -> (*mut T, usize, usize, A) { + let mut me = ManuallyDrop::new(self); + let alloc = unsafe { core::ptr::read(me.allocator()) }; + (me.as_mut_ptr(), me.len(), me.capacity(), alloc) + } + + #[inline] + pub const fn from_buf_in(elements: [T; S], alloc: A) -> Self { const { assert!(S <= N); } @@ -875,18 +1334,18 @@ impl SmallVec { // SAFETY: all the members in 0..S are initialized Self { len: TaggedLen::new(S, false), - raw: RawSmallVec::new_inline(buf), + raw: RawSmallVec::new_inline_in(buf, alloc), _marker: PhantomData, } } #[inline] - pub fn from_buf_and_len(buf: [T; N], len: usize) -> Self { + pub fn from_buf_and_len_in(buf: [T; N], len: usize, alloc: A) -> Self { assert!(len <= N); // SAFETY: all the members in 0..len are initialized let mut vec = Self { len: TaggedLen::new(len, false), - raw: RawSmallVec::new_inline(MaybeUninit::new(buf)), + raw: RawSmallVec::new_inline_in(MaybeUninit::new(buf), alloc), _marker: PhantomData, }; // Deallocate the remaining elements so no memory is leaked. @@ -906,67 +1365,153 @@ impl SmallVec { vec } - /// Constructs a new `SmallVec` on the stack from an A without copying - /// elements. Also sets the length. The user is responsible for ensuring - /// that `len <= A::size()`. - /// - /// # Examples - /// - /// ``` - /// use smallvec::SmallVec; - /// use std::mem::MaybeUninit; - /// - /// let buf = [1, 2, 3, 4, 5, 0, 0, 0]; - /// let small_vec = unsafe { SmallVec::from_buf_and_len_unchecked(MaybeUninit::new(buf), 5) }; - /// - /// assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); - /// ``` - /// - /// # Safety - /// - /// `len <= N`, and all the elements in `buf[..len]` must be initialized #[inline] - pub const unsafe fn from_buf_and_len_unchecked(buf: MaybeUninit<[T; N]>, len: usize) -> Self { + pub const unsafe fn from_buf_and_len_in_unchecked( + buf: MaybeUninit<[T; N]>, + len: usize, + alloc: A, + ) -> Self { debug_assert!(len <= N); Self { len: TaggedLen::new(len, false), - raw: RawSmallVec::new_inline(buf), + raw: RawSmallVec::new_inline_in(buf, alloc), _marker: PhantomData, } } } -impl SmallVec { - const IS_ZST: bool = is_zst::(); +macro_rules! make_heap_methods { + ( + $from_vec:ident, + $into_vec:ident, + $into_boxed_slice:ident, + $vec:path, + $box:path, + $into_raw_parts:ident $(,)? + ) => { + impl SmallVec { + #[inline] + pub fn $from_vec(mut vec: $vec) -> Self { + if Self::IS_ZST { + let len = vec.len(); + + // We don't wrap the vector in ManuallyDrop so that when it's dropped, the + // memory is deallocated, if it needs to be. + // SAFETY: `0` is less than or equal to the vector's capacity. + // old_len..new_len is an empty range. So there are no uninitialized elements + unsafe { vec.set_len(0) }; + vec.shrink_to_fit(); + + let (_ptr, _len, _cap, alloc) = vec.$into_raw_parts(); + + Self { + len: TaggedLen::new(len, false), + raw: RawSmallVec::new_in(alloc), + _marker: PhantomData, + } + } else { + // FIXME: Use `into_parts_with_allocator` once it is stable/available. + let (ptr, len, cap, alloc) = vec.$into_raw_parts(); + // SAFETY: The pointer of a `Vec` is never null. + let ptr = unsafe { NonNull::new_unchecked(ptr) }; + + Self { + len: TaggedLen::new(len, true), + raw: RawSmallVec::new_heap_in(ptr, cap, alloc), + _marker: PhantomData, + } + } + } - #[inline] - pub fn from_vec(vec: Vec) -> Self { - if vec.capacity() == 0 { - return Self::new(); + #[inline] + pub fn $into_vec(self) -> $vec { + let len = self.len(); + let this = ManuallyDrop::new(self); + + // SAFETY: The pointer is created using a normal reference + // so the pointer must be valid. + let alloc = unsafe { core::ptr::read(this.allocator()) }; + + if !this.spilled() { + let mut vec = <$vec>::with_capacity_in(len, alloc); + // SAFETY: we create a new vector with sufficient capacity, copy our elements + // into it to transfer ownership and then set the length + // we don't drop the elements we previously held + unsafe { + copy_nonoverlapping(this.raw.as_ptr_inline(), vec.as_mut_ptr(), len); + vec.set_len(len); + } + vec + } else { + // SAFETY: + // - `ptr` was created with the global allocator + // - `ptr` was created with the appropriate alignment for `T` + // - the allocation pointed to by ptr is exactly cap * sizeof(T) + // - `len` is less than or equal to `cap` + // - the first `len` entries are proper `T`-values + // - the allocation is not larger than `isize::MAX` + unsafe { + let (ptr, cap) = this.raw.inner.heap; + <$vec>::from_raw_parts_in(ptr.as_ptr(), len, cap, alloc) + } + } + } + + #[inline] + pub fn $into_boxed_slice(self) -> $box { + self.$into_vec().into_boxed_slice() + } } + }; +} + +#[cfg(not(feature = "allocator-api2"))] +make_heap_methods! { + from_vec, + into_vec, + into_boxed_slice, + Vec, + Box<[T], A>, + into_raw_parts_with_allocator, +} +#[cfg(feature = "allocator-api2")] +make_heap_methods! { + from_vec2, + into_vec2, + into_boxed_slice2, + allocator_api2::vec::Vec, + allocator_api2::boxed::Box<[T], A>, + into_raw_parts_with_alloc, +} + +/// Functions for interacting with `std` types whenever +/// the "allocator-api2" feature is enabled. +#[cfg(feature = "allocator-api2")] +impl SmallVec { + #[inline] + pub fn from_vec(mut vec: Vec) -> Self { if Self::IS_ZST { - // "Move" elements to stack buffer. They're ZST so we don't actually have to do - // anything. Just make sure they're not dropped. - // We don't wrap the vector in ManuallyDrop so that when it's dropped, the - // memory is deallocated, if it needs to be. - let mut vec = vec; let len = vec.len(); - // SAFETY: `0` is less than the vector's capacity. + // We don't wrap the vector in ManuallyDrop so that when it's dropped, the + // memory is deallocated, if it needs to be. + // SAFETY: `0` is less than or equal to the vector's capacity. // old_len..new_len is an empty range. So there are no uninitialized elements unsafe { vec.set_len(0) }; + Self { len: TaggedLen::new(len, false), raw: RawSmallVec::new(), _marker: PhantomData, } } else { + // FIXME: Use `into_parts` once it is stable. + // `into_raw_parts` cannot be use here because of the crates MSRV. let mut vec = ManuallyDrop::new(vec); let len = vec.len(); let cap = vec.capacity(); - // SAFETY: vec.capacity is not `0` (checked above), so the pointer - // can not dangle and thus specifically cannot be null. + // SAFETY: The pointer of a `Vec` is never null. let ptr = unsafe { NonNull::new_unchecked(vec.as_mut_ptr()) }; Self { @@ -977,11 +1522,54 @@ impl SmallVec { } } + #[inline] + pub fn into_vec(self) -> Vec { + let len = self.len(); + let this = ManuallyDrop::new(self); + + if !this.spilled() { + let mut vec = Vec::with_capacity(len); + // SAFETY: we create a new vector with sufficient capacity, copy our elements + // into it to transfer ownership and then set the length + // we don't drop the elements we previously held + unsafe { + copy_nonoverlapping(this.raw.as_ptr_inline(), vec.as_mut_ptr(), len); + vec.set_len(len); + } + vec + } else { + // SAFETY: + // - `ptr` was created with the global allocator + // - `ptr` was created with the appropriate alignment for `T` + // - the allocation pointed to by ptr is exactly cap * sizeof(T) + // - `len` is less than or equal to `cap` + // - the first `len` entries are proper `T`-values + // - the allocation is not larger than `isize::MAX` + unsafe { + let (ptr, cap) = this.raw.inner.heap; + Vec::from_raw_parts(ptr.as_ptr(), len, cap) + } + } + } + + #[inline] + pub fn into_boxed_slice(self) -> Box<[T]> { + self.into_vec().into_boxed_slice() + } +} + +impl SmallVec { + /// Returns a reference to the underlying allocator. + #[inline] + pub fn allocator(&self) -> &A { + &self.raw.allocator + } + /// Sets the tag to be on the heap /// /// # Safety /// - /// The active union member must be the self.raw.heap + /// The active union member must be the self.raw.inner.heap #[inline] unsafe fn set_on_heap(&mut self) { self.len = TaggedLen::new(self.len(), true); @@ -991,7 +1579,7 @@ impl SmallVec { /// /// # Safety /// - /// The active union member must be the self.raw.inline + /// The active union member must be the self.raw.inner.inline #[inline] unsafe fn set_inline(&mut self) { self.len = TaggedLen::new(self.len(), false); @@ -1016,11 +1604,7 @@ impl SmallVec { #[inline] pub const fn inline_size() -> usize { - if Self::IS_ZST { - usize::MAX - } else { - N - } + const { inline_size::() } } #[inline] @@ -1037,8 +1621,8 @@ impl SmallVec { #[inline] pub const fn capacity(&self) -> usize { if self.len.on_heap() { - // SAFETY: raw.heap is active - unsafe { self.raw.heap.1 } + // SAFETY: raw.inner.heap is active + unsafe { self.raw.inner.heap.1 } } else { Self::inline_size() } @@ -1076,12 +1660,15 @@ impl SmallVec { /// assert_eq!(vec2, [2, 3]); /// ``` #[inline] - pub fn split_off(&mut self, at: usize) -> Self { + pub fn split_off(&mut self, at: usize) -> Self + where + A: Clone, + { let len = self.len(); assert!(at <= len); let other_len = len - at; - let mut other = Self::with_capacity(other_len); + let mut other = Self::with_capacity_in(other_len, self.allocator().clone()); // Unsafely `set_len` and copy items to `other`. unsafe { @@ -1093,7 +1680,7 @@ impl SmallVec { other } - pub fn drain(&mut self, range: R) -> Drain<'_, T, N> + pub fn drain(&mut self, range: R) -> Drain<'_, T, N, A> where R: core::ops::RangeBounds, { @@ -1204,7 +1791,7 @@ impl SmallVec { /// ); /// assert_eq!(ones.len(), 3); /// ``` - pub fn extract_if(&mut self, range: R, filter: F) -> ExtractIf<'_, T, N, F> + pub fn extract_if(&mut self, range: R, filter: F) -> ExtractIf<'_, T, N, F, A> where F: FnMut(&mut T) -> bool, R: core::ops::RangeBounds, @@ -1227,7 +1814,7 @@ impl SmallVec { } } - pub fn splice(&mut self, range: R, replace_with: I) -> Splice<'_, I::IntoIter, N> + pub fn splice(&mut self, range: R, replace_with: I) -> Splice<'_, I::IntoIter, N, A> where R: core::ops::RangeBounds, I: IntoIterator, @@ -1301,7 +1888,7 @@ impl SmallVec { } #[inline] - pub fn append(&mut self, other: &mut SmallVec) { + pub fn append(&mut self, other: &mut SmallVec) { // can't overflow since both are smaller than isize::MAX and 2 * isize::MAX < // usize::MAX let len = self.len(); @@ -1321,161 +1908,86 @@ impl SmallVec { } #[inline] - pub fn grow(&mut self, new_capacity: usize) { - infallible(self.try_grow(new_capacity)); + pub fn reserve(&mut self, additional: usize) { + infallible(self.try_reserve(additional)); } - #[cold] - pub fn try_grow(&mut self, new_capacity: usize) -> Result<(), CollectionAllocErr> { + #[inline] + pub fn try_reserve(&mut self, additional: usize) -> Result<(), CollectionAllocErr> { if Self::IS_ZST { return Ok(()); } - let len = self.len(); - assert!(new_capacity >= len); - - if new_capacity > Self::inline_size() { - // SAFETY: we checked all the preconditions - let result = unsafe { self.raw.try_grow_raw(self.len, new_capacity) }; - - if result.is_ok() { - // SAFETY: the allocation succeeded, so self.raw.heap is now active - unsafe { self.set_on_heap() }; - } - result - } else { - // new_capacity <= Self::inline_size() - if self.spilled() { - unsafe { - // SAFETY: heap member is active - let (ptr, old_cap) = self.raw.heap; - // inline member is now active - - // SAFETY: len <= new_capacity <= Self::inline_size() - // so the copy is within bounds of the inline member - copy_nonoverlapping(ptr.as_ptr(), self.raw.as_mut_ptr_inline(), len); - drop(DropDealloc { - ptr: ptr.cast(), - size_bytes: old_cap * size_of::(), - align: align_of::(), - }); - self.set_inline(); - } + // SAFETY: The tag inside the length of the vector corresponds to the way + // elements are stored inside the vector. The same goes for the return value + // of the function. + unsafe { + let on_heap = self.raw.try_reserve(self.len, additional)?; + if on_heap { + self.set_on_heap(); + } else { + self.set_inline(); } - Ok(()) - } - } - - #[inline] - pub fn reserve(&mut self, additional: usize) { - // can't overflow since len <= capacity - if additional > self.capacity() - self.len() { - let new_capacity = infallible( - self.len() - .checked_add(additional) - .and_then(usize::checked_next_power_of_two) - .ok_or(CollectionAllocErr::CapacityOverflow), - ); - self.grow(new_capacity); - } - } + }; - #[inline] - pub fn try_reserve(&mut self, additional: usize) -> Result<(), CollectionAllocErr> { - if additional > self.capacity() - self.len() { - let new_capacity = self - .len() - .checked_add(additional) - .and_then(usize::checked_next_power_of_two) - .ok_or(CollectionAllocErr::CapacityOverflow)?; - self.try_grow(new_capacity) - } else { - Ok(()) - } + Ok(()) } #[inline] pub fn reserve_exact(&mut self, additional: usize) { - // can't overflow since len <= capacity - if additional > self.capacity() - self.len() { - let new_capacity = infallible( - self.len() - .checked_add(additional) - .ok_or(CollectionAllocErr::CapacityOverflow), - ); - self.grow(new_capacity); - } + infallible(self.try_reserve_exact(additional)); } #[inline] pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), CollectionAllocErr> { - if additional > self.capacity() - self.len() { - let new_capacity = self - .len() - .checked_add(additional) - .ok_or(CollectionAllocErr::CapacityOverflow)?; - self.try_grow(new_capacity) - } else { - Ok(()) + if Self::IS_ZST { + return Ok(()); } + + // SAFETY: The tag inside the length of the vector corresponds to the way + // elements are stored inside the vector. The same goes for the return value + // of the function. + unsafe { + let on_heap = self.raw.try_reserve_exact(self.len, additional)?; + if on_heap { + self.set_on_heap(); + } else { + self.set_inline(); + } + }; + + Ok(()) } #[inline] pub fn shrink_to_fit(&mut self) { - if !self.spilled() { + if Self::IS_ZST { return; } + let len = self.len(); - if len <= Self::inline_size() { - // SAFETY: self.spilled() is true, so we're on the heap - unsafe { - let (ptr, capacity) = self.raw.heap; - self.raw = RawSmallVec::new_inline(MaybeUninit::uninit()); - copy_nonoverlapping(ptr.as_ptr(), self.raw.as_mut_ptr_inline(), len); - self.set_inline(); - alloc::alloc::dealloc( - ptr.cast().as_ptr(), - Layout::from_size_align_unchecked(capacity * size_of::(), align_of::()), - ); - } - } else if len < self.capacity() { - // SAFETY: len > Self::inline_size() >= 0 - // so new capacity is non zero, it is equal to the length - // T can't be a ZST because SmallVec is never spilled. - unsafe { infallible(self.raw.try_grow_raw(self.len, len)) }; - } + let on_heap = self.spilled(); + + // SAFETY: The tag inside the length of the vector corresponds to the way + // elements are stored inside the vector. + let on_heap = unsafe { infallible(self.raw.shrink_to_fit(on_heap, len)) }; + self.len = TaggedLen::new(len, on_heap); } #[inline] pub fn shrink_to(&mut self, min_capacity: usize) { - if !self.spilled() { + if Self::IS_ZST { return; } - if self.capacity() > min_capacity { - let len = self.len(); - let target = core::cmp::max(len, min_capacity); - if target <= Self::inline_size() { - // SAFETY: self.spilled() is true, so we're on the heap - unsafe { - let (ptr, capacity) = self.raw.heap; - self.raw = RawSmallVec::new_inline(MaybeUninit::uninit()); - copy_nonoverlapping(ptr.as_ptr(), self.raw.as_mut_ptr_inline(), len); - self.set_inline(); - alloc::alloc::dealloc( - ptr.cast().as_ptr(), - Layout::from_size_align_unchecked( - capacity * size_of::(), - align_of::(), - ), - ); - } - } else if target < self.capacity() { - // SAFETY: len > Self::inline_size() >= 0 - // so new capacity is non zero, it is equal to the length - // T can't be a ZST because SmallVec is never spilled. - unsafe { infallible(self.raw.try_grow_raw(self.len, target)) }; - } - } + + let len = self.len(); + let min_capacity = len.max(min_capacity); + let on_heap = self.spilled(); + + // SAFETY: The tag inside the length of the vector corresponds to the way + // elements are stored inside the vector. + let on_heap = unsafe { infallible(self.raw.shrink_to_fit(on_heap, min_capacity)) }; + self.len = TaggedLen::new(len, on_heap); } #[inline] @@ -1632,41 +2144,6 @@ impl SmallVec { } } - #[inline] - pub fn into_vec(self) -> Vec { - let len = self.len(); - if !self.spilled() { - let mut vec = Vec::with_capacity(len); - let this = ManuallyDrop::new(self); - // SAFETY: we create a new vector with sufficient capacity, copy our elements - // into it to transfer ownership and then set the length - // we don't drop the elements we previously held - unsafe { - copy_nonoverlapping(this.raw.as_ptr_inline(), vec.as_mut_ptr(), len); - vec.set_len(len); - } - vec - } else { - let this = ManuallyDrop::new(self); - // SAFETY: - // - `ptr` was created with the global allocator - // - `ptr` was created with the appropriate alignment for `T` - // - the allocation pointed to by ptr is exactly cap * sizeof(T) - // - `len` is less than or equal to `cap` - // - the first `len` entries are proper `T`-values - // - the allocation is not larger than `isize::MAX` - unsafe { - let (ptr, cap) = this.raw.heap; - Vec::from_raw_parts(ptr.as_ptr(), len, cap) - } - } - } - - #[inline] - pub fn into_boxed_slice(self) -> Box<[T]> { - self.into_vec().into_boxed_slice() - } - #[inline] pub fn into_inner(self) -> Result<[T; N], Self> { if self.len() != N { @@ -1806,90 +2283,40 @@ impl SmallVec { ) } } +} - /// Creates a `SmallVec` directly from the raw components of another - /// `SmallVec`. - /// - /// # Safety - /// - /// This is highly unsafe, due to the number of invariants that aren’t - /// checked: - /// - /// - `ptr` needs to have been previously allocated via `SmallVec` from its - /// spilled storage (at least, it’s highly likely to be incorrect if it - /// wasn’t). - /// - `ptr`’s `A::Item` type needs to be the same size and alignment that it - /// was allocated with - /// - `length` needs to be less than or equal to `capacity`. - /// - `capacity` needs to be the capacity that the pointer was allocated - /// with. - /// - /// Violating these may cause problems like corrupting the allocator’s - /// internal data structures. - /// - /// Additionally, `capacity` must be greater than the amount of inline - /// storage `A` has; that is, the new `SmallVec` must need to spill over - /// into heap allocated storage. This condition is asserted against. - /// - /// The ownership of `ptr` is effectively transferred to the `SmallVec` - /// which may then deallocate, reallocate or change the contents of memory - /// pointed to by the pointer at will. Ensure that nothing else uses the - /// pointer after calling this function. - /// - /// # Examples - /// - /// ``` - /// use smallvec::{smallvec, SmallVec}; - /// - /// let mut v: SmallVec<_, 1> = smallvec![1, 2, 3]; - /// - /// // Pull out the important parts of `v`. - /// let p = v.as_mut_ptr(); - /// let len = v.len(); - /// let cap = v.capacity(); - /// let spilled = v.spilled(); - /// - /// unsafe { - /// // Forget all about `v`. The heap allocation that stored the - /// // three values won't be deallocated. - /// std::mem::forget(v); - /// - /// // Overwrite memory with [4, 5, 6]. - /// // - /// // This is only safe if `spilled` is true! Otherwise, we are - /// // writing into the old `SmallVec`'s inline storage on the - /// // stack. - /// assert!(spilled); - /// for i in 0..len { - /// std::ptr::write(p.add(i), 4 + i); - /// } - /// - /// // Put everything back together into a SmallVec with a different - /// // amount of inline storage, but which is still less than `cap`. - /// let rebuilt = SmallVec::<_, 2>::from_raw_parts(p, len, cap); - /// assert_eq!(&*rebuilt, &[4, 5, 6]); - /// } - /// ``` - #[inline] - pub unsafe fn from_raw_parts(ptr: *mut T, length: usize, capacity: usize) -> SmallVec { - assert!(!Self::IS_ZST); +impl SmallVec { + /// Creates a [`SmallVec`] value from the slice `slice` with the specified + /// allocator. + pub fn from_slice_in(slice: &[T], alloc: A) -> Self { + if slice.len() > const { Self::inline_size() } { + #[cfg(feature = "allocator-api2")] + { + // Standard Rust vectors are already specialized. + Self::from_vec2(slice.to_vec_in2(alloc)) + } - // SAFETY: We require caller to provide same ptr as we alloc - // and we never alloc null pointer. - let ptr = unsafe { - debug_assert!(!ptr.is_null(), "Called `from_raw_parts` with null pointer."); - NonNull::new_unchecked(ptr) - }; + #[cfg(not(feature = "allocator-api2"))] + { + // Standard Rust vectors are already specialized. + Self::from_vec(slice.to_vec_in(alloc)) + } + } else { + // SAFETY: The precondition is checked in the initial comparison above. + unsafe { + #[cfg(feature = "specialization")] + { + >::spec_from(slice, alloc) + } - SmallVec { - len: TaggedLen::new(length, true), - raw: RawSmallVec::new_heap(ptr, capacity), - _marker: PhantomData, + #[cfg(not(feature = "specialization"))] + { + Self::from_slice_fallback(slice, alloc) + } + } } } -} -impl SmallVec { #[inline] pub fn resize(&mut self, len: usize, value: T) { let old_len = self.len(); @@ -1988,16 +2415,30 @@ impl SmallVec { self.set_len(l + len); } } +} +impl SmallVec { /// A function for creating [`SmallVec`] values out of slices /// for types with the [`Copy`] trait. + #[inline] pub fn from_slice_copy(slice: &[T]) -> Self + where + T: Copy, + { + Self::from_slice_copy_in(slice, Global) + } +} + +impl SmallVec { + /// A function for creating [`SmallVec`] values out of slices + /// for types with the [`Copy`] trait. Supports custom allocators. + pub fn from_slice_copy_in(slice: &[T], alloc: A) -> Self where T: Copy, { let src = slice.as_ptr(); let len = slice.len(); - let mut result = Self::with_capacity(len); + let mut result = Self::with_capacity_in(len, alloc); // SAFETY: By using `with_capacity`, the pointer will point to valid memory. unsafe { @@ -2044,7 +2485,7 @@ impl Drop for DropDealloc { } #[cfg(feature = "may_dangle")] -unsafe impl<#[may_dangle] T, const N: usize> Drop for SmallVec { +unsafe impl<#[may_dangle] T, const N: usize, A: Allocator> Drop for SmallVec { fn drop(&mut self) { let on_heap = self.spilled(); let len = self.len(); @@ -2068,7 +2509,7 @@ unsafe impl<#[may_dangle] T, const N: usize> Drop for SmallVec { } #[cfg(not(feature = "may_dangle"))] -impl Drop for SmallVec { +impl Drop for SmallVec { fn drop(&mut self) { let on_heap = self.spilled(); let len = self.len(); @@ -2090,7 +2531,7 @@ impl Drop for SmallVec { } } -impl Drop for IntoIter { +impl Drop for IntoIter { fn drop(&mut self) { // SAFETY: see above unsafe { @@ -2099,7 +2540,7 @@ impl Drop for IntoIter { let end = self.end.value(); let ptr = self.as_mut_ptr(); let _drop_dealloc = if on_heap { - let capacity = self.raw.heap.1; + let capacity = self.raw.inner.heap.1; Some(DropDealloc { ptr: NonNull::new_unchecked(ptr as *mut u8), size_bytes: capacity * size_of::(), @@ -2113,7 +2554,7 @@ impl Drop for IntoIter { } } -impl core::ops::Deref for SmallVec { +impl core::ops::Deref for SmallVec { type Target = [T]; #[inline] @@ -2121,7 +2562,7 @@ impl core::ops::Deref for SmallVec { self.as_slice() } } -impl core::ops::DerefMut for SmallVec { +impl core::ops::DerefMut for SmallVec { #[inline] fn deref_mut(&mut self) -> &mut Self::Target { self.as_mut_slice() @@ -2129,7 +2570,7 @@ impl core::ops::DerefMut for SmallVec { } /// This function is used in the [`smallvec`] macro. -/// It is recommended to use the macro instead of using thís function. +/// It is recommended to use the macro instead of using this function. #[doc(hidden)] #[track_caller] pub fn from_elem(elem: T, n: usize) -> SmallVec { @@ -2140,13 +2581,54 @@ pub fn from_elem(elem: T, n: usize) -> SmallVec #[cfg(feature = "specialization")] { // SAFETY: The precondition is checked in the initial comparison above. - unsafe { as spec_traits::SpecFromElem>::spec_from_elem(elem, n) } + unsafe { + as spec_traits::SpecFromElem>::spec_from_elem( + elem, n, Global, + ) + } + } + + #[cfg(not(feature = "specialization"))] + { + // SAFETY: The precondition is checked in the initial comparison above. + unsafe { SmallVec::::from_elem_fallback(elem, n, Global) } + } + } +} + +#[doc(hidden)] +#[track_caller] +pub fn from_elem_in( + elem: T, + n: usize, + alloc: A, +) -> SmallVec { + if n > SmallVec::::inline_size() { + #[cfg(feature = "allocator-api2")] + { + SmallVec::::from_vec2(allocator_api2::vec::from_elem_in(elem, n, alloc)) + } + + #[cfg(not(feature = "allocator-api2"))] + { + // Standard Rust vectors are already specialized. + SmallVec::::from_vec(alloc::vec::from_elem_in(elem, n, alloc)) + } + } else { + #[cfg(feature = "specialization")] + { + // SAFETY: The precondition is checked in the initial comparison above. + unsafe { + as spec_traits::SpecFromElem>::spec_from_elem( + elem, n, alloc, + ) + } } #[cfg(not(feature = "specialization"))] { // SAFETY: The precondition is checked in the initial comparison above. - unsafe { SmallVec::::from_elem_fallback(elem, n) } + unsafe { SmallVec::::from_elem_fallback(elem, n, alloc) } } } } @@ -2158,27 +2640,27 @@ mod spec_traits { /// A trait for specializing the implementation of [`from_elem`]. /// /// [`from_elem`]: crate::from_elem - pub(crate) trait SpecFromElem { + pub(crate) trait SpecFromElem { /// Creates a `Smallvec` value where `elem` is repeated `n` times. /// This will use the inline storage, not the heap. /// /// # Safety /// /// The caller must ensure that `n <= Self::inline_size()`. - unsafe fn spec_from_elem(elem: T, n: usize) -> Self; + unsafe fn spec_from_elem(elem: T, n: usize, alloc: A) -> Self; } - impl SpecFromElem for SmallVec { + impl SpecFromElem for SmallVec { #[inline] - default unsafe fn spec_from_elem(elem: T, n: usize) -> Self { + default unsafe fn spec_from_elem(elem: T, n: usize, alloc: A) -> Self { // SAFETY: Safety conditions are identical. - unsafe { SmallVec::from_elem_fallback(elem, n) } + unsafe { SmallVec::from_elem_fallback(elem, n, alloc) } } } - impl SpecFromElem for SmallVec { - unsafe fn spec_from_elem(elem: T, n: usize) -> Self { - let mut result = Self::new(); + impl SpecFromElem for SmallVec { + unsafe fn spec_from_elem(elem: T, n: usize, alloc: A) -> Self { + let mut result = Self::new_in(alloc); if n > 0 { let ptr = result.raw.as_mut_ptr_inline(); @@ -2210,7 +2692,7 @@ mod spec_traits { fn spec_extend(&mut self, iter: I); } - impl SpecExtend for SmallVec + impl SpecExtend for SmallVec where I: Iterator, { @@ -2220,7 +2702,7 @@ mod spec_traits { } } - impl SpecExtend for SmallVec + impl SpecExtend for SmallVec where I: core::iter::TrustedLen, { @@ -2251,7 +2733,9 @@ mod spec_traits { } } - impl SpecExtend> for SmallVec { + impl SpecExtend> + for SmallVec + { fn spec_extend(&mut self, mut iter: IntoIter) { let slice = iter.as_slice(); let len = slice.len(); @@ -2277,7 +2761,7 @@ mod spec_traits { } } - impl<'a, T: 'a, const N: usize, I> SpecExtend<&'a T, I> for SmallVec + impl<'a, T: 'a, const N: usize, I, A: Allocator> SpecExtend<&'a T, I> for SmallVec where I: Iterator, T: Clone, @@ -2288,7 +2772,8 @@ mod spec_traits { } } - impl<'a, T: 'a, const N: usize> SpecExtend<&'a T, core::slice::Iter<'a, T>> for SmallVec + impl<'a, T: 'a, const N: usize, A: Allocator> SpecExtend<&'a T, core::slice::Iter<'a, T>> + for SmallVec where T: Copy, { @@ -2330,7 +2815,7 @@ mod spec_traits { unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range); } - impl SpecExtendFromWithin for SmallVec { + impl SpecExtendFromWithin for SmallVec { default unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range) { // SAFETY: Safety conditions are identical. unsafe { @@ -2339,7 +2824,7 @@ mod spec_traits { } } - impl SpecExtendFromWithin for SmallVec { + impl SpecExtendFromWithin for SmallVec { unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range) { let old_len = self.len(); @@ -2364,29 +2849,27 @@ mod spec_traits { } /// A trait for specializing the implementation of [`FromIterator`]. - /// - /// [`clone_from`]: Clone::clone_from - pub(crate) trait SpecFromIterator { - fn spec_from_iter(iter: I) -> Self; + pub(crate) trait SpecFromIterator { + fn spec_from_iter(iter: I, alloc: A) -> Self; } - impl SpecFromIterator for SmallVec + impl SpecFromIterator for SmallVec where I: Iterator, { #[inline] - default fn spec_from_iter(iter: I) -> Self { - Self::from_iter_fallback(iter) + default fn spec_from_iter(iter: I, alloc: A) -> Self { + Self::from_iter_fallback(iter, alloc) } } - impl SpecFromIterator for SmallVec + impl SpecFromIterator for SmallVec where I: core::iter::TrustedLen, { - fn spec_from_iter(iter: I) -> Self { + fn spec_from_iter(iter: I, alloc: A) -> Self { let mut v = match iter.size_hint() { - (_, Some(upper)) => SmallVec::with_capacity(upper), + (_, Some(upper)) => SmallVec::with_capacity_in(upper, alloc), // TrustedLen contract guarantees that `size_hint() == (_, None)` means that there // are more than `usize::MAX` elements. // Since the previous branch would eagerly panic if the capacity is too large @@ -2406,14 +2889,14 @@ mod spec_traits { fn spec_clone_from(&mut self, source: &[T]); } - impl SpecCloneFrom for SmallVec { + impl SpecCloneFrom for SmallVec { #[inline] default fn spec_clone_from(&mut self, source: &[T]) { self.clone_from_fallback(source); } } - impl SpecCloneFrom for SmallVec { + impl SpecCloneFrom for SmallVec { fn spec_clone_from(&mut self, source: &[T]) { self.clear(); self.extend_from_slice(source); @@ -2422,26 +2905,26 @@ mod spec_traits { /// A trait for specializing the implementation of [`From`] /// with the source type being slices. - pub(crate) trait SpecFromSlice { + pub(crate) trait SpecFromSlice { /// Creates a `SmallVec` value based on the contents of `slice`. /// This will use the inline storage, not the heap. /// /// # Safety /// /// The caller must ensure that `slice.len() <= Self::inline_size()`. - unsafe fn spec_from(slice: &[T]) -> Self; + unsafe fn spec_from(slice: &[T], alloc: A) -> Self; } - impl SpecFromSlice for SmallVec { - default unsafe fn spec_from(slice: &[T]) -> Self { + impl SpecFromSlice for SmallVec { + default unsafe fn spec_from(slice: &[T], alloc: A) -> Self { // SAFETY: Safety conditions are identical. - unsafe { Self::from_slice_fallback(slice) } + unsafe { Self::from_slice_fallback(slice, alloc) } } } - impl SpecFromSlice for SmallVec { - unsafe fn spec_from(slice: &[T]) -> Self { - let mut v = Self::new(); + impl SpecFromSlice for SmallVec { + unsafe fn spec_from(slice: &[T], alloc: A) -> Self { + let mut v = Self::new_in(alloc); let src = slice.as_ptr(); let len = slice.len(); @@ -2466,18 +2949,18 @@ mod spec_traits { /// Fallback functions for various specialized methods. These are kept in /// a separate implementation block for easy access whenever specialization is /// disabled. -impl SmallVec { +impl SmallVec { /// Creates a `Smallvec` value where `elem` is repeated `n` times. /// This will use the inline storage, not the heap. /// /// # Safety /// /// The caller must ensure that `n <= Self::inline_size()`. - unsafe fn from_elem_fallback(elem: T, n: usize) -> Self + unsafe fn from_elem_fallback(elem: T, n: usize, alloc: A) -> Self where T: Clone, { - let mut result = Self::new(); + let mut result = Self::new_in(alloc); if n > 0 { let ptr = result.raw.as_mut_ptr_inline(); @@ -2557,12 +3040,12 @@ impl SmallVec { } } - fn from_iter_fallback(iter: I) -> Self + fn from_iter_fallback(iter: I, alloc: A) -> Self where I: Iterator, { let (size, _) = iter.size_hint(); - let mut v = Self::with_capacity(size); + let mut v = Self::with_capacity_in(size, alloc); for x in iter { v.push(x); } @@ -2593,11 +3076,11 @@ impl SmallVec { /// # Safety /// /// The caller must ensure that `slice.len() <= Self::inline_size()`. - unsafe fn from_slice_fallback(slice: &[T]) -> Self + unsafe fn from_slice_fallback(slice: &[T], alloc: A) -> Self where T: Clone, { - let mut v = Self::new(); + let mut v = Self::new_in(alloc); let src = slice.as_ptr(); let len = slice.len(); @@ -2635,12 +3118,12 @@ impl From<&[T]> for SmallVec { unsafe { #[cfg(feature = "specialization")] { - >::spec_from(slice) + >::spec_from(slice, Global) } #[cfg(not(feature = "specialization"))] { - Self::from_slice_fallback(slice) + Self::from_slice_fallback(slice, Global) } } } @@ -2673,7 +3156,7 @@ impl From<[T; M]> for SmallVec { if M > N { // If M > N, we'd have to heap allocate anyway, // so delegate for Vec for the allocation. - Self::from(Vec::from(array)) + Self::from_vec(Vec::from(array)) } else { // M <= N let mut this = Self::new(); @@ -2689,16 +3172,33 @@ impl From<[T; M]> for SmallVec { } } +#[cfg(feature = "allocator-api2")] impl From> for SmallVec { fn from(array: Vec) -> Self { Self::from_vec(array) } } -impl Clone for SmallVec { +macro_rules! make_from_impl { + ($vec:path, $from_vec:ident) => { + impl From<$vec> for SmallVec { + fn from(array: $vec) -> Self { + Self::$from_vec(array) + } + } + }; +} + +#[cfg(feature = "allocator-api2")] +make_from_impl!(allocator_api2::vec::Vec, from_vec2); +#[cfg(not(feature = "allocator-api2"))] +make_from_impl!(Vec, from_vec); + +impl Clone for SmallVec { #[inline] - fn clone(&self) -> SmallVec { - SmallVec::from(self.as_slice()) + fn clone(&self) -> SmallVec { + let alloc = self.raw.allocator.clone(); + Self::from_slice_in(self.as_slice(), alloc) } #[inline] @@ -2715,14 +3215,15 @@ impl Clone for SmallVec { } } -impl Clone for IntoIter { +impl Clone for IntoIter { #[inline] - fn clone(&self) -> IntoIter { - SmallVec::from(self.as_slice()).into_iter() + fn clone(&self) -> IntoIter { + let alloc = self.raw.allocator.clone(); + SmallVec::from_slice_in(self.as_slice(), alloc).into_iter() } } -impl Extend for SmallVec { +impl Extend for SmallVec { #[inline] fn extend>(&mut self, iter: I) { #[cfg(feature = "specialization")] @@ -2737,7 +3238,7 @@ impl Extend for SmallVec { } } -impl<'a, T: Clone + 'a, const N: usize> Extend<&'a T> for SmallVec { +impl<'a, T: Clone + 'a, const N: usize, A: Allocator> Extend<&'a T> for SmallVec { #[inline] fn extend>(&mut self, iter: I) { #[cfg(feature = "specialization")] @@ -2757,12 +3258,12 @@ impl core::iter::FromIterator for SmallVec { fn from_iter>(iter: I) -> Self { #[cfg(feature = "specialization")] { - spec_traits::SpecFromIterator::::spec_from_iter(iter.into_iter()) + spec_traits::SpecFromIterator::::spec_from_iter(iter.into_iter(), Global) } #[cfg(not(feature = "specialization"))] { - Self::from_iter_fallback(iter.into_iter()) + Self::from_iter_fallback(iter.into_iter(), Global) } } } @@ -2789,8 +3290,8 @@ macro_rules! smallvec_inline { }); } -impl IntoIterator for SmallVec { - type IntoIter = IntoIter; +impl IntoIterator for SmallVec { + type IntoIter = IntoIter; type Item = T; fn into_iter(self) -> Self::IntoIter { // SAFETY: we move out of this.raw by reading the value at its address, which is @@ -2799,7 +3300,7 @@ impl IntoIterator for SmallVec { // Set SmallVec len to zero as `IntoIter` drop handles dropping of the elements let this = ManuallyDrop::new(self); IntoIter { - raw: (&this.raw as *const RawSmallVec).read(), + raw: (&this.raw as *const RawSmallVec).read(), begin: 0, end: this.len, _marker: PhantomData, @@ -2808,7 +3309,7 @@ impl IntoIterator for SmallVec { } } -impl<'a, T, const N: usize> IntoIterator for &'a SmallVec { +impl<'a, T, const N: usize, A: Allocator> IntoIterator for &'a SmallVec { type IntoIter = core::slice::Iter<'a, T>; type Item = &'a T; fn into_iter(self) -> Self::IntoIter { @@ -2816,7 +3317,7 @@ impl<'a, T, const N: usize> IntoIterator for &'a SmallVec { } } -impl<'a, T, const N: usize> IntoIterator for &'a mut SmallVec { +impl<'a, T, const N: usize, A: Allocator> IntoIterator for &'a mut SmallVec { type IntoIter = core::slice::IterMut<'a, T>; type Item = &'a mut T; fn into_iter(self) -> Self::IntoIter { @@ -2824,18 +3325,19 @@ impl<'a, T, const N: usize> IntoIterator for &'a mut SmallVec { } } -impl PartialEq> for SmallVec +impl + PartialEq> for SmallVec where T: PartialEq, { #[inline] - fn eq(&self, other: &SmallVec) -> bool { + fn eq(&self, other: &SmallVec) -> bool { self.as_slice().eq(other.as_slice()) } } -impl Eq for SmallVec where T: Eq {} +impl Eq for SmallVec where T: Eq {} -impl PartialEq<[U; M]> for SmallVec +impl PartialEq<[U; M]> for SmallVec where T: PartialEq, { @@ -2855,7 +3357,7 @@ where } } -impl PartialEq<[U]> for SmallVec +impl PartialEq<[U]> for SmallVec where T: PartialEq, { @@ -2865,7 +3367,7 @@ where } } -impl PartialEq<&[U]> for SmallVec +impl PartialEq<&[U]> for SmallVec where T: PartialEq, { @@ -2875,7 +3377,7 @@ where } } -impl PartialEq<&mut [U]> for SmallVec +impl PartialEq<&mut [U]> for SmallVec where T: PartialEq, { @@ -2885,73 +3387,74 @@ where } } -impl PartialOrd for SmallVec +impl + PartialOrd> for SmallVec where T: PartialOrd, { #[inline] - fn partial_cmp(&self, other: &SmallVec) -> Option { + fn partial_cmp(&self, other: &SmallVec) -> Option { self.as_slice().partial_cmp(other.as_slice()) } } -impl Ord for SmallVec +impl Ord for SmallVec where T: Ord, { #[inline] - fn cmp(&self, other: &SmallVec) -> core::cmp::Ordering { + fn cmp(&self, other: &SmallVec) -> core::cmp::Ordering { self.as_slice().cmp(other.as_slice()) } } -impl Hash for SmallVec { +impl Hash for SmallVec { fn hash(&self, state: &mut H) { self.as_slice().hash(state) } } -impl Borrow<[T]> for SmallVec { +impl Borrow<[T]> for SmallVec { #[inline] fn borrow(&self) -> &[T] { self.as_slice() } } -impl BorrowMut<[T]> for SmallVec { +impl BorrowMut<[T]> for SmallVec { #[inline] fn borrow_mut(&mut self) -> &mut [T] { self.as_mut_slice() } } -impl AsRef<[T]> for SmallVec { +impl AsRef<[T]> for SmallVec { #[inline] fn as_ref(&self) -> &[T] { self.as_slice() } } -impl AsMut<[T]> for SmallVec { +impl AsMut<[T]> for SmallVec { #[inline] fn as_mut(&mut self) -> &mut [T] { self.as_mut_slice() } } -impl Debug for SmallVec { +impl Debug for SmallVec { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_list().entries(self.iter()).finish() } } -impl Debug for IntoIter { +impl Debug for IntoIter { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_tuple("IntoIter").field(&self.as_slice()).finish() } } -impl Debug for Drain<'_, T, N> { +impl Debug for Drain<'_, T, N, A> { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_tuple("Drain").field(&self.iter.as_slice()).finish() } @@ -2959,7 +3462,7 @@ impl Debug for Drain<'_, T, N> { #[cfg(feature = "serde")] #[cfg_attr(docsrs, doc(cfg(feature = "serde")))] -impl Serialize for SmallVec +impl Serialize for SmallVec where T: Serialize, { @@ -3019,7 +3522,7 @@ where } #[cfg(feature = "malloc_size_of")] -impl MallocShallowSizeOf for SmallVec { +impl MallocShallowSizeOf for SmallVec { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { if self.spilled() { unsafe { ops.malloc_size_of(self.as_ptr()) } @@ -3030,7 +3533,7 @@ impl MallocShallowSizeOf for SmallVec { } #[cfg(feature = "malloc_size_of")] -impl MallocSizeOf for SmallVec { +impl MallocSizeOf for SmallVec { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { let mut n = self.shallow_size_of(ops); for elem in self.iter() { @@ -3042,7 +3545,7 @@ impl MallocSizeOf for SmallVec { #[cfg(feature = "std")] #[cfg_attr(docsrs, doc(cfg(feature = "std")))] -impl io::Write for SmallVec { +impl io::Write for SmallVec { #[inline] fn write(&mut self, buf: &[u8]) -> io::Result { self.extend_from_slice(buf); @@ -3062,7 +3565,7 @@ impl io::Write for SmallVec { } #[cfg(feature = "bytes")] -unsafe impl BufMut for SmallVec { +unsafe impl BufMut for SmallVec { #[inline] fn remaining_mut(&self) -> usize { // A vector can never have more than isize::MAX bytes diff --git a/src/rawsmallvec.rs b/src/rawsmallvec.rs index dadbf958..3e1ef23c 100644 --- a/src/rawsmallvec.rs +++ b/src/rawsmallvec.rs @@ -1,3 +1,4 @@ +use super::{Allocator, Global}; use core::mem::{ManuallyDrop, MaybeUninit}; use core::ptr::NonNull; @@ -7,7 +8,16 @@ use core::ptr::NonNull; /// We store a `NonNull` instead of a `*mut T` so that type is covariant /// with respect to `T`, and since the heap pointer is never null. #[repr(C)] -pub union RawSmallVec { +pub union RawSmallVecUnion { pub inline: ManuallyDrop>, pub heap: (NonNull, usize), } + +/// A wrapper around a [`RawSmallVecUnion`] and an allocator. +/// It is assumed that any pointer inside the `heap` field +/// was allocated using this allocator. +#[repr(C)] +pub struct RawSmallVec { + pub inner: RawSmallVecUnion, + pub allocator: A, +} diff --git a/src/tests.rs b/src/tests.rs index e05f24d4..bdbd6650 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -389,14 +389,6 @@ fn test_append() { ); } -#[test] -#[should_panic] -fn test_invalid_grow() { - let mut v: SmallVec = SmallVec::new(); - v.extend(0..8); - v.grow(5); -} - #[test] #[should_panic] fn drain_overflow() { @@ -861,7 +853,7 @@ fn grow_to_shrink() { assert!(v.spilled()); v.clear(); // Shrink to inline. - v.grow(2); + v.shrink_to(2); assert!(!v.spilled()); assert_eq!(v.capacity(), 2); assert_eq!(v.len(), 0); @@ -888,20 +880,6 @@ fn uninhabited() { let _sv = SmallVec::::new(); } -#[test] -fn grow_spilled_same_size() { - let mut v: SmallVec = SmallVec::new(); - v.push(0); - v.push(1); - v.push(2); - assert!(v.spilled()); - assert_eq!(v.capacity(), 4); - // grow with the same capacity - v.grow(4); - assert_eq!(v.capacity(), 4); - assert_eq!(v[..], [0, 1, 2]); -} - #[test] fn const_generics() { let _v = SmallVec::::default();