diff --git a/benches/bench.rs b/benches/bench.rs index e881130..17a8864 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -1,9 +1,10 @@ #![allow(deprecated)] -use criterion::{criterion_group, criterion_main, Bencher, Criterion}; -use smallvec::{smallvec, SmallVec}; -use std::hint::black_box; -use std::time::Duration; +use { + criterion::{criterion_group, criterion_main, Bencher, Criterion}, + smallvec::{smallvec, SmallVec}, + std::{hint::black_box, time::Duration}, +}; const VEC_SIZE: usize = 16; const SPILLED_SIZE: usize = 100; @@ -26,27 +27,35 @@ impl Vector for Vec { fn new() -> Self { Self::with_capacity(VEC_SIZE) } + fn push(&mut self, val: T) { self.push(val) } + fn pop(&mut self) -> Option { self.pop() } + fn remove(&mut self, p: usize) -> T { self.remove(p) } + fn insert(&mut self, n: usize, val: T) { self.insert(n, val) } + fn from_elem(val: T, n: usize) -> Self { vec![val; n] } + fn from_elems(val: &[T]) -> Self { val.to_owned() } + fn extend_from_slice(&mut self, other: &[T]) { Vec::extend_from_slice(self, other) } + fn retain_mut(&mut self, f: F) where F: FnMut(&mut T) -> bool, @@ -59,27 +68,35 @@ impl Vector for SmallVec { fn new() -> Self { Self::new() } + fn push(&mut self, val: T) { self.push(val) } + fn pop(&mut self) -> Option { self.pop() } + fn remove(&mut self, p: usize) -> T { self.remove(p) } + fn insert(&mut self, n: usize, val: T) { self.insert(n, val) } + fn from_elem(val: T, n: usize) -> Self { smallvec![val; n] } + fn from_elems(val: &[T]) -> Self { SmallVec::from(val) } + fn extend_from_slice(&mut self, other: &[T]) { SmallVec::extend_from_slice(self, other) } + fn retain_mut(&mut self, f: F) where F: FnMut(&mut T) -> bool, @@ -100,8 +117,8 @@ macro_rules! make_benches { } } -/* ---------- Bench generation (same list, just using the new macro) - * ---------- */ +// ---------- Bench generation (same list, just using the new macro) +// ---------- make_benches! { SmallVec { bench_push => gen_push(SPILLED_SIZE as _), diff --git a/src/bytes.rs b/src/bytes.rs new file mode 100644 index 0000000..c261032 --- /dev/null +++ b/src/bytes.rs @@ -0,0 +1,65 @@ +use { + super::SmallVec, + bytes::{buf::UninitSlice, BufMut}, +}; + +unsafe impl BufMut for SmallVec { + fn remaining_mut(&self) -> usize { + // A vector can never have more than isize::MAX bytes + isize::MAX as usize - self.len() + } + + unsafe fn advance_mut(&mut self, cnt: usize) { + let len = self.len(); + let remaining = self.capacity() - len; + + if remaining < cnt { + panic!("advance out of bounds: the len is {remaining} but advancing by {cnt}"); + } + + // Addition will not overflow since the sum is at most the capacity. + self.set_len(len + cnt); + } + + fn chunk_mut(&mut self) -> &mut UninitSlice { + if self.capacity() == self.len() { + self.reserve(64); // Grow the smallvec + } + + let cap = self.capacity(); + let len = self.len(); + + let ptr = self.as_mut_ptr(); + // SAFETY: Since `ptr` is valid for `cap` bytes, `ptr.add(len)` must be + // valid for `cap - len` bytes. The subtraction will not underflow since + // `len <= cap`. + unsafe { UninitSlice::from_raw_parts_mut(ptr.add(len), cap - len) } + } + + // Specialize these methods so they can skip checking `remaining_mut` + // and `advance_mut`. + fn put(&mut self, mut src: T) + where + Self: Sized, + { + // In case the src isn't contiguous, reserve upfront. + self.reserve(src.remaining()); + + while src.has_remaining() { + let s = src.chunk(); + let l = s.len(); + self.extend_from_slice(s); + src.advance(l); + } + } + + fn put_slice(&mut self, src: &[u8]) { + self.extend_from_slice(src); + } + + fn put_bytes(&mut self, val: u8, cnt: usize) { + // If the addition overflows, then the `resize` will fail. + let new_len = self.len().saturating_add(cnt); + self.resize(new_len, val); + } +} diff --git a/src/comparisons.rs b/src/comparisons.rs new file mode 100644 index 0000000..478666c --- /dev/null +++ b/src/comparisons.rs @@ -0,0 +1,52 @@ +use super::SmallVec; + +impl, U, const N: usize, const M: usize> PartialEq> + for SmallVec +{ + fn eq(&self, other: &SmallVec) -> bool { + self.as_slice().eq(other.as_slice()) + } +} +impl Eq for SmallVec where T: Eq {} + +impl, U, const N: usize, const M: usize> PartialEq<[U; M]> for SmallVec { + fn eq(&self, other: &[U; M]) -> bool { + self[..] == other[..] + } +} + +impl, U, const N: usize, const M: usize> PartialEq<&[U; M]> for SmallVec { + fn eq(&self, other: &&[U; M]) -> bool { + self[..] == other[..] + } +} + +impl, U, const N: usize> PartialEq<[U]> for SmallVec { + fn eq(&self, other: &[U]) -> bool { + self[..] == other[..] + } +} + +impl, U, const N: usize> PartialEq<&[U]> for SmallVec { + fn eq(&self, other: &&[U]) -> bool { + self[..] == other[..] + } +} + +impl, U, const N: usize> PartialEq<&mut [U]> for SmallVec { + fn eq(&self, other: &&mut [U]) -> bool { + self[..] == other[..] + } +} + +impl PartialOrd for SmallVec { + fn partial_cmp(&self, other: &SmallVec) -> Option { + self.as_slice().partial_cmp(other.as_slice()) + } +} + +impl Ord for SmallVec { + fn cmp(&self, other: &SmallVec) -> core::cmp::Ordering { + self.as_slice().cmp(other.as_slice()) + } +} diff --git a/src/lib.rs b/src/lib.rs index af12044..a46d58f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -65,41 +65,36 @@ pub extern crate alloc; #[cfg(any(test, feature = "std"))] extern crate std; +#[cfg(feature = "bytes")] +mod bytes; +mod comparisons; +#[cfg(feature = "malloc_size_of")] +mod mallocsizeof; mod rawsmallvec; +mod references; +#[cfg(feature = "serde")] +mod serde; +mod taggedlen; #[cfg(test)] mod tests; -use alloc::alloc::Layout; -use alloc::boxed::Box; -use alloc::vec; -use alloc::vec::Vec; -#[cfg(feature = "bytes")] -use bytes::{buf::UninitSlice, BufMut}; -use core::borrow::Borrow; -use core::borrow::BorrowMut; -use core::fmt::Debug; -use core::hash::{Hash, Hasher}; -use core::marker::PhantomData; -use core::mem::align_of; -use core::mem::size_of; -use core::mem::ManuallyDrop; -use core::mem::MaybeUninit; -use core::ptr::copy; -use core::ptr::copy_nonoverlapping; -use core::ptr::NonNull; -#[cfg(feature = "malloc_size_of")] -use malloc_size_of::{MallocShallowSizeOf, MallocSizeOf, MallocSizeOfOps}; +#[cfg(feature = "std")] +use std::io::{Result as IoResult, Write}; +use { + alloc::{alloc::Layout, boxed::Box, vec::Vec}, + core::{ + fmt::Debug, + hash::{Hash, Hasher}, + iter::repeat_n, + marker::PhantomData, + mem::{align_of, size_of, ManuallyDrop, MaybeUninit}, + ptr::{copy, copy_nonoverlapping, NonNull}, + }, +}; #[cfg(feature = "internals")] -pub use rawsmallvec::RawSmallVec; +pub use {rawsmallvec::RawSmallVec, taggedlen::TaggedLen}; #[cfg(not(feature = "internals"))] -use rawsmallvec::RawSmallVec; -#[cfg(feature = "serde")] -use serde_core::{ - de::{Deserialize, Deserializer, SeqAccess, Visitor}, - ser::{Serialize, SerializeSeq, Serializer}, -}; -#[cfg(feature = "std")] -use std::io; +use {rawsmallvec::RawSmallVec, taggedlen::TaggedLen}; /// Error type for APIs with fallible heap allocation #[derive(Debug)] @@ -129,12 +124,6 @@ fn infallible(result: Result) -> T { } } -/// Helper function to check if a type is a ZST. -#[inline] -const fn is_zst() -> bool { - const { size_of::() == 0 } -} - #[inline] /// A local copy of [`core::slice::range`]. The latter function is unstable /// and thus cannot be used yet. @@ -170,173 +159,6 @@ where core::ops::Range { start, end } } -impl RawSmallVec { - const IS_ZST: bool = is_zst::(); - - #[inline] - const fn new() -> Self { - Self::new_inline(MaybeUninit::uninit()) - } - #[inline] - const fn new_inline(inline: MaybeUninit<[T; N]>) -> Self { - Self { - inline: ManuallyDrop::new(inline), - } - } - #[inline] - const fn new_heap(ptr: NonNull, capacity: usize) -> Self { - Self { - heap: (ptr, capacity), - } - } - - #[inline] - const fn as_ptr_inline(&self) -> *const T { - // SAFETY: it is safe because we aren't reading the value, just getting a - // reference to it. reading it would be UB potentially, but for that downstream - // unsafe is required - (unsafe { &raw const self.inline }) as *mut T - } - - #[inline] - const fn as_mut_ptr_inline(&mut self) -> *mut T { - // SAFETY: same as above - (unsafe { &raw mut self.inline }) as *mut T - } - - /// # Safety - /// - /// The vector must be on the heap - #[inline] - const unsafe fn as_ptr_heap(&self) -> *const T { - self.heap.0.as_ptr() - } - - /// # Safety - /// - /// The vector must be on the heap - #[inline] - const unsafe fn as_mut_ptr_heap(&mut self) -> *mut T { - self.heap.0.as_ptr() - } - - /// # Safety - /// - /// `new_capacity` must be non zero, and greater or equal to the length. - /// T must not be a ZST. - unsafe fn try_grow_raw( - &mut self, - len: TaggedLen, - new_capacity: usize, - ) -> Result<(), CollectionAllocErr> { - use alloc::alloc::{alloc, realloc}; - debug_assert!(!Self::IS_ZST); - debug_assert!(new_capacity > 0); - debug_assert!(new_capacity >= len.value()); - - let was_on_heap = len.on_heap(); - let ptr = if was_on_heap { - self.as_mut_ptr_heap() - } else { - self.as_mut_ptr_inline() - }; - let len = len.value(); - - let new_layout = - Layout::array::(new_capacity).map_err(|_| CollectionAllocErr::CapacityOverflow)?; - if new_layout.size() > isize::MAX as usize { - return Err(CollectionAllocErr::CapacityOverflow); - } - - let new_ptr = if !was_on_heap { - // get a fresh allocation - let new_ptr = alloc(new_layout) as *mut T; // `new_layout` has nonzero size. - let new_ptr = - NonNull::new(new_ptr).ok_or(CollectionAllocErr::AllocErr { layout: new_layout })?; - copy_nonoverlapping(ptr, new_ptr.as_ptr(), len); - new_ptr - } else { - // use realloc - - // this can't overflow since we already constructed an equivalent layout during - // the previous allocation - let old_layout = - Layout::from_size_align_unchecked(self.heap.1 * size_of::(), align_of::()); - - // SAFETY: ptr was allocated with this allocator - // old_layout is the same as the layout used to allocate the previous memory - // block new_layout.size() is greater than zero - // does not overflow when rounded up to alignment. since it was constructed - // with Layout::array - let new_ptr = realloc(ptr as *mut u8, old_layout, new_layout.size()) as *mut T; - NonNull::new(new_ptr).ok_or(CollectionAllocErr::AllocErr { layout: new_layout })? - }; - *self = Self::new_heap(new_ptr, new_capacity); - Ok(()) - } -} - -/// Vec guarantees that its length is always less than [`isize::MAX`] in -/// *bytes*. -/// -/// For a non ZST, this means that the length is less than `isize::MAX` objects, -/// which implies we have at least one free bit we can use. We use the least -/// significant bit for the tag. And store the length in the `usize::BITS - 1` -/// most significant bits. -/// -/// For a ZST, we never use the heap, so we just store the length directly. -#[repr(transparent)] -struct TaggedLen(usize, PhantomData); - -// Clone and Copy must be manually implemented because the generic interferes -// with the derive attribute implementations. -impl Clone for TaggedLen { - #[inline] - fn clone(&self) -> Self { - Self(self.0, PhantomData) - } - - #[inline] - fn clone_from(&mut self, source: &Self) { - self.0 = source.0; - } -} - -impl Copy for TaggedLen {} - -impl TaggedLen { - const IS_ZST: bool = is_zst::(); - #[inline] - pub const fn new(len: usize, on_heap: bool) -> Self { - if Self::IS_ZST { - debug_assert!(!on_heap); - Self(len, PhantomData) - } else { - debug_assert!(len < isize::MAX as usize); - Self((len << 1) | on_heap as usize, PhantomData) - } - } - - #[inline] - #[must_use] - pub const fn on_heap(self) -> bool { - if Self::IS_ZST { - false - } else { - (self.0 & 1_usize) == 1 - } - } - - #[inline] - pub const fn value(self) -> usize { - if Self::IS_ZST { - self.0 - } else { - self.0 >> 1 - } - } -} - #[repr(C)] pub struct SmallVec { len: TaggedLen, @@ -477,7 +299,7 @@ impl<'a, T: 'a, const N: usize> Drop for Drain<'a, T, N> { // raw pointers to it which some unsafe code might rely on. let vec_ptr = vec.as_mut().as_mut_ptr(); // May be replaced with the line below later, once this crate's MSRV is >= 1.87. - //let drop_offset = drop_ptr.offset_from_unsigned(vec_ptr); + // let drop_offset = drop_ptr.offset_from_unsigned(vec_ptr); let drop_offset = drop_ptr.offset_from(vec_ptr) as usize; let to_drop = core::ptr::slice_from_raw_parts_mut(vec_ptr.add(drop_offset), drop_len); core::ptr::drop_in_place(to_drop); @@ -913,8 +735,7 @@ impl SmallVec { /// # Examples /// /// ``` - /// use smallvec::SmallVec; - /// use std::mem::MaybeUninit; + /// use {smallvec::SmallVec, std::mem::MaybeUninit}; /// /// let buf = [1, 2, 3, 4, 5, 0, 0, 0]; /// let small_vec = unsafe { SmallVec::from_buf_and_len_unchecked(MaybeUninit::new(buf), 5) }; @@ -937,7 +758,7 @@ impl SmallVec { } impl SmallVec { - const IS_ZST: bool = is_zst::(); + const IS_ZST: bool = size_of::() == 0; #[inline] pub fn from_vec(vec: Vec) -> Self { @@ -1115,7 +936,6 @@ impl SmallVec { // Since self is a &mut, passing it to a function would invalidate the slice // iterator. vec: core::ptr::NonNull::new_unchecked(self as *mut _), - //vec: core::ptr::NonNull::from(self), } } } @@ -2113,29 +1933,14 @@ impl Drop for IntoIter { } } -impl core::ops::Deref for SmallVec { - type Target = [T]; - - #[inline] - fn deref(&self) -> &Self::Target { - self.as_slice() - } -} -impl core::ops::DerefMut for SmallVec { - #[inline] - fn deref_mut(&mut self) -> &mut Self::Target { - self.as_mut_slice() - } -} - /// This function is used in the [`smallvec`] macro. /// It is recommended to use the macro instead of using thís function. #[doc(hidden)] #[track_caller] pub fn from_elem(elem: T, n: usize) -> SmallVec { if n > SmallVec::::inline_size() { - // Standard Rust vectors are already specialized. - SmallVec::::from_vec(vec![elem; n]) + // Standard Rust iterators are already specialized. + repeat_n(elem, n).collect() } else { #[cfg(feature = "specialization")] { @@ -2792,6 +2597,7 @@ macro_rules! smallvec_inline { impl IntoIterator for SmallVec { type IntoIter = IntoIter; type Item = T; + fn into_iter(self) -> Self::IntoIter { // SAFETY: we move out of this.raw by reading the value at its address, which is // fine since we don't drop it @@ -2811,6 +2617,7 @@ impl IntoIterator for SmallVec { impl<'a, T, const N: usize> IntoIterator for &'a SmallVec { type IntoIter = core::slice::Iter<'a, T>; type Item = &'a T; + fn into_iter(self) -> Self::IntoIter { self.iter() } @@ -2819,126 +2626,18 @@ impl<'a, T, const N: usize> IntoIterator for &'a SmallVec { impl<'a, T, const N: usize> IntoIterator for &'a mut SmallVec { type IntoIter = core::slice::IterMut<'a, T>; type Item = &'a mut T; + fn into_iter(self) -> Self::IntoIter { self.iter_mut() } } -impl PartialEq> for SmallVec -where - T: PartialEq, -{ - #[inline] - fn eq(&self, other: &SmallVec) -> bool { - self.as_slice().eq(other.as_slice()) - } -} -impl Eq for SmallVec where T: Eq {} - -impl PartialEq<[U; M]> for SmallVec -where - T: PartialEq, -{ - #[inline] - fn eq(&self, other: &[U; M]) -> bool { - self[..] == other[..] - } -} - -impl PartialEq<&[U; M]> for SmallVec -where - T: PartialEq, -{ - #[inline] - fn eq(&self, other: &&[U; M]) -> bool { - self[..] == other[..] - } -} - -impl PartialEq<[U]> for SmallVec -where - T: PartialEq, -{ - #[inline] - fn eq(&self, other: &[U]) -> bool { - self[..] == other[..] - } -} - -impl PartialEq<&[U]> for SmallVec -where - T: PartialEq, -{ - #[inline] - fn eq(&self, other: &&[U]) -> bool { - self[..] == other[..] - } -} - -impl PartialEq<&mut [U]> for SmallVec -where - T: PartialEq, -{ - #[inline] - fn eq(&self, other: &&mut [U]) -> bool { - self[..] == other[..] - } -} - -impl PartialOrd for SmallVec -where - T: PartialOrd, -{ - #[inline] - fn partial_cmp(&self, other: &SmallVec) -> Option { - self.as_slice().partial_cmp(other.as_slice()) - } -} - -impl Ord for SmallVec -where - T: Ord, -{ - #[inline] - fn cmp(&self, other: &SmallVec) -> core::cmp::Ordering { - self.as_slice().cmp(other.as_slice()) - } -} - impl Hash for SmallVec { fn hash(&self, state: &mut H) { self.as_slice().hash(state) } } -impl Borrow<[T]> for SmallVec { - #[inline] - fn borrow(&self) -> &[T] { - self.as_slice() - } -} - -impl BorrowMut<[T]> for SmallVec { - #[inline] - fn borrow_mut(&mut self) -> &mut [T] { - self.as_mut_slice() - } -} - -impl AsRef<[T]> for SmallVec { - #[inline] - fn as_ref(&self) -> &[T] { - self.as_slice() - } -} - -impl AsMut<[T]> for SmallVec { - #[inline] - fn as_mut(&mut self) -> &mut [T] { - self.as_mut_slice() - } -} - impl Debug for SmallVec { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_list().entries(self.iter()).finish() @@ -2957,174 +2656,19 @@ impl Debug for Drain<'_, T, N> { } } -#[cfg(feature = "serde")] -#[cfg_attr(docsrs, doc(cfg(feature = "serde")))] -impl Serialize for SmallVec -where - T: Serialize, -{ - fn serialize(&self, serializer: S) -> Result { - let mut state = serializer.serialize_seq(Some(self.len()))?; - for item in self { - state.serialize_element(item)?; - } - state.end() - } -} - -#[cfg(feature = "serde")] -#[cfg_attr(docsrs, doc(cfg(feature = "serde")))] -impl<'de, T, const N: usize> Deserialize<'de> for SmallVec -where - T: Deserialize<'de>, -{ - fn deserialize>(deserializer: D) -> Result { - deserializer.deserialize_seq(SmallVecVisitor { - phantom: PhantomData, - }) - } -} - -#[cfg(feature = "serde")] -struct SmallVecVisitor { - phantom: PhantomData, -} - -#[cfg(feature = "serde")] -impl<'de, T, const N: usize> Visitor<'de> for SmallVecVisitor -where - T: Deserialize<'de>, -{ - type Value = SmallVec; - - fn expecting(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - formatter.write_str("a sequence") - } - - fn visit_seq(self, mut seq: B) -> Result - where - B: SeqAccess<'de>, - { - use serde_core::de::Error; - let len = seq.size_hint().unwrap_or(0); - let mut values = SmallVec::new(); - values.try_reserve(len).map_err(B::Error::custom)?; - - while let Some(value) = seq.next_element()? { - values.push(value); - } - - Ok(values) - } -} - -#[cfg(feature = "malloc_size_of")] -impl MallocShallowSizeOf for SmallVec { - fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - if self.spilled() { - unsafe { ops.malloc_size_of(self.as_ptr()) } - } else { - 0 - } - } -} - -#[cfg(feature = "malloc_size_of")] -impl MallocSizeOf for SmallVec { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - let mut n = self.shallow_size_of(ops); - for elem in self.iter() { - n += elem.size_of(ops); - } - n - } -} - #[cfg(feature = "std")] -#[cfg_attr(docsrs, doc(cfg(feature = "std")))] -impl io::Write for SmallVec { - #[inline] - fn write(&mut self, buf: &[u8]) -> io::Result { +impl Write for SmallVec { + fn write(&mut self, buf: &[u8]) -> IoResult { self.extend_from_slice(buf); Ok(buf.len()) } - #[inline] - fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + fn write_all(&mut self, buf: &[u8]) -> IoResult<()> { self.extend_from_slice(buf); Ok(()) } - #[inline] - fn flush(&mut self) -> io::Result<()> { + fn flush(&mut self) -> IoResult<()> { Ok(()) } } - -#[cfg(feature = "bytes")] -unsafe impl BufMut for SmallVec { - #[inline] - fn remaining_mut(&self) -> usize { - // A vector can never have more than isize::MAX bytes - isize::MAX as usize - self.len() - } - - #[inline] - unsafe fn advance_mut(&mut self, cnt: usize) { - let len = self.len(); - let remaining = self.capacity() - len; - - if remaining < cnt { - panic!("advance out of bounds: the len is {remaining} but advancing by {cnt}"); - } - - // Addition will not overflow since the sum is at most the capacity. - self.set_len(len + cnt); - } - - #[inline] - fn chunk_mut(&mut self) -> &mut UninitSlice { - if self.capacity() == self.len() { - self.reserve(64); // Grow the smallvec - } - - let cap = self.capacity(); - let len = self.len(); - - let ptr = self.as_mut_ptr(); - // SAFETY: Since `ptr` is valid for `cap` bytes, `ptr.add(len)` must be - // valid for `cap - len` bytes. The subtraction will not underflow since - // `len <= cap`. - unsafe { UninitSlice::from_raw_parts_mut(ptr.add(len), cap - len) } - } - - // Specialize these methods so they can skip checking `remaining_mut` - // and `advance_mut`. - #[inline] - fn put(&mut self, mut src: T) - where - Self: Sized, - { - // In case the src isn't contiguous, reserve upfront. - self.reserve(src.remaining()); - - while src.has_remaining() { - let s = src.chunk(); - let l = s.len(); - self.extend_from_slice(s); - src.advance(l); - } - } - - #[inline] - fn put_slice(&mut self, src: &[u8]) { - self.extend_from_slice(src); - } - - #[inline] - fn put_bytes(&mut self, val: u8, cnt: usize) { - // If the addition overflows, then the `resize` will fail. - let new_len = self.len().saturating_add(cnt); - self.resize(new_len, val); - } -} diff --git a/src/mallocsizeof.rs b/src/mallocsizeof.rs new file mode 100644 index 0000000..6377b63 --- /dev/null +++ b/src/mallocsizeof.rs @@ -0,0 +1,24 @@ +use { + super::SmallVec, + malloc_size_of::{MallocShallowSizeOf, MallocSizeOf, MallocSizeOfOps}, +}; + +impl MallocShallowSizeOf for SmallVec { + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + if self.spilled() { + unsafe { ops.malloc_size_of(self.as_ptr()) } + } else { + 0 + } + } +} + +impl MallocSizeOf for SmallVec { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + let mut n = self.shallow_size_of(ops); + for elem in self.iter() { + n += elem.size_of(ops); + } + n + } +} diff --git a/src/rawsmallvec.rs b/src/rawsmallvec.rs index dadbf95..9cf1ed6 100644 --- a/src/rawsmallvec.rs +++ b/src/rawsmallvec.rs @@ -1,5 +1,11 @@ -use core::mem::{ManuallyDrop, MaybeUninit}; -use core::ptr::NonNull; +use { + super::{CollectionAllocErr, TaggedLen}, + core::{ + alloc::Layout, + mem::{ManuallyDrop, MaybeUninit}, + ptr::{copy_nonoverlapping, NonNull}, + }, +}; /// Either a stack array with `length <= N` or a heap array /// whose pointer and capacity are stored here. @@ -11,3 +17,106 @@ pub union RawSmallVec { pub inline: ManuallyDrop>, pub heap: (NonNull, usize), } + +impl RawSmallVec { + const IS_ZST: bool = size_of::() == 0; + + pub const fn new() -> Self { + Self::new_inline(MaybeUninit::uninit()) + } + + pub const fn new_inline(inline: MaybeUninit<[T; N]>) -> Self { + Self { + inline: ManuallyDrop::new(inline), + } + } + + pub const fn new_heap(ptr: NonNull, capacity: usize) -> Self { + Self { + heap: (ptr, capacity), + } + } + + pub const fn as_ptr_inline(&self) -> *const T { + // SAFETY: it is safe because we aren't reading the value, just getting a + // reference to it. reading it would be UB potentially, but for that downstream + // unsafe is required + #[allow(unused_unsafe, reason = "Unsafe in MSRV 1.83.0")] + (unsafe { &raw const self.inline }).cast() + } + + pub const fn as_mut_ptr_inline(&mut self) -> *mut T { + // SAFETY: same as above + #[allow(unused_unsafe, reason = "Unsafe in MSRV 1.83.0")] + (unsafe { &raw mut self.inline }).cast() + } + + /// # Safety + /// + /// The vector must be on the heap + pub const unsafe fn as_ptr_heap(&self) -> *const T { + self.heap.0.as_ptr() + } + + /// # Safety + /// + /// The vector must be on the heap + pub const unsafe fn as_mut_ptr_heap(&mut self) -> *mut T { + self.heap.0.as_ptr() + } + + /// # Safety + /// + /// `new_capacity` must be non zero, and greater or equal to the length. + /// T must not be a ZST. + pub unsafe fn try_grow_raw( + &mut self, + len: TaggedLen, + new_capacity: usize, + ) -> Result<(), CollectionAllocErr> { + use alloc::alloc::{alloc, realloc}; + debug_assert!(!Self::IS_ZST); + debug_assert!(new_capacity > 0); + debug_assert!(new_capacity >= len.value()); + + let was_on_heap = len.on_heap(); + let ptr = if was_on_heap { + self.as_mut_ptr_heap() + } else { + self.as_mut_ptr_inline() + }; + let len = len.value(); + + let new_layout = + Layout::array::(new_capacity).map_err(|_| CollectionAllocErr::CapacityOverflow)?; + if new_layout.size() > isize::MAX as usize { + return Err(CollectionAllocErr::CapacityOverflow); + } + + let new_ptr = if !was_on_heap { + // get a fresh allocation + let new_ptr = alloc(new_layout) as *mut T; // `new_layout` has nonzero size. + let new_ptr = + NonNull::new(new_ptr).ok_or(CollectionAllocErr::AllocErr { layout: new_layout })?; + copy_nonoverlapping(ptr, new_ptr.as_ptr(), len); + new_ptr + } else { + // use realloc + + // this can't overflow since we already constructed an equivalent layout during + // the previous allocation + let old_layout = + Layout::from_size_align_unchecked(self.heap.1 * size_of::(), align_of::()); + + // SAFETY: ptr was allocated with this allocator + // old_layout is the same as the layout used to allocate the previous memory + // block new_layout.size() is greater than zero + // does not overflow when rounded up to alignment. since it was constructed + // with Layout::array + let new_ptr = realloc(ptr as *mut u8, old_layout, new_layout.size()) as *mut T; + NonNull::new(new_ptr).ok_or(CollectionAllocErr::AllocErr { layout: new_layout })? + }; + *self = Self::new_heap(new_ptr, new_capacity); + Ok(()) + } +} diff --git a/src/references.rs b/src/references.rs new file mode 100644 index 0000000..41d6fb7 --- /dev/null +++ b/src/references.rs @@ -0,0 +1,45 @@ +use { + super::SmallVec, + core::{ + borrow::{Borrow, BorrowMut}, + ops::{Deref, DerefMut}, + }, +}; + +impl Borrow<[T]> for SmallVec { + fn borrow(&self) -> &[T] { + self.as_slice() + } +} + +impl BorrowMut<[T]> for SmallVec { + fn borrow_mut(&mut self) -> &mut [T] { + self.as_mut_slice() + } +} + +impl AsRef<[T]> for SmallVec { + fn as_ref(&self) -> &[T] { + self.as_slice() + } +} + +impl AsMut<[T]> for SmallVec { + fn as_mut(&mut self) -> &mut [T] { + self.as_mut_slice() + } +} + +impl Deref for SmallVec { + type Target = [T]; + + fn deref(&self) -> &Self::Target { + self.as_slice() + } +} + +impl DerefMut for SmallVec { + fn deref_mut(&mut self) -> &mut Self::Target { + self.as_mut_slice() + } +} diff --git a/src/serde.rs b/src/serde.rs new file mode 100644 index 0000000..0f7d7e8 --- /dev/null +++ b/src/serde.rs @@ -0,0 +1,64 @@ +use { + super::SmallVec, + core::marker::PhantomData, + serde_core::{ + de::{SeqAccess, Visitor}, + ser::SerializeSeq, + Deserialize, Deserializer, Serialize, Serializer, + }, +}; + +impl Serialize for SmallVec +where + T: Serialize, +{ + fn serialize(&self, serializer: S) -> Result { + let mut state = serializer.serialize_seq(Some(self.len()))?; + for item in self { + state.serialize_element(item)?; + } + state.end() + } +} + +impl<'de, T, const N: usize> Deserialize<'de> for SmallVec +where + T: Deserialize<'de>, +{ + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_seq(SmallVecVisitor { + phantom: PhantomData, + }) + } +} + +struct SmallVecVisitor { + phantom: PhantomData, +} + +impl<'de, T, const N: usize> Visitor<'de> for SmallVecVisitor +where + T: Deserialize<'de>, +{ + type Value = SmallVec; + + fn expecting(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter.write_str("a sequence") + } + + fn visit_seq(self, mut seq: B) -> Result + where + B: SeqAccess<'de>, + { + use serde_core::de::Error; + let len = seq.size_hint().unwrap_or(0); + let mut values = SmallVec::new(); + values.try_reserve(len).map_err(B::Error::custom)?; + + while let Some(value) = seq.next_element()? { + values.push(value); + } + + Ok(values) + } +} diff --git a/src/taggedlen.rs b/src/taggedlen.rs new file mode 100644 index 0000000..817775f --- /dev/null +++ b/src/taggedlen.rs @@ -0,0 +1,52 @@ +use core::marker::PhantomData; + +/// Vec guarantees that its length is always less than [`isize::MAX`] in +/// *bytes*. +/// +/// For a non ZST, this means that the length is less than `isize::MAX` objects, +/// which implies we have at least one free bit we can use. We use the least +/// significant bit for the tag. And store the length in the `usize::BITS - 1` +/// most significant bits. +/// +/// For a ZST, we never use the heap, so we just store the length directly. +#[repr(transparent)] +pub struct TaggedLen(usize, PhantomData); + +// Clone and Copy must be manually implemented because the generic interferes +// with the derive attribute implementations. +impl Clone for TaggedLen { + fn clone(&self) -> Self { + Self(self.0, PhantomData) + } + + fn clone_from(&mut self, source: &Self) { + self.0 = source.0; + } +} + +impl Copy for TaggedLen {} + +impl TaggedLen { + pub const fn new(len: usize, on_heap: bool) -> Self { + if size_of::() == 0 { + debug_assert!(!on_heap); + Self(len, PhantomData) + } else { + debug_assert!(len < isize::MAX as usize); + Self((len << 1) | on_heap as usize, PhantomData) + } + } + + #[must_use] + pub const fn on_heap(self) -> bool { + return (size_of::() != 0) && ((self.0 & 1usize) == 1); + } + + pub const fn value(self) -> usize { + return if size_of::() == 0 { + self.0 + } else { + self.0 >> 1 + }; + } +} diff --git a/src/tests.rs b/src/tests.rs index e05f24d..8295f7d 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -1,10 +1,8 @@ -use crate::{smallvec, SmallVec}; -use alloc::borrow::ToOwned; -use alloc::boxed::Box; -use alloc::rc::Rc; -use alloc::{vec, vec::Vec}; -use core::hash::Hasher; -use core::iter::FromIterator; +use { + crate::SmallVec, + alloc::{borrow::ToOwned, boxed::Box, rc::Rc, vec::Vec}, + core::{hash::Hasher, iter::FromIterator}, +}; #[test] pub fn test_zero() { @@ -167,7 +165,7 @@ fn drain_rev() { #[test] fn drain_forget() { - let mut v: SmallVec = smallvec![0, 1, 2, 3, 4, 5, 6, 7]; + let mut v: SmallVec = SmallVec::from([0, 1, 2, 3, 4, 5, 6, 7]); std::mem::forget(v.drain(2..5)); assert_eq!(v.len(), 2); } @@ -175,21 +173,21 @@ fn drain_forget() { #[test] fn splice() { // The range starts right before the end. - let mut v: SmallVec = smallvec![0, 1, 2, 3, 4, 5, 6]; + let mut v: SmallVec = SmallVec::from([0, 1, 2, 3, 4, 5, 6]); let new = [7, 8, 9, 10]; let u: SmallVec = v.splice(6.., new).collect(); assert_eq!(v, [0, 1, 2, 3, 4, 5, 7, 8, 9, 10]); assert_eq!(u, [6]); // The range is empty. - let mut v: SmallVec = smallvec![0, 1, 2, 3, 4, 5, 6]; + let mut v: SmallVec = SmallVec::from([0, 1, 2, 3, 4, 5, 6]); let new = [7, 8, 9, 10]; let u: SmallVec = v.splice(1..1, new).collect(); assert_eq!(v, [0, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6]); assert_eq!(u, [0u8; 0]); // The range is at the beginning and nonempty. - let mut v: SmallVec = smallvec![0, 1, 2, 3, 4, 5, 6]; + let mut v: SmallVec = SmallVec::from([0, 1, 2, 3, 4, 5, 6]); let new = [7, 8, 9, 10]; let u: SmallVec = v.splice(..3, new).collect(); assert_eq!(v, [7, 8, 9, 10, 3, 4, 5, 6]); @@ -317,7 +315,7 @@ fn test_truncate() { #[test] fn test_truncate_references() { - let mut v = vec![0, 1, 2, 3, 4, 5, 6, 7]; + let mut v = Vec::from([0, 1, 2, 3, 4, 5, 6, 7]); let mut i = 8; let mut v: SmallVec<&mut u8, 8> = v.iter_mut().collect(); @@ -338,7 +336,7 @@ fn test_truncate_references() { #[test] fn test_split_off() { - let mut vec: SmallVec = smallvec![1, 2, 3, 4, 5, 6]; + let mut vec: SmallVec = SmallVec::from([1, 2, 3, 4, 5, 6]); let orig_ptr = vec.as_ptr(); let orig_capacity = vec.capacity(); @@ -400,7 +398,7 @@ fn test_invalid_grow() { #[test] #[should_panic] fn drain_overflow() { - let mut v: SmallVec = smallvec![0]; + let mut v: SmallVec = SmallVec::from([0]); v.drain(..=usize::MAX); } @@ -420,7 +418,7 @@ fn test_extend_from_slice() { #[test] fn test_extend_from_within() { - let mut v: SmallVec = smallvec![0, 1, 2, 3]; + let mut v: SmallVec = SmallVec::from([0, 1, 2, 3]); v.extend_from_within(1..3); assert_eq!( &v.iter().map(|v| *v).collect::>(), @@ -486,8 +484,7 @@ fn test_ord() { #[test] fn test_hash() { - use std::collections::hash_map::DefaultHasher; - use std::hash::Hash; + use std::{collections::hash_map::DefaultHasher, hash::Hash}; fn hash(value: impl Hash) -> u64 { let mut hasher = DefaultHasher::new(); @@ -567,17 +564,17 @@ fn test_from() { assert_eq!(&SmallVec::::from(&[1][..])[..], [1]); assert_eq!(&SmallVec::::from(&[1, 2, 3][..])[..], [1, 2, 3]); - let vec = vec![]; + let vec = Vec::new(); let small_vec: SmallVec = SmallVec::from(vec); assert_eq!(&*small_vec, &[0u8; 0]); drop(small_vec); - let vec = vec![1, 2, 3, 4, 5]; + let vec = Vec::from([1, 2, 3, 4, 5]); let small_vec: SmallVec = SmallVec::from(vec); assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); drop(small_vec); - let vec = vec![1, 2, 3, 4, 5]; + let vec = Vec::from([1, 2, 3, 4, 5]); let small_vec: SmallVec = SmallVec::from(vec); assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); drop(small_vec); @@ -589,7 +586,7 @@ fn test_from() { let array = [99; 128]; let small_vec: SmallVec = SmallVec::from(array); - assert_eq!(&*small_vec, vec![99u8; 128].as_slice()); + assert_eq!(&*small_vec, Vec::from([99u8; 128]).as_slice()); drop(small_vec); #[derive(PartialEq, Eq, Debug)] @@ -599,14 +596,14 @@ fn test_from() { assert_eq!(&*small_vec, &[NoClone(42)]); drop(small_vec); - let vec = vec![NoClone(42)]; + let vec = Vec::from([NoClone(42)]); let small_vec: SmallVec = SmallVec::from(vec); assert_eq!(&*small_vec, &[NoClone(42)]); drop(small_vec); let array = [1; 128]; let small_vec: SmallVec = SmallVec::from(array); - assert_eq!(&*small_vec, vec![1; 128].as_slice()); + assert_eq!(&*small_vec, Vec::from([1; 128]).as_slice()); drop(small_vec); let array = [99]; @@ -686,7 +683,7 @@ fn shrink_to_fit_unspill() { #[test] fn shrink_after_from_empty_vec() { - let mut v = SmallVec::::from_vec(vec![]); + let mut v = SmallVec::::from_vec(Vec::new()); v.shrink_to_fit(); assert!(!v.spilled()) } @@ -694,10 +691,10 @@ fn shrink_after_from_empty_vec() { #[test] fn test_into_vec() { let vec = SmallVec::::from_iter(0..2); - assert_eq!(vec.into_vec(), vec![0, 1]); + assert_eq!(vec.into_vec(), Vec::from([0, 1])); let vec = SmallVec::::from_iter(0..3); - assert_eq!(vec.into_vec(), vec![0, 1, 2]); + assert_eq!(vec.into_vec(), Vec::from([0, 1, 2])); } #[test] @@ -714,32 +711,32 @@ fn test_into_inner() { #[test] fn test_from_vec() { - let vec = vec![]; + let vec = Vec::new(); let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[0u8; 0]); drop(small_vec); - let vec = vec![]; + let vec = Vec::new(); let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[0u8; 0]); drop(small_vec); - let vec = vec![1]; + let vec = Vec::from([1]); let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[1]); drop(small_vec); - let vec = vec![1, 2, 3]; + let vec = Vec::from([1, 2, 3]); let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[1, 2, 3]); drop(small_vec); - let vec = vec![1, 2, 3, 4, 5]; + let vec = Vec::from([1, 2, 3, 4, 5]); let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); drop(small_vec); - let vec = vec![1, 2, 3, 4, 5]; + let vec = Vec::from([1, 2, 3, 4, 5]); let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); drop(small_vec); @@ -926,15 +923,15 @@ const fn const_new_inner() -> SmallVec { SmallVec::::new() } const fn const_new_inline_sized() -> SmallVec { - crate::smallvec_inline![1; 4] + SmallVec::from_buf([1; 4]) } const fn const_new_inline_args() -> SmallVec { - crate::smallvec_inline![1, 4] + SmallVec::from_buf([1, 4]) } #[test] fn empty_macro() { - let _v: SmallVec = smallvec![]; + let _v: SmallVec = SmallVec::new(); } #[test] @@ -966,7 +963,7 @@ fn test_clone_from() { #[test] fn test_extract_if() { - let mut a: SmallVec = smallvec![0, 1u8, 2, 3, 4, 5, 6, 7, 8, 0]; + let mut a: SmallVec = SmallVec::from([0, 1u8, 2, 3, 4, 5, 6, 7, 8, 0]); let b: SmallVec = a.extract_if(1..9, |x| *x % 3 == 0).collect(); @@ -983,7 +980,7 @@ fn test_extract_if() { /// wrong" args. #[test] fn max_dont_panic() { - let mut sv: SmallVec = smallvec![0]; + let mut sv: SmallVec = SmallVec::from([0]); let _ = sv.get(usize::MAX); sv.truncate(usize::MAX); } @@ -991,21 +988,21 @@ fn max_dont_panic() { #[test] #[should_panic] fn max_remove() { - let mut sv: SmallVec = smallvec![0]; + let mut sv: SmallVec = SmallVec::from([0]); sv.remove(usize::MAX); } #[test] #[should_panic] fn max_swap_remove() { - let mut sv: SmallVec = smallvec![0]; + let mut sv: SmallVec = SmallVec::from([0]); sv.swap_remove(usize::MAX); } #[test] #[should_panic] fn max_insert() { - let mut sv: SmallVec = smallvec![0]; + let mut sv: SmallVec = SmallVec::from([0]); sv.insert(usize::MAX, 0); } @@ -1016,6 +1013,7 @@ fn collect_from_iter() { impl Iterator for IterNoHint { type Item = I::Item; + fn next(&mut self) -> Option { self.0.next() }