diff --git a/crates/wasmtime/proptest-regressions/runtime/vm/gc/enabled/free_list.txt b/crates/wasmtime/proptest-regressions/runtime/vm/gc/enabled/free_list.txt new file mode 100644 index 000000000000..c3a26b7b9713 --- /dev/null +++ b/crates/wasmtime/proptest-regressions/runtime/vm/gc/enabled/free_list.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc b26e69fbaf46deb79652859039538e422818fd40b9afff63faa7aacbddecfd3d # shrinks to (capacity, ops) = (219544665809630458, [(10, Alloc(Layout { size: 193045289231815352, align: 8 (1 << 3) })), (10, Dealloc(Layout { size: 193045289231815352, align: 8 (1 << 3) }))]) diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs index 774f4c4c3f51..a414ba8a659b 100644 --- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs +++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs @@ -176,17 +176,21 @@ impl DrcHeap { fn dealloc(&mut self, gc_ref: VMGcRef) { let drc_ref = drc_ref(&gc_ref); - let size = self.index(drc_ref).object_size(); - let layout = FreeList::layout(size); + let size = self.index(drc_ref).object_size; + let alloc_size = FreeList::aligned_size(size); let index = gc_ref.as_heap_index().unwrap(); // Poison the freed memory so that any stale access is detectable. if cfg!(gc_zeal) { let index = usize::try_from(index.get()).unwrap(); - self.heap_slice_mut()[index..][..layout.size()].fill(POISON); + let alloc_size = usize::try_from(alloc_size).unwrap(); + self.heap_slice_mut()[index..][..alloc_size].fill(POISON); } - self.free_list.as_mut().unwrap().dealloc(index, layout); + self.free_list + .as_mut() + .unwrap() + .dealloc_fast(index, alloc_size); } /// Increment the ref count for the associated object. @@ -921,6 +925,7 @@ unsafe impl GcHeap for DrcHeap { fn alloc_raw(&mut self, header: VMGcHeader, layout: Layout) -> Result> { debug_assert!(layout.size() >= core::mem::size_of::()); debug_assert!(layout.align() >= core::mem::align_of::()); + debug_assert!(FreeList::can_align_to(layout.align())); debug_assert_eq!(header.reserved_u26(), 0); // We must have trace info for every GC type that we allocate in this @@ -934,8 +939,9 @@ unsafe impl GcHeap for DrcHeap { } let object_size = u32::try_from(layout.size()).unwrap(); + let alloc_size = FreeList::aligned_size(object_size); - let gc_ref = match self.free_list.as_mut().unwrap().alloc(layout)? { + let gc_ref = match self.free_list.as_mut().unwrap().alloc_fast(alloc_size) { None => return Ok(Err(u64::try_from(layout.size()).unwrap())), Some(index) => VMGcRef::from_heap_index(index).unwrap(), }; diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs index e638e5ea90a6..d21d9026e4bb 100644 --- a/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs +++ b/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs @@ -1,8 +1,9 @@ use crate::prelude::*; use alloc::collections::BTreeMap; -use core::{alloc::Layout, num::NonZeroU32, ops::Bound}; +use core::{alloc::Layout, num::NonZeroU32}; -/// A very simple first-fit free list for use by our garbage collectors. +/// A free list for use by our garbage collectors, using a sorted Vec of +/// (index, length) pairs for cache-friendly operations. pub(crate) struct FreeList { /// The total capacity of the contiguous range of memory we are managing. /// @@ -25,6 +26,11 @@ pub(crate) struct FreeList { /// Our free blocks, as a map from index to length of the free block at that /// index. free_block_index_to_len: BTreeMap, + /// Bump allocator: current position in the active free block. + /// Allocations bump this forward. When exhausted, refilled from blocks. + bump_ptr: u32, + /// End of the current bump allocation region. + bump_end: u32, } /// Our minimum and maximum supported alignment. Every allocation is aligned to @@ -40,6 +46,13 @@ impl FreeList { Layout::from_size_align(size, ALIGN_USIZE).unwrap() } + /// Compute the aligned allocation size for a given byte size. Returns the + /// size rounded up to this free list's alignment, as a u32. + #[inline] + pub fn aligned_size(size: u32) -> u32 { + (size + ALIGN_U32 - 1) & !(ALIGN_U32 - 1) + } + /// Get the current total capacity this free list manages. pub fn current_capacity(&self) -> usize { self.capacity @@ -53,6 +66,8 @@ impl FreeList { let mut free_list = FreeList { capacity, free_block_index_to_len: BTreeMap::new(), + bump_ptr: 0, + bump_end: 0, }; let end = u32::try_from(free_list.capacity).unwrap_or_else(|_| { @@ -67,13 +82,11 @@ impl FreeList { let len = round_u32_down_to_pow2(end.saturating_sub(start), ALIGN_U32); - let entire_range = if len >= ALIGN_U32 { - Some((start, len)) - } else { - None - }; - - free_list.free_block_index_to_len.extend(entire_range); + if len >= ALIGN_U32 { + // Initialize bump allocator with the entire range. + free_list.bump_ptr = start; + free_list.bump_end = start + len; + } free_list } @@ -146,11 +159,23 @@ impl FreeList { round_usize_down_to_pow2(cap.saturating_sub(ALIGN_USIZE), ALIGN_USIZE) } + /// Total number of free blocks (including bump region if non-empty). + #[cfg(test)] + fn num_free_blocks(&self) -> usize { + self.free_block_index_to_len.len() + if self.bump_end > self.bump_ptr { 1 } else { 0 } + } + + /// Can this free list align allocations to the given value? + pub fn can_align_to(align: usize) -> bool { + debug_assert!(align.is_power_of_two()); + align <= ALIGN_USIZE + } + /// Check the given layout for compatibility with this free list and return /// the actual block size we will use for this layout. fn check_layout(&self, layout: Layout) -> Result { ensure!( - layout.align() <= ALIGN_USIZE, + Self::can_align_to(layout.align()), "requested allocation's alignment of {} is greater than max supported \ alignment of {ALIGN_USIZE}", layout.align(), @@ -175,109 +200,146 @@ impl FreeList { }) } - /// Find the first free block that can hold an allocation of the given size - /// and remove it from the free list. - fn first_fit(&mut self, alloc_size: u32) -> Option<(u32, u32)> { - debug_assert_eq!(alloc_size % ALIGN_U32, 0); - - let (&block_index, &block_len) = self - .free_block_index_to_len - .iter() - .find(|(_idx, len)| **len >= alloc_size)?; - - debug_assert_eq!(block_index % ALIGN_U32, 0); - debug_assert_eq!(block_len % ALIGN_U32, 0); - - let entry = self.free_block_index_to_len.remove(&block_index); - debug_assert!(entry.is_some()); + #[cfg(test)] + pub fn alloc(&mut self, layout: Layout) -> Result> { + log::trace!("FreeList::alloc({layout:?})"); + let alloc_size = self.check_layout(layout)?; + Ok(self.alloc_impl(alloc_size)) + } - Some((block_index, block_len)) + /// Fast-path allocation with a pre-computed aligned size, as returned from + /// `Self::aligned_size`. + #[inline] + pub fn alloc_fast(&mut self, alloc_size: u32) -> Option { + debug_assert_eq!(alloc_size % ALIGN_U32, 0); + debug_assert!(alloc_size > 0); + self.alloc_impl(alloc_size) } - /// If the given allocated block is large enough such that we can split it - /// and still have enough space left for future allocations, then split it. - /// - /// Returns the new length of the allocated block. - fn maybe_split(&mut self, alloc_size: u32, block_index: u32, block_len: u32) -> u32 { + #[inline] + fn alloc_impl(&mut self, alloc_size: u32) -> Option { + debug_assert_eq!( + Self::layout(usize::try_from(alloc_size).unwrap()).size(), + usize::try_from(alloc_size).unwrap() + ); debug_assert_eq!(alloc_size % ALIGN_U32, 0); - debug_assert_eq!(block_index % ALIGN_U32, 0); - debug_assert_eq!(block_len % ALIGN_U32, 0); - if block_len - alloc_size < ALIGN_U32 { - // The block is not large enough to split. - return block_len; - } + // Fast path: bump allocate from the current region. + let new_ptr = self.bump_ptr + alloc_size; + if new_ptr <= self.bump_end { + let result = self.bump_ptr; + self.bump_ptr = new_ptr; + debug_assert_ne!(result, 0); + debug_assert_eq!(result % ALIGN_U32, 0); - // The block is large enough to split. Split the block at exactly the - // requested allocation size and put the tail back in the free list. - let new_block_len = alloc_size; - let split_start = block_index + alloc_size; - let split_len = block_len - alloc_size; + self.check_integrity(); - debug_assert_eq!(new_block_len % ALIGN_U32, 0); - debug_assert_eq!(split_start % ALIGN_U32, 0); - debug_assert_eq!(split_len % ALIGN_U32, 0); + log::trace!("FreeList::alloc -> {result:#x}"); + return Some(unsafe { NonZeroU32::new_unchecked(result) }); + } - self.free_block_index_to_len.insert(split_start, split_len); + // After we've mutated the free list, double check its integrity. + self.check_integrity(); - new_block_len + // Slow path: find a block in the blocks list, then set it as bump region. + self.alloc_slow(alloc_size) } - /// Allocate space for an object of the given layout. - /// - /// Returns: - /// - /// * `Ok(Some(_))`: Allocation succeeded. - /// - /// * `Ok(None)`: Can't currently fulfill the allocation request, but might - /// be able to if some stuff was reallocated. - /// - /// * `Err(_)`: - pub fn alloc(&mut self, layout: Layout) -> Result> { - log::trace!("FreeList::alloc({layout:?})"); - let alloc_size = self.check_layout(layout)?; - debug_assert_eq!(alloc_size % ALIGN_U32, 0); + #[cold] + fn alloc_slow(&mut self, alloc_size: u32) -> Option { + // Put the remaining bump region back into blocks if non-empty. + let remaining_ptr = self.bump_ptr; + let remaining = self.bump_end - self.bump_ptr; + self.bump_ptr = 0; + self.bump_end = 0; + if remaining >= ALIGN_U32 { + self.insert_free_block(remaining_ptr, remaining); + } + + // Find a block big enough. + let (&block_index, &block_len) = self + .free_block_index_to_len + .iter() + .find(|(_, len)| **len >= alloc_size)?; + self.free_block_index_to_len.remove(&block_index); - let (block_index, block_len) = match self.first_fit(alloc_size) { - None => return Ok(None), - Some(tup) => tup, - }; - debug_assert_ne!(block_index, 0); debug_assert_eq!(block_index % ALIGN_U32, 0); - debug_assert!(block_len >= alloc_size); debug_assert_eq!(block_len % ALIGN_U32, 0); - let block_len = self.maybe_split(alloc_size, block_index, block_len); - debug_assert!(block_len >= alloc_size); - debug_assert_eq!(block_len % ALIGN_U32, 0); + // Set this block as the new bump region and allocate from it. + self.bump_ptr = block_index + alloc_size; + self.bump_end = block_index + block_len; - // After we've mutated the free list, double check its integrity. - #[cfg(debug_assertions)] + debug_assert_ne!(block_index, 0); self.check_integrity(); - log::trace!("FreeList::alloc({layout:?}) -> {block_index:#x}"); - Ok(Some(unsafe { NonZeroU32::new_unchecked(block_index) })) + Some(unsafe { NonZeroU32::new_unchecked(block_index) }) } /// Deallocate an object with the given layout. pub fn dealloc(&mut self, index: NonZeroU32, layout: Layout) { log::trace!("FreeList::dealloc({index:#x}, {layout:?})"); + let alloc_size = self.check_layout(layout).unwrap(); + self.dealloc_impl(index.get(), alloc_size); + } - let index = index.get(); - debug_assert_eq!(index % ALIGN_U32, 0); + /// Fast-path deallocation with a pre-computed aligned size. + #[inline] + pub fn dealloc_fast(&mut self, index: NonZeroU32, alloc_size: u32) { + debug_assert_eq!(alloc_size % ALIGN_U32, 0); + debug_assert_eq!(index.get() % ALIGN_U32, 0); + self.dealloc_impl(index.get(), alloc_size); + } - let alloc_size = self.check_layout(layout).unwrap(); + #[inline] + fn dealloc_impl(&mut self, index: u32, alloc_size: u32) { + debug_assert_eq!( + Self::layout(usize::try_from(alloc_size).unwrap()).size(), + usize::try_from(alloc_size).unwrap() + ); + debug_assert_eq!(index % ALIGN_U32, 0); debug_assert_eq!(alloc_size % ALIGN_U32, 0); + // Check if the freed block is directly below the bump region. + if index + alloc_size == self.bump_ptr { + self.bump_ptr = index; + + // Also check if the last block in the list is now contiguous with + // the extended bump region. + if let Some((&bi, &bl)) = self.free_block_index_to_len.last_key_value() { + if bi + bl == self.bump_ptr { + self.bump_ptr = bi; + self.free_block_index_to_len.pop_last(); + } + } + + self.check_integrity(); + return; + } + + // Check if the freed block is directly above the bump region. + if self.bump_end == index { + self.bump_end = index + alloc_size; + + // Also check if the first block above the bump region is now + // contiguous. + if let Some(block_len) = self.free_block_index_to_len.remove(&self.bump_end) { + self.bump_end += block_len; + } + + self.check_integrity(); + return; + } + let prev_block = self .free_block_index_to_len - .range((Bound::Unbounded, Bound::Excluded(index))) + .range(..index) .next_back() .map(|(idx, len)| (*idx, *len)); let next_block = self .free_block_index_to_len - .range((Bound::Excluded(index), Bound::Unbounded)) + .range(index + 1..) .next() .map(|(idx, len)| (*idx, *len)); @@ -291,9 +353,9 @@ impl FreeList { && blocks_are_contiguous(index, alloc_size, next_index) => { log::trace!( - "merging blocks {prev_index:#x}..{prev_len:#x}, {index:#x}..{index_end:#x}, {next_index:#x}..{next_end:#x}", - prev_len = prev_index + prev_len, - index_end = index + u32::try_from(layout.size()).unwrap(), + "merging blocks {prev_index:#x}..{prev_end:#x}, {index:#x}..{index_end:#x}, {next_index:#x}..{next_end:#x}", + prev_end = prev_index + prev_len, + index_end = index + alloc_size, next_end = next_index + next_len, ); self.free_block_index_to_len.remove(&next_index); @@ -307,9 +369,9 @@ impl FreeList { if blocks_are_contiguous(prev_index, prev_len, index) => { log::trace!( - "merging blocks {prev_index:#x}..{prev_len:#x}, {index:#x}..{index_end:#x}", - prev_len = prev_index + prev_len, - index_end = index + u32::try_from(layout.size()).unwrap(), + "merging blocks {prev_index:#x}..{prev_end:#x}, {index:#x}..{index_end:#x}", + prev_end = prev_index + prev_len, + index_end = index + alloc_size, ); let merged_block_len = index + alloc_size - prev_index; debug_assert_eq!(merged_block_len % ALIGN_U32, 0); @@ -322,7 +384,7 @@ impl FreeList { { log::trace!( "merging blocks {index:#x}..{index_end:#x}, {next_index:#x}..{next_end:#x}", - index_end = index + u32::try_from(layout.size()).unwrap(), + index_end = index + alloc_size, next_end = next_index + next_len, ); self.free_block_index_to_len.remove(&next_index); @@ -339,15 +401,39 @@ impl FreeList { } } + // After merge, check if the last block is now contiguous with the bump + // region and absorb it. + if let Some((&block_index, &block_len)) = self.free_block_index_to_len.last_key_value() { + if block_index + block_len == self.bump_ptr { + self.bump_ptr = block_index; + self.free_block_index_to_len.pop_last(); + } + } + // After we've added to/mutated the free list, double check its // integrity. - #[cfg(debug_assertions)] self.check_integrity(); } /// Iterate over all free blocks as `(index, len)` pairs. pub fn iter_free_blocks(&self) -> impl Iterator + '_ { - self.free_block_index_to_len.iter().map(|(&i, &l)| (i, l)) + let bump = if self.bump_end > self.bump_ptr { + Some((self.bump_ptr, self.bump_end - self.bump_ptr)) + } else { + None + }; + self.free_block_index_to_len + .iter() + .map(|(idx, len)| (*idx, *len)) + .chain(bump) + } + + /// Insert a free block into the sorted blocks list with merging. + fn insert_free_block(&mut self, index: u32, size: u32) { + debug_assert_eq!(index % ALIGN_U32, 0); + debug_assert_eq!(size % ALIGN_U32, 0); + // Reuse dealloc_impl which handles insertion and merging. + self.dealloc_impl(index, size); } /// Assert that the free list is valid: @@ -359,8 +445,11 @@ impl FreeList { /// 3. All blocks are aligned to `ALIGN` /// /// 4. All block sizes are a multiple of `ALIGN` - #[cfg(debug_assertions)] fn check_integrity(&self) { + if !cfg!(gc_zeal) { + return; + } + let mut prev_end = None; for (&index, &len) in self.free_block_index_to_len.iter() { // (1) @@ -385,6 +474,26 @@ impl FreeList { prev_end = Some(end); } + + // Check bump region validity. + assert!(self.bump_ptr <= self.bump_end); + if self.bump_ptr < self.bump_end { + assert_eq!(self.bump_ptr % ALIGN_U32, 0); + assert_eq!(self.bump_end % ALIGN_U32, 0); + assert!(usize::try_from(self.bump_end).unwrap() <= self.capacity); + // Bump region should not overlap with any block. + for (&index, &len) in self.free_block_index_to_len.iter() { + let block_end = index + len; + assert!( + self.bump_end <= index || self.bump_ptr >= block_end, + "bump region [{}, {}) overlaps with block [{}, {})", + self.bump_ptr, + self.bump_end, + index, + block_end + ); + } + } } } @@ -431,12 +540,15 @@ mod tests { use std::num::NonZeroUsize; fn free_list_block_len_and_size(free_list: &FreeList) -> (usize, Option) { - let len = free_list.free_block_index_to_len.len(); - let size = free_list - .free_block_index_to_len - .values() - .next() - .map(|s| usize::try_from(*s).unwrap()); + let len = free_list.num_free_blocks(); + let size = if free_list.bump_end > free_list.bump_ptr { + Some(usize::try_from(free_list.bump_end - free_list.bump_ptr).unwrap()) + } else { + free_list + .free_block_index_to_len + .first_key_value() + .map(|(_, &s)| usize::try_from(s).unwrap()) + }; (len, size) } @@ -598,7 +710,7 @@ mod tests { // `ALIGN_U32`. let mut free_list = FreeList::new(ALIGN_USIZE + ALIGN_USIZE * 2); - assert_eq!(free_list.free_block_index_to_len.len(), 1); + assert_eq!(free_list.num_free_blocks(), 1); assert_eq!(free_list.max_size(), ALIGN_USIZE * 2); // Allocate a block such that the remainder is not worth splitting. @@ -608,7 +720,7 @@ mod tests { .expect("have free space available for allocation"); // Should not have split the block. - assert_eq!(free_list.free_block_index_to_len.len(), 0); + assert_eq!(free_list.num_free_blocks(), 0); } #[test] @@ -617,7 +729,7 @@ mod tests { // `ALIGN_U32`. let mut free_list = FreeList::new(ALIGN_USIZE + ALIGN_USIZE * 3); - assert_eq!(free_list.free_block_index_to_len.len(), 1); + assert_eq!(free_list.num_free_blocks(), 1); assert_eq!(free_list.max_size(), ALIGN_USIZE * 3); // Allocate a block such that the remainder is not worth splitting. @@ -627,7 +739,7 @@ mod tests { .expect("have free space available for allocation"); // Should have split the block. - assert_eq!(free_list.free_block_index_to_len.len(), 1); + assert_eq!(free_list.num_free_blocks(), 1); } #[test] @@ -636,7 +748,7 @@ mod tests { let mut free_list = FreeList::new(ALIGN_USIZE + ALIGN_USIZE * 100); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "initially one big free block" ); @@ -646,7 +758,7 @@ mod tests { .expect("allocation within 'static' free list limits") .expect("have free space available for allocation"); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "should have split the block to allocate `a`" ); @@ -656,21 +768,21 @@ mod tests { .expect("allocation within 'static' free list limits") .expect("have free space available for allocation"); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "should have split the block to allocate `b`" ); free_list.dealloc(a, layout); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 2, "should have two non-contiguous free blocks after deallocating `a`" ); free_list.dealloc(b, layout); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "should have merged `a` and `b` blocks with the rest to form a \ single, contiguous free block after deallocating `b`" @@ -683,7 +795,7 @@ mod tests { let mut free_list = FreeList::new(ALIGN_USIZE + ALIGN_USIZE * 100); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "initially one big free block" ); @@ -701,21 +813,21 @@ mod tests { .expect("allocation within 'static' free list limits") .expect("have free space available for allocation"); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "should have split the block to allocate `a`, `b`, and `c`" ); free_list.dealloc(a, layout); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 2, "should have two non-contiguous free blocks after deallocating `a`" ); free_list.dealloc(b, layout); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 2, "should have merged `a` and `b` blocks, but not merged with the \ rest of the free space" @@ -730,7 +842,7 @@ mod tests { let mut free_list = FreeList::new(ALIGN_USIZE + ALIGN_USIZE * 100); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "initially one big free block" ); @@ -748,21 +860,21 @@ mod tests { .expect("allocation within 'static' free list limits") .expect("have free space available for allocation"); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "should have split the block to allocate `a`, `b`, and `c`" ); free_list.dealloc(a, layout); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 2, "should have two non-contiguous free blocks after deallocating `a`" ); free_list.dealloc(c, layout); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 2, "should have merged `c` block with rest of the free space, but not \ with `a` block" @@ -777,7 +889,7 @@ mod tests { let mut free_list = FreeList::new(ALIGN_USIZE + ALIGN_USIZE * 100); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "initially one big free block" ); @@ -799,21 +911,21 @@ mod tests { .expect("allocation within 'static' free list limits") .expect("have free space available for allocation"); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "should have split the block to allocate `a`, `b`, `c`, and `d`" ); free_list.dealloc(a, layout); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 2, "should have two non-contiguous free blocks after deallocating `a`" ); free_list.dealloc(c, layout); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 3, "should not have merged `c` block `a` block or rest of the free \ space" @@ -921,14 +1033,14 @@ mod tests { free_list.dealloc(a, layout); free_list.dealloc(b, layout); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "`dealloc` should merge blocks from different `add_capacity` calls together" ); free_list.add_capacity(ALIGN_USIZE); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "`add_capacity` should eagerly merge new capacity into the last block \ in the free list, when possible"