From fc82c639a60094ddf091ee0fb6f7929957de5b6f Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Mon, 6 Apr 2026 07:41:29 -0700 Subject: [PATCH 1/2] Use bump allocation in DRC free list and other improvements Also add fast-path entry points that take a `u32` size directly that has already been rounded to the free list's alignment. Altogether, this shaves off ~309B instructions retired (48%) from the benchmark in https://github.com/bytecodealliance/wasmtime/issues/11141 --- .../runtime/vm/gc/enabled/free_list.txt | 7 + .../wasmtime/src/runtime/vm/gc/enabled/drc.rs | 16 +- .../src/runtime/vm/gc/enabled/free_list.rs | 348 ++++++++++++------ 3 files changed, 251 insertions(+), 120 deletions(-) create mode 100644 crates/wasmtime/proptest-regressions/runtime/vm/gc/enabled/free_list.txt diff --git a/crates/wasmtime/proptest-regressions/runtime/vm/gc/enabled/free_list.txt b/crates/wasmtime/proptest-regressions/runtime/vm/gc/enabled/free_list.txt new file mode 100644 index 000000000000..c3a26b7b9713 --- /dev/null +++ b/crates/wasmtime/proptest-regressions/runtime/vm/gc/enabled/free_list.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc b26e69fbaf46deb79652859039538e422818fd40b9afff63faa7aacbddecfd3d # shrinks to (capacity, ops) = (219544665809630458, [(10, Alloc(Layout { size: 193045289231815352, align: 8 (1 << 3) })), (10, Dealloc(Layout { size: 193045289231815352, align: 8 (1 << 3) }))]) diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs index 774f4c4c3f51..a414ba8a659b 100644 --- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs +++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs @@ -176,17 +176,21 @@ impl DrcHeap { fn dealloc(&mut self, gc_ref: VMGcRef) { let drc_ref = drc_ref(&gc_ref); - let size = self.index(drc_ref).object_size(); - let layout = FreeList::layout(size); + let size = self.index(drc_ref).object_size; + let alloc_size = FreeList::aligned_size(size); let index = gc_ref.as_heap_index().unwrap(); // Poison the freed memory so that any stale access is detectable. if cfg!(gc_zeal) { let index = usize::try_from(index.get()).unwrap(); - self.heap_slice_mut()[index..][..layout.size()].fill(POISON); + let alloc_size = usize::try_from(alloc_size).unwrap(); + self.heap_slice_mut()[index..][..alloc_size].fill(POISON); } - self.free_list.as_mut().unwrap().dealloc(index, layout); + self.free_list + .as_mut() + .unwrap() + .dealloc_fast(index, alloc_size); } /// Increment the ref count for the associated object. @@ -921,6 +925,7 @@ unsafe impl GcHeap for DrcHeap { fn alloc_raw(&mut self, header: VMGcHeader, layout: Layout) -> Result> { debug_assert!(layout.size() >= core::mem::size_of::()); debug_assert!(layout.align() >= core::mem::align_of::()); + debug_assert!(FreeList::can_align_to(layout.align())); debug_assert_eq!(header.reserved_u26(), 0); // We must have trace info for every GC type that we allocate in this @@ -934,8 +939,9 @@ unsafe impl GcHeap for DrcHeap { } let object_size = u32::try_from(layout.size()).unwrap(); + let alloc_size = FreeList::aligned_size(object_size); - let gc_ref = match self.free_list.as_mut().unwrap().alloc(layout)? { + let gc_ref = match self.free_list.as_mut().unwrap().alloc_fast(alloc_size) { None => return Ok(Err(u64::try_from(layout.size()).unwrap())), Some(index) => VMGcRef::from_heap_index(index).unwrap(), }; diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs index e638e5ea90a6..5a0bd4d90c0b 100644 --- a/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs +++ b/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs @@ -1,8 +1,9 @@ use crate::prelude::*; use alloc::collections::BTreeMap; -use core::{alloc::Layout, num::NonZeroU32, ops::Bound}; +use core::{alloc::Layout, num::NonZeroU32}; -/// A very simple first-fit free list for use by our garbage collectors. +/// A free list for use by our garbage collectors, using a sorted Vec of +/// (index, length) pairs for cache-friendly operations. pub(crate) struct FreeList { /// The total capacity of the contiguous range of memory we are managing. /// @@ -25,6 +26,11 @@ pub(crate) struct FreeList { /// Our free blocks, as a map from index to length of the free block at that /// index. free_block_index_to_len: BTreeMap, + /// Bump allocator: current position in the active free block. + /// Allocations bump this forward. When exhausted, refilled from blocks. + bump_ptr: u32, + /// End of the current bump allocation region. + bump_end: u32, } /// Our minimum and maximum supported alignment. Every allocation is aligned to @@ -40,6 +46,13 @@ impl FreeList { Layout::from_size_align(size, ALIGN_USIZE).unwrap() } + /// Compute the aligned allocation size for a given byte size. Returns the + /// size rounded up to this free list's alignment, as a u32. + #[inline] + pub fn aligned_size(size: u32) -> u32 { + (size + ALIGN_U32 - 1) & !(ALIGN_U32 - 1) + } + /// Get the current total capacity this free list manages. pub fn current_capacity(&self) -> usize { self.capacity @@ -53,6 +66,8 @@ impl FreeList { let mut free_list = FreeList { capacity, free_block_index_to_len: BTreeMap::new(), + bump_ptr: 0, + bump_end: 0, }; let end = u32::try_from(free_list.capacity).unwrap_or_else(|_| { @@ -67,13 +82,11 @@ impl FreeList { let len = round_u32_down_to_pow2(end.saturating_sub(start), ALIGN_U32); - let entire_range = if len >= ALIGN_U32 { - Some((start, len)) - } else { - None - }; - - free_list.free_block_index_to_len.extend(entire_range); + if len >= ALIGN_U32 { + // Initialize bump allocator with the entire range. + free_list.bump_ptr = start; + free_list.bump_end = start + len; + } free_list } @@ -146,11 +159,23 @@ impl FreeList { round_usize_down_to_pow2(cap.saturating_sub(ALIGN_USIZE), ALIGN_USIZE) } + /// Total number of free blocks (including bump region if non-empty). + #[cfg(test)] + fn num_free_blocks(&self) -> usize { + self.free_block_index_to_len.len() + if self.bump_end > self.bump_ptr { 1 } else { 0 } + } + + /// Can this free list align allocations to the given value? + pub fn can_align_to(align: usize) -> bool { + debug_assert!(align.is_power_of_two()); + align <= ALIGN_USIZE + } + /// Check the given layout for compatibility with this free list and return /// the actual block size we will use for this layout. fn check_layout(&self, layout: Layout) -> Result { ensure!( - layout.align() <= ALIGN_USIZE, + Self::can_align_to(layout.align()), "requested allocation's alignment of {} is greater than max supported \ alignment of {ALIGN_USIZE}", layout.align(), @@ -175,109 +200,154 @@ impl FreeList { }) } - /// Find the first free block that can hold an allocation of the given size - /// and remove it from the free list. - fn first_fit(&mut self, alloc_size: u32) -> Option<(u32, u32)> { - debug_assert_eq!(alloc_size % ALIGN_U32, 0); - - let (&block_index, &block_len) = self - .free_block_index_to_len - .iter() - .find(|(_idx, len)| **len >= alloc_size)?; - - debug_assert_eq!(block_index % ALIGN_U32, 0); - debug_assert_eq!(block_len % ALIGN_U32, 0); - - let entry = self.free_block_index_to_len.remove(&block_index); - debug_assert!(entry.is_some()); + #[cfg(test)] + pub fn alloc(&mut self, layout: Layout) -> Result> { + log::trace!("FreeList::alloc({layout:?})"); + let alloc_size = self.check_layout(layout)?; + Ok(self.alloc_impl(alloc_size)) + } - Some((block_index, block_len)) + /// Fast-path allocation with a pre-computed aligned size, as returned from + /// `Self::aligned_size`. + #[inline] + pub fn alloc_fast(&mut self, alloc_size: u32) -> Option { + debug_assert_eq!(alloc_size % ALIGN_U32, 0); + debug_assert!(alloc_size > 0); + self.alloc_impl(alloc_size) } - /// If the given allocated block is large enough such that we can split it - /// and still have enough space left for future allocations, then split it. - /// - /// Returns the new length of the allocated block. - fn maybe_split(&mut self, alloc_size: u32, block_index: u32, block_len: u32) -> u32 { + #[inline] + fn alloc_impl(&mut self, alloc_size: u32) -> Option { + debug_assert_eq!( + Self::layout(usize::try_from(alloc_size).unwrap()).size(), + usize::try_from(alloc_size).unwrap() + ); debug_assert_eq!(alloc_size % ALIGN_U32, 0); - debug_assert_eq!(block_index % ALIGN_U32, 0); - debug_assert_eq!(block_len % ALIGN_U32, 0); - if block_len - alloc_size < ALIGN_U32 { - // The block is not large enough to split. - return block_len; - } + // Fast path: bump allocate from the current region. + let new_ptr = self.bump_ptr + alloc_size; + if new_ptr <= self.bump_end { + let result = self.bump_ptr; + self.bump_ptr = new_ptr; + debug_assert_ne!(result, 0); + debug_assert_eq!(result % ALIGN_U32, 0); - // The block is large enough to split. Split the block at exactly the - // requested allocation size and put the tail back in the free list. - let new_block_len = alloc_size; - let split_start = block_index + alloc_size; - let split_len = block_len - alloc_size; + #[cfg(debug_assertions)] + self.check_integrity(); - debug_assert_eq!(new_block_len % ALIGN_U32, 0); - debug_assert_eq!(split_start % ALIGN_U32, 0); - debug_assert_eq!(split_len % ALIGN_U32, 0); + log::trace!("FreeList::alloc -> {result:#x}"); + return Some(unsafe { NonZeroU32::new_unchecked(result) }); + } - self.free_block_index_to_len.insert(split_start, split_len); + // After we've mutated the free list, double check its integrity. + #[cfg(debug_assertions)] + self.check_integrity(); - new_block_len + // Slow path: find a block in the blocks list, then set it as bump region. + self.alloc_slow(alloc_size) } - /// Allocate space for an object of the given layout. - /// - /// Returns: - /// - /// * `Ok(Some(_))`: Allocation succeeded. - /// - /// * `Ok(None)`: Can't currently fulfill the allocation request, but might - /// be able to if some stuff was reallocated. - /// - /// * `Err(_)`: - pub fn alloc(&mut self, layout: Layout) -> Result> { - log::trace!("FreeList::alloc({layout:?})"); - let alloc_size = self.check_layout(layout)?; - debug_assert_eq!(alloc_size % ALIGN_U32, 0); + #[inline(never)] + #[cold] + fn alloc_slow(&mut self, alloc_size: u32) -> Option { + // Put the remaining bump region back into blocks if non-empty. + let remaining_ptr = self.bump_ptr; + let remaining = self.bump_end - self.bump_ptr; + self.bump_ptr = 0; + self.bump_end = 0; + if remaining >= ALIGN_U32 { + self.insert_free_block(remaining_ptr, remaining); + } + + // Find a block big enough. + let (&block_index, &block_len) = self + .free_block_index_to_len + .iter() + .find(|(_, len)| **len >= alloc_size)?; + self.free_block_index_to_len.remove(&block_index); - let (block_index, block_len) = match self.first_fit(alloc_size) { - None => return Ok(None), - Some(tup) => tup, - }; - debug_assert_ne!(block_index, 0); debug_assert_eq!(block_index % ALIGN_U32, 0); - debug_assert!(block_len >= alloc_size); debug_assert_eq!(block_len % ALIGN_U32, 0); - let block_len = self.maybe_split(alloc_size, block_index, block_len); - debug_assert!(block_len >= alloc_size); - debug_assert_eq!(block_len % ALIGN_U32, 0); + // Set this block as the new bump region and allocate from it. + self.bump_ptr = block_index + alloc_size; + self.bump_end = block_index + block_len; - // After we've mutated the free list, double check its integrity. + debug_assert_ne!(block_index, 0); #[cfg(debug_assertions)] self.check_integrity(); - log::trace!("FreeList::alloc({layout:?}) -> {block_index:#x}"); - Ok(Some(unsafe { NonZeroU32::new_unchecked(block_index) })) + Some(unsafe { NonZeroU32::new_unchecked(block_index) }) } /// Deallocate an object with the given layout. pub fn dealloc(&mut self, index: NonZeroU32, layout: Layout) { log::trace!("FreeList::dealloc({index:#x}, {layout:?})"); + let alloc_size = self.check_layout(layout).unwrap(); + self.dealloc_impl(index.get(), alloc_size); + } - let index = index.get(); - debug_assert_eq!(index % ALIGN_U32, 0); + /// Fast-path deallocation with a pre-computed aligned size. + #[inline] + pub fn dealloc_fast(&mut self, index: NonZeroU32, alloc_size: u32) { + debug_assert_eq!(alloc_size % ALIGN_U32, 0); + debug_assert_eq!(index.get() % ALIGN_U32, 0); + self.dealloc_impl(index.get(), alloc_size); + } - let alloc_size = self.check_layout(layout).unwrap(); + #[inline] + fn dealloc_impl(&mut self, index: u32, alloc_size: u32) { + debug_assert_eq!( + Self::layout(usize::try_from(alloc_size).unwrap()).size(), + usize::try_from(alloc_size).unwrap() + ); + debug_assert_eq!(index % ALIGN_U32, 0); debug_assert_eq!(alloc_size % ALIGN_U32, 0); + // Check if the freed block is directly below the bump region. + if index + alloc_size == self.bump_ptr { + self.bump_ptr = index; + + // Also check if the last block in the list is now contiguous with + // the extended bump region. + if let Some((&bi, &bl)) = self.free_block_index_to_len.last_key_value() { + if bi + bl == self.bump_ptr { + self.bump_ptr = bi; + self.free_block_index_to_len.pop_last(); + } + } + + #[cfg(debug_assertions)] + self.check_integrity(); + + return; + } + + // Check if the freed block is directly above the bump region. + if self.bump_end == index { + self.bump_end = index + alloc_size; + + // Also check if the first block above the bump region is now + // contiguous. + if let Some(block_len) = self.free_block_index_to_len.remove(&self.bump_end) { + self.bump_end += block_len; + } + + #[cfg(debug_assertions)] + self.check_integrity(); + + return; + } + let prev_block = self .free_block_index_to_len - .range((Bound::Unbounded, Bound::Excluded(index))) + .range(..index) .next_back() .map(|(idx, len)| (*idx, *len)); let next_block = self .free_block_index_to_len - .range((Bound::Excluded(index), Bound::Unbounded)) + .range(index + 1..) .next() .map(|(idx, len)| (*idx, *len)); @@ -291,9 +361,9 @@ impl FreeList { && blocks_are_contiguous(index, alloc_size, next_index) => { log::trace!( - "merging blocks {prev_index:#x}..{prev_len:#x}, {index:#x}..{index_end:#x}, {next_index:#x}..{next_end:#x}", - prev_len = prev_index + prev_len, - index_end = index + u32::try_from(layout.size()).unwrap(), + "merging blocks {prev_index:#x}..{prev_end:#x}, {index:#x}..{index_end:#x}, {next_index:#x}..{next_end:#x}", + prev_end = prev_index + prev_len, + index_end = index + alloc_size, next_end = next_index + next_len, ); self.free_block_index_to_len.remove(&next_index); @@ -307,9 +377,9 @@ impl FreeList { if blocks_are_contiguous(prev_index, prev_len, index) => { log::trace!( - "merging blocks {prev_index:#x}..{prev_len:#x}, {index:#x}..{index_end:#x}", - prev_len = prev_index + prev_len, - index_end = index + u32::try_from(layout.size()).unwrap(), + "merging blocks {prev_index:#x}..{prev_end:#x}, {index:#x}..{index_end:#x}", + prev_end = prev_index + prev_len, + index_end = index + alloc_size, ); let merged_block_len = index + alloc_size - prev_index; debug_assert_eq!(merged_block_len % ALIGN_U32, 0); @@ -322,7 +392,7 @@ impl FreeList { { log::trace!( "merging blocks {index:#x}..{index_end:#x}, {next_index:#x}..{next_end:#x}", - index_end = index + u32::try_from(layout.size()).unwrap(), + index_end = index + alloc_size, next_end = next_index + next_len, ); self.free_block_index_to_len.remove(&next_index); @@ -339,6 +409,15 @@ impl FreeList { } } + // After merge, check if the last block is now contiguous with the bump + // region and absorb it. + if let Some((&block_index, &block_len)) = self.free_block_index_to_len.last_key_value() { + if block_index + block_len == self.bump_ptr { + self.bump_ptr = block_index; + self.free_block_index_to_len.pop_last(); + } + } + // After we've added to/mutated the free list, double check its // integrity. #[cfg(debug_assertions)] @@ -347,7 +426,23 @@ impl FreeList { /// Iterate over all free blocks as `(index, len)` pairs. pub fn iter_free_blocks(&self) -> impl Iterator + '_ { - self.free_block_index_to_len.iter().map(|(&i, &l)| (i, l)) + let bump = if self.bump_end > self.bump_ptr { + Some((self.bump_ptr, self.bump_end - self.bump_ptr)) + } else { + None + }; + self.free_block_index_to_len + .iter() + .map(|(idx, len)| (*idx, *len)) + .chain(bump) + } + + /// Insert a free block into the sorted blocks list with merging. + fn insert_free_block(&mut self, index: u32, size: u32) { + debug_assert_eq!(index % ALIGN_U32, 0); + debug_assert_eq!(size % ALIGN_U32, 0); + // Reuse dealloc_impl which handles insertion and merging. + self.dealloc_impl(index, size); } /// Assert that the free list is valid: @@ -385,6 +480,26 @@ impl FreeList { prev_end = Some(end); } + + // Check bump region validity. + assert!(self.bump_ptr <= self.bump_end); + if self.bump_ptr < self.bump_end { + assert_eq!(self.bump_ptr % ALIGN_U32, 0); + assert_eq!(self.bump_end % ALIGN_U32, 0); + assert!(usize::try_from(self.bump_end).unwrap() <= self.capacity); + // Bump region should not overlap with any block. + for (&index, &len) in self.free_block_index_to_len.iter() { + let block_end = index + len; + assert!( + self.bump_end <= index || self.bump_ptr >= block_end, + "bump region [{}, {}) overlaps with block [{}, {})", + self.bump_ptr, + self.bump_end, + index, + block_end + ); + } + } } } @@ -431,12 +546,15 @@ mod tests { use std::num::NonZeroUsize; fn free_list_block_len_and_size(free_list: &FreeList) -> (usize, Option) { - let len = free_list.free_block_index_to_len.len(); - let size = free_list - .free_block_index_to_len - .values() - .next() - .map(|s| usize::try_from(*s).unwrap()); + let len = free_list.num_free_blocks(); + let size = if free_list.bump_end > free_list.bump_ptr { + Some(usize::try_from(free_list.bump_end - free_list.bump_ptr).unwrap()) + } else { + free_list + .free_block_index_to_len + .first_key_value() + .map(|(_, &s)| usize::try_from(s).unwrap()) + }; (len, size) } @@ -598,7 +716,7 @@ mod tests { // `ALIGN_U32`. let mut free_list = FreeList::new(ALIGN_USIZE + ALIGN_USIZE * 2); - assert_eq!(free_list.free_block_index_to_len.len(), 1); + assert_eq!(free_list.num_free_blocks(), 1); assert_eq!(free_list.max_size(), ALIGN_USIZE * 2); // Allocate a block such that the remainder is not worth splitting. @@ -608,7 +726,7 @@ mod tests { .expect("have free space available for allocation"); // Should not have split the block. - assert_eq!(free_list.free_block_index_to_len.len(), 0); + assert_eq!(free_list.num_free_blocks(), 0); } #[test] @@ -617,7 +735,7 @@ mod tests { // `ALIGN_U32`. let mut free_list = FreeList::new(ALIGN_USIZE + ALIGN_USIZE * 3); - assert_eq!(free_list.free_block_index_to_len.len(), 1); + assert_eq!(free_list.num_free_blocks(), 1); assert_eq!(free_list.max_size(), ALIGN_USIZE * 3); // Allocate a block such that the remainder is not worth splitting. @@ -627,7 +745,7 @@ mod tests { .expect("have free space available for allocation"); // Should have split the block. - assert_eq!(free_list.free_block_index_to_len.len(), 1); + assert_eq!(free_list.num_free_blocks(), 1); } #[test] @@ -636,7 +754,7 @@ mod tests { let mut free_list = FreeList::new(ALIGN_USIZE + ALIGN_USIZE * 100); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "initially one big free block" ); @@ -646,7 +764,7 @@ mod tests { .expect("allocation within 'static' free list limits") .expect("have free space available for allocation"); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "should have split the block to allocate `a`" ); @@ -656,21 +774,21 @@ mod tests { .expect("allocation within 'static' free list limits") .expect("have free space available for allocation"); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "should have split the block to allocate `b`" ); free_list.dealloc(a, layout); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 2, "should have two non-contiguous free blocks after deallocating `a`" ); free_list.dealloc(b, layout); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "should have merged `a` and `b` blocks with the rest to form a \ single, contiguous free block after deallocating `b`" @@ -683,7 +801,7 @@ mod tests { let mut free_list = FreeList::new(ALIGN_USIZE + ALIGN_USIZE * 100); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "initially one big free block" ); @@ -701,21 +819,21 @@ mod tests { .expect("allocation within 'static' free list limits") .expect("have free space available for allocation"); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "should have split the block to allocate `a`, `b`, and `c`" ); free_list.dealloc(a, layout); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 2, "should have two non-contiguous free blocks after deallocating `a`" ); free_list.dealloc(b, layout); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 2, "should have merged `a` and `b` blocks, but not merged with the \ rest of the free space" @@ -730,7 +848,7 @@ mod tests { let mut free_list = FreeList::new(ALIGN_USIZE + ALIGN_USIZE * 100); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "initially one big free block" ); @@ -748,21 +866,21 @@ mod tests { .expect("allocation within 'static' free list limits") .expect("have free space available for allocation"); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "should have split the block to allocate `a`, `b`, and `c`" ); free_list.dealloc(a, layout); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 2, "should have two non-contiguous free blocks after deallocating `a`" ); free_list.dealloc(c, layout); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 2, "should have merged `c` block with rest of the free space, but not \ with `a` block" @@ -777,7 +895,7 @@ mod tests { let mut free_list = FreeList::new(ALIGN_USIZE + ALIGN_USIZE * 100); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "initially one big free block" ); @@ -799,21 +917,21 @@ mod tests { .expect("allocation within 'static' free list limits") .expect("have free space available for allocation"); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "should have split the block to allocate `a`, `b`, `c`, and `d`" ); free_list.dealloc(a, layout); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 2, "should have two non-contiguous free blocks after deallocating `a`" ); free_list.dealloc(c, layout); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 3, "should not have merged `c` block `a` block or rest of the free \ space" @@ -921,14 +1039,14 @@ mod tests { free_list.dealloc(a, layout); free_list.dealloc(b, layout); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "`dealloc` should merge blocks from different `add_capacity` calls together" ); free_list.add_capacity(ALIGN_USIZE); assert_eq!( - free_list.free_block_index_to_len.len(), + free_list.num_free_blocks(), 1, "`add_capacity` should eagerly merge new capacity into the last block \ in the free list, when possible" From a8d2f01c9d0fafd3cda4dcc09c177ed15c083f17 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Tue, 7 Apr 2026 12:42:41 -0700 Subject: [PATCH 2/2] Address review feedback --- .../src/runtime/vm/gc/enabled/free_list.rs | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs index 5a0bd4d90c0b..d21d9026e4bb 100644 --- a/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs +++ b/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs @@ -232,7 +232,6 @@ impl FreeList { debug_assert_ne!(result, 0); debug_assert_eq!(result % ALIGN_U32, 0); - #[cfg(debug_assertions)] self.check_integrity(); log::trace!("FreeList::alloc -> {result:#x}"); @@ -240,14 +239,12 @@ impl FreeList { } // After we've mutated the free list, double check its integrity. - #[cfg(debug_assertions)] self.check_integrity(); // Slow path: find a block in the blocks list, then set it as bump region. self.alloc_slow(alloc_size) } - #[inline(never)] #[cold] fn alloc_slow(&mut self, alloc_size: u32) -> Option { // Put the remaining bump region back into blocks if non-empty. @@ -274,7 +271,6 @@ impl FreeList { self.bump_end = block_index + block_len; debug_assert_ne!(block_index, 0); - #[cfg(debug_assertions)] self.check_integrity(); Some(unsafe { NonZeroU32::new_unchecked(block_index) }) @@ -317,9 +313,7 @@ impl FreeList { } } - #[cfg(debug_assertions)] self.check_integrity(); - return; } @@ -333,9 +327,7 @@ impl FreeList { self.bump_end += block_len; } - #[cfg(debug_assertions)] self.check_integrity(); - return; } @@ -420,7 +412,6 @@ impl FreeList { // After we've added to/mutated the free list, double check its // integrity. - #[cfg(debug_assertions)] self.check_integrity(); } @@ -454,8 +445,11 @@ impl FreeList { /// 3. All blocks are aligned to `ALIGN` /// /// 4. All block sizes are a multiple of `ALIGN` - #[cfg(debug_assertions)] fn check_integrity(&self) { + if !cfg!(gc_zeal) { + return; + } + let mut prev_end = None; for (&index, &len) in self.free_block_index_to_len.iter() { // (1)