From 065fb6479491adbe9ff85794c39a691f8e0943d9 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 6 Apr 2026 10:57:38 -0700 Subject: [PATCH] Prepare FACT for supporting more memory types This commit refactors the internals of FACT, wasmtime's component-to-component trampoline compiler, to better handle memories of different types. When FACT was originally written it had support for memory64, despite components not having support for memory64, but since then more types of memories have showed up in core wasm. For example FACT didn't handle shared memories or custom-page-sizes memories well. This commit manually updates some `memory64: bool` fields/etc to `ty: Memory` to have a full-fledged memory type on-hand during translation. This affects how the memory is imported, for example, as well as bounds checks. Note that components do not currently support 64-bit memories, nor shared memories, nor custom-page-size memories. This refactoring is for future support of these features internally within FACT itself, but more support will be necessary to fully support these features throughout the runtime. --- .../environ/src/component/translate/adapt.rs | 16 +-- .../environ/src/component/translate/inline.rs | 48 +++------ crates/environ/src/fact.rs | 87 ++++++++------- crates/environ/src/fact/signature.rs | 2 +- crates/environ/src/fact/trampoline.rs | 100 ++++++++---------- 5 files changed, 116 insertions(+), 137 deletions(-) diff --git a/crates/environ/src/component/translate/adapt.rs b/crates/environ/src/component/translate/adapt.rs index 111533281b83..baedeedf70e5 100644 --- a/crates/environ/src/component/translate/adapt.rs +++ b/crates/environ/src/component/translate/adapt.rs @@ -115,9 +115,9 @@ //! time this may want to be revisited if too many adapter modules are being //! created. -use crate::EntityType; use crate::component::translate::*; use crate::fact; +use crate::{EntityType, Memory}; use std::collections::HashSet; /// Metadata information about a fused adapter. @@ -150,10 +150,8 @@ pub enum DataModel { /// Data is stored in a linear memory. LinearMemory { - /// An optional memory definition supplied. - memory: Option>, - /// If `memory` is specified, whether it's a 64-bit memory. - memory64: bool, + /// An optional memory definition supplied, and its type. + memory: Option<(dfg::CoreExport, Memory)>, /// An optional definition of `realloc` to used. realloc: Option, }, @@ -416,12 +414,8 @@ impl PartitionAdapterModules { DataModel::Gc {} => { // Nothing to do here yet. } - DataModel::LinearMemory { - memory, - memory64: _, - realloc, - } => { - if let Some(memory) = memory { + DataModel::LinearMemory { memory, realloc } => { + if let Some((memory, _ty)) = memory { self.core_export(dfg, memory); } if let Some(def) = realloc { diff --git a/crates/environ/src/component/translate/inline.rs b/crates/environ/src/component/translate/inline.rs index e1e87643c973..5b9a32623fbf 100644 --- a/crates/environ/src/component/translate/inline.rs +++ b/crates/environ/src/component/translate/inline.rs @@ -46,7 +46,7 @@ //! final `Component`. use crate::component::translate::*; -use crate::{EntityType, IndexType}; +use crate::{EntityType, Memory}; use core::str::FromStr; use std::borrow::Cow; use wasmparser::component_types::{ComponentAnyTypeId, ComponentCoreModuleTypeId}; @@ -1520,34 +1520,25 @@ impl<'a> Inliner<'a> { frame: &InlinerFrame<'a>, types: &ComponentTypesBuilder, memory: MemoryIndex, - ) -> (dfg::CoreExport, bool) { + ) -> (dfg::CoreExport, Memory) { let memory = frame.memories[memory].clone().map_index(|i| match i { EntityIndex::Memory(i) => i, _ => unreachable!(), }); - let memory64 = match &self.runtime_instances[memory.instance] { + let ty = match &self.runtime_instances[memory.instance] { InstanceModule::Static(idx) => match &memory.item { - ExportItem::Index(i) => { - let ty = &self.nested_modules[*idx].module.memories[*i]; - match ty.idx_type { - IndexType::I32 => false, - IndexType::I64 => true, - } - } + ExportItem::Index(i) => self.nested_modules[*idx].module.memories[*i], ExportItem::Name(_) => unreachable!(), }, InstanceModule::Import(ty) => match &memory.item { ExportItem::Name(name) => match types[*ty].exports[name] { - EntityType::Memory(m) => match m.idx_type { - IndexType::I32 => false, - IndexType::I64 => true, - }, + EntityType::Memory(m) => m, _ => unreachable!(), }, ExportItem::Index(_) => unreachable!(), }, }; - (memory, memory64) + (memory, ty) } /// Translates a `LocalCanonicalOptions` which indexes into the `frame` @@ -1562,18 +1553,9 @@ impl<'a> Inliner<'a> { let data_model = match options.data_model { LocalDataModel::Gc {} => DataModel::Gc {}, LocalDataModel::LinearMemory { memory, realloc } => { - let (memory, memory64) = memory - .map(|i| { - let (memory, memory64) = self.memory(frame, types, i); - (Some(memory), memory64) - }) - .unwrap_or((None, false)); + let memory = memory.map(|i| self.memory(frame, types, i)); let realloc = realloc.map(|i| frame.funcs[i].1.clone()); - DataModel::LinearMemory { - memory, - memory64, - realloc, - } + DataModel::LinearMemory { memory, realloc } } }; let callback = options.callback.map(|i| frame.funcs[i].1.clone()); @@ -1603,14 +1585,12 @@ impl<'a> Inliner<'a> { fn canonical_options(&mut self, options: AdapterOptions) -> dfg::OptionsId { let data_model = match options.data_model { DataModel::Gc {} => dfg::CanonicalOptionsDataModel::Gc {}, - DataModel::LinearMemory { - memory, - memory64: _, - realloc, - } => dfg::CanonicalOptionsDataModel::LinearMemory { - memory: memory.map(|export| self.result.memories.push(export)), - realloc: realloc.map(|def| self.result.reallocs.push(def)), - }, + DataModel::LinearMemory { memory, realloc } => { + dfg::CanonicalOptionsDataModel::LinearMemory { + memory: memory.map(|(export, _)| self.result.memories.push(export)), + realloc: realloc.map(|def| self.result.reallocs.push(def)), + } + } }; let callback = options.callback.map(|def| self.result.callbacks.push(def)); let post_return = options diff --git a/crates/environ/src/fact.rs b/crates/environ/src/fact.rs index e3be059bbf79..935d746eb7d3 100644 --- a/crates/environ/src/fact.rs +++ b/crates/environ/src/fact.rs @@ -20,12 +20,16 @@ use crate::component::dfg::CoreDef; use crate::component::{ - Adapter, AdapterOptions as AdapterOptionsDfg, ComponentTypesBuilder, FlatType, InterfaceType, - RuntimeComponentInstanceIndex, StringEncoding, Transcode, TypeFuncIndex, + Adapter, AdapterOptions as AdapterOptionsDfg, CanonicalAbiInfo, ComponentTypesBuilder, + FlatType, InterfaceType, RuntimeComponentInstanceIndex, StringEncoding, Transcode, + TypeFuncIndex, }; use crate::fact::transcode::Transcoder; -use crate::{EntityRef, FuncIndex, GlobalIndex, MemoryIndex, PrimaryMap, Tunables}; -use crate::{ModuleInternedTypeIndex, prelude::*}; +use crate::prelude::*; +use crate::{ + EntityRef, FuncIndex, GlobalIndex, IndexType, Memory, MemoryIndex, ModuleInternedTypeIndex, + PrimaryMap, Tunables, +}; use std::collections::HashMap; use wasm_encoder::*; @@ -142,11 +146,9 @@ struct AdapterOptions { #[derive(PartialEq, Eq, Hash, Copy, Clone)] /// Linear memory. struct LinearMemoryOptions { - /// Whether or not the `memory` field, if present, is a 64-bit memory. - memory64: bool, /// An optionally-specified memory where values may travel through for /// types like lists. - memory: Option, + memory: Option<(MemoryIndex, Memory)>, /// An optionally-specified function to be used to allocate space for /// types such as strings as they go into a module. realloc: Option, @@ -154,7 +156,7 @@ struct LinearMemoryOptions { impl LinearMemoryOptions { fn ptr(&self) -> ValType { - if self.memory64 { + if self.memory64() { ValType::I64 } else { ValType::I32 @@ -162,7 +164,22 @@ impl LinearMemoryOptions { } fn ptr_size(&self) -> u8 { - if self.memory64 { 8 } else { 4 } + if self.memory64() { 8 } else { 4 } + } + + fn memory64(&self) -> bool { + self.memory + .as_ref() + .map(|(_, ty)| ty.idx_type == IndexType::I64) + .unwrap_or(false) + } + + fn sizealign(&self, abi: &CanonicalAbiInfo) -> (u32, u32) { + if self.memory64() { + (abi.size64, abi.align64) + } else { + (abi.size32, abi.align32) + } } } @@ -345,30 +362,32 @@ impl<'a> Module<'a> { let data_model = match data_model { crate::component::DataModel::Gc {} => DataModel::Gc {}, - crate::component::DataModel::LinearMemory { - memory, - memory64, - realloc, - } => { - let memory = memory.as_ref().map(|memory| { - self.import_memory( - "memory", - &format!("m{}", self.imported_memories.len()), - MemoryType { - minimum: 0, - maximum: None, - shared: false, - memory64: *memory64, - page_size_log2: None, - }, - memory.clone().into(), + crate::component::DataModel::LinearMemory { memory, realloc } => { + let memory = memory.as_ref().map(|(memory, ty)| { + ( + self.import_memory( + "memory", + &format!("m{}", self.imported_memories.len()), + MemoryType { + minimum: 0, + maximum: None, + shared: ty.shared, + memory64: ty.idx_type == IndexType::I64, + page_size_log2: if ty.page_size_log2 == 16 { + None + } else { + Some(ty.page_size_log2.into()) + }, + }, + memory.clone().into(), + ), + *ty, ) }); let realloc = realloc.as_ref().map(|func| { - let ptr = if *memory64 { - ValType::I64 - } else { - ValType::I32 + let ptr = match memory.as_ref().unwrap().1.idx_type { + IndexType::I32 => ValType::I32, + IndexType::I64 => ValType::I64, }; let ty = self.core_types.function(&[ptr, ptr, ptr, ptr], &[ptr]); self.import_func( @@ -378,11 +397,7 @@ impl<'a> Module<'a> { func.clone(), ) }); - DataModel::LinearMemory(LinearMemoryOptions { - memory64: *memory64, - memory, - realloc, - }) + DataModel::LinearMemory(LinearMemoryOptions { memory, realloc }) } }; @@ -907,7 +922,7 @@ impl Options { let flat = types.flat_types(ty)?; match self.data_model { DataModel::Gc {} => todo!("CM+GC"), - DataModel::LinearMemory(mem_opts) => Some(if mem_opts.memory64 { + DataModel::LinearMemory(mem_opts) => Some(if mem_opts.memory64() { flat.memory64 } else { flat.memory32 diff --git a/crates/environ/src/fact/signature.rs b/crates/environ/src/fact/signature.rs index d0c170a0c533..bfe3a865d252 100644 --- a/crates/environ/src/fact/signature.rs +++ b/crates/environ/src/fact/signature.rs @@ -215,7 +215,7 @@ impl ComponentTypesBuilder { // seems like it would be best to intern this in some sort of map somewhere. pub(super) fn size_align(&self, opts: &LinearMemoryOptions, ty: &InterfaceType) -> (u32, u32) { let abi = self.canonical_abi(ty); - if opts.memory64 { + if opts.memory64() { (abi.size64, abi.align64) } else { (abi.size32, abi.align32) diff --git a/crates/environ/src/fact/trampoline.rs b/crates/environ/src/fact/trampoline.rs index 23b54a7699a1..8667dc7c2f61 100644 --- a/crates/environ/src/fact/trampoline.rs +++ b/crates/environ/src/fact/trampoline.rs @@ -31,7 +31,7 @@ use crate::fact::{ LinearMemoryOptions, Module, Options, }; use crate::prelude::*; -use crate::{FuncIndex, GlobalIndex, Trap}; +use crate::{FuncIndex, GlobalIndex, IndexType, Trap}; use std::collections::HashMap; use std::mem; use std::ops::Range; @@ -536,7 +536,7 @@ impl<'a, 'b> Compiler<'a, 'b> { &lower_sig.params, match adapter.lift.options.data_model { DataModel::Gc {} => todo!("CM+GC"), - DataModel::LinearMemory(LinearMemoryOptions { memory, .. }) => memory, + DataModel::LinearMemory(LinearMemoryOptions { memory, .. }) => memory.map(|m| m.0), }, ); @@ -957,13 +957,8 @@ impl<'a, 'b> Compiler<'a, 'b> { let abi = CanonicalAbiInfo::record(dst_tys.iter().map(|t| self.types.canonical_abi(t))); match lift_opts.data_model { DataModel::Gc {} => todo!("CM+GC"), - DataModel::LinearMemory(LinearMemoryOptions { memory64, .. }) => { - let (size, align) = if memory64 { - (abi.size64, abi.align64) - } else { - (abi.size32, abi.align32) - }; - + DataModel::LinearMemory(opts) => { + let (size, align) = opts.sizealign(&abi); // If there are too many parameters then space is allocated in the // destination module for the parameters via its `realloc` function. let size = MallocSize::Const(size); @@ -2421,22 +2416,24 @@ impl<'a, 'b> Compiler<'a, 'b> { } ( DataModel::LinearMemory(LinearMemoryOptions { - memory64: src64, - memory: src_mem, + memory: Some((src_mem, src_ty)), realloc: _, }), DataModel::LinearMemory(LinearMemoryOptions { - memory64: dst64, - memory: dst_mem, + memory: Some((dst_mem, dst_ty)), realloc: _, }), ) => self.module.import_transcoder(Transcoder { - from_memory: src_mem.unwrap(), - from_memory64: src64, - to_memory: dst_mem.unwrap(), - to_memory64: dst64, + from_memory: src_mem, + from_memory64: src_ty.idx_type == IndexType::I64, + to_memory: dst_mem, + to_memory64: dst_ty.idx_type == IndexType::I64, op, }), + (DataModel::LinearMemory(LinearMemoryOptions { memory: None, .. }), _) + | (_, DataModel::LinearMemory(LinearMemoryOptions { memory: None, .. })) => { + unreachable!() + } } } @@ -2457,21 +2454,22 @@ impl<'a, 'b> Compiler<'a, 'b> { trap: Trap, ) { let extend_to_64 = |me: &mut Self| { - if !opts.memory64 { + if !opts.memory64() { me.instruction(I64ExtendI32U); } }; self.instruction(Block(BlockType::Empty)); self.instruction(Block(BlockType::Empty)); + let (memory, ty) = opts.memory.unwrap(); // Calculate the full byte size of memory with `memory.size`. Note that // arithmetic here is done always in 64-bits to accommodate 4G memories. // Additionally it's assumed that 64-bit memories never fill up // entirely. - self.instruction(MemorySize(opts.memory.unwrap().as_u32())); + self.instruction(MemorySize(memory.as_u32())); extend_to_64(self); - self.instruction(I64Const(16)); + self.instruction(I64Const(ty.page_size_log2.into())); self.instruction(I64Shl); // Calculate the end address of the string. This is done by adding the @@ -2483,7 +2481,7 @@ impl<'a, 'b> Compiler<'a, 'b> { self.instruction(LocalGet(byte_len_local)); extend_to_64(self); self.instruction(I64Add); - if opts.memory64 { + if opts.memory64() { let tmp = self.local_tee_new_tmp(ValType::I64); self.instruction(LocalGet(ptr_local)); self.ptr_lt_u(opts); @@ -2783,14 +2781,10 @@ impl<'a, 'b> Compiler<'a, 'b> { let src_entry_abi = CanonicalAbiInfo::record([src_key_abi, src_value_abi].into_iter()); let (_, src_key_align) = self.types.size_align(src_mem_opts, &src_map_ty.key); let (_, src_value_align) = self.types.size_align(src_mem_opts, &src_map_ty.value); - let (src_tuple_size, src_entry_align) = if src_mem_opts.memory64 { - (src_entry_abi.size64, src_entry_abi.align64) - } else { - (src_entry_abi.size32, src_entry_abi.align32) - }; + let (src_tuple_size, src_entry_align) = src_mem_opts.sizealign(&src_entry_abi); let src_value_offset = { let mut offset = 0u32; - if src_mem_opts.memory64 { + if src_mem_opts.memory64() { src_key_abi.next_field64(&mut offset); src_value_abi.next_field64(&mut offset) } else { @@ -2804,14 +2798,10 @@ impl<'a, 'b> Compiler<'a, 'b> { let dst_entry_abi = CanonicalAbiInfo::record([dst_key_abi, dst_value_abi].into_iter()); let (_, dst_key_align) = self.types.size_align(dst_mem_opts, &dst_map_ty.key); let (_, dst_value_align) = self.types.size_align(dst_mem_opts, &dst_map_ty.value); - let (dst_tuple_size, dst_entry_align) = if dst_mem_opts.memory64 { - (dst_entry_abi.size64, dst_entry_abi.align64) - } else { - (dst_entry_abi.size32, dst_entry_abi.align32) - }; + let (dst_tuple_size, dst_entry_align) = dst_mem_opts.sizealign(&dst_entry_abi); let dst_value_offset = { let mut offset = 0u32; - if dst_mem_opts.memory64 { + if dst_mem_opts.memory64() { dst_key_abi.next_field64(&mut offset); dst_value_abi.next_field64(&mut offset) } else { @@ -3973,7 +3963,7 @@ impl<'a, 'b> Compiler<'a, 'b> { } fn ptr_load(&mut self, mem: &Memory) { - if mem.mem_opts().memory64 { + if mem.mem_opts().memory64() { self.i64_load(mem); } else { self.i32_load(mem); @@ -3981,7 +3971,7 @@ impl<'a, 'b> Compiler<'a, 'b> { } fn ptr_add(&mut self, opts: &LinearMemoryOptions) { - if opts.memory64 { + if opts.memory64() { self.instruction(I64Add); } else { self.instruction(I32Add); @@ -3989,7 +3979,7 @@ impl<'a, 'b> Compiler<'a, 'b> { } fn ptr_sub(&mut self, opts: &LinearMemoryOptions) { - if opts.memory64 { + if opts.memory64() { self.instruction(I64Sub); } else { self.instruction(I32Sub); @@ -3997,7 +3987,7 @@ impl<'a, 'b> Compiler<'a, 'b> { } fn ptr_mul(&mut self, opts: &LinearMemoryOptions) { - if opts.memory64 { + if opts.memory64() { self.instruction(I64Mul); } else { self.instruction(I32Mul); @@ -4005,7 +3995,7 @@ impl<'a, 'b> Compiler<'a, 'b> { } fn ptr_gt_u(&mut self, opts: &LinearMemoryOptions) { - if opts.memory64 { + if opts.memory64() { self.instruction(I64GtU); } else { self.instruction(I32GtU); @@ -4013,7 +4003,7 @@ impl<'a, 'b> Compiler<'a, 'b> { } fn ptr_lt_u(&mut self, opts: &LinearMemoryOptions) { - if opts.memory64 { + if opts.memory64() { self.instruction(I64LtU); } else { self.instruction(I32LtU); @@ -4021,7 +4011,7 @@ impl<'a, 'b> Compiler<'a, 'b> { } fn ptr_shl(&mut self, opts: &LinearMemoryOptions) { - if opts.memory64 { + if opts.memory64() { self.instruction(I64Shl); } else { self.instruction(I32Shl); @@ -4029,7 +4019,7 @@ impl<'a, 'b> Compiler<'a, 'b> { } fn ptr_eqz(&mut self, opts: &LinearMemoryOptions) { - if opts.memory64 { + if opts.memory64() { self.instruction(I64Eqz); } else { self.instruction(I32Eqz); @@ -4037,7 +4027,7 @@ impl<'a, 'b> Compiler<'a, 'b> { } fn ptr_uconst(&mut self, opts: &LinearMemoryOptions, val: u32) { - if opts.memory64 { + if opts.memory64() { self.instruction(I64Const(val.into())); } else { self.instruction(I32Const(val.cast_signed())); @@ -4045,7 +4035,7 @@ impl<'a, 'b> Compiler<'a, 'b> { } fn ptr_iconst(&mut self, opts: &LinearMemoryOptions, val: i32) { - if opts.memory64 { + if opts.memory64() { self.instruction(I64Const(val.into())); } else { self.instruction(I32Const(val)); @@ -4053,7 +4043,7 @@ impl<'a, 'b> Compiler<'a, 'b> { } fn ptr_eq(&mut self, opts: &LinearMemoryOptions) { - if opts.memory64 { + if opts.memory64() { self.instruction(I64Eq); } else { self.instruction(I32Eq); @@ -4061,7 +4051,7 @@ impl<'a, 'b> Compiler<'a, 'b> { } fn ptr_ne(&mut self, opts: &LinearMemoryOptions) { - if opts.memory64 { + if opts.memory64() { self.instruction(I64Ne); } else { self.instruction(I32Ne); @@ -4069,7 +4059,7 @@ impl<'a, 'b> Compiler<'a, 'b> { } fn ptr_and(&mut self, opts: &LinearMemoryOptions) { - if opts.memory64 { + if opts.memory64() { self.instruction(I64And); } else { self.instruction(I32And); @@ -4077,7 +4067,7 @@ impl<'a, 'b> Compiler<'a, 'b> { } fn ptr_or(&mut self, opts: &LinearMemoryOptions) { - if opts.memory64 { + if opts.memory64() { self.instruction(I64Or); } else { self.instruction(I32Or); @@ -4085,7 +4075,7 @@ impl<'a, 'b> Compiler<'a, 'b> { } fn ptr_xor(&mut self, opts: &LinearMemoryOptions) { - if opts.memory64 { + if opts.memory64() { self.instruction(I64Xor); } else { self.instruction(I32Xor); @@ -4093,7 +4083,7 @@ impl<'a, 'b> Compiler<'a, 'b> { } fn ptr_if(&mut self, opts: &LinearMemoryOptions, ty: BlockType) { - if opts.memory64 { + if opts.memory64() { self.instruction(I64Const(0)); self.instruction(I64Ne); } @@ -4101,7 +4091,7 @@ impl<'a, 'b> Compiler<'a, 'b> { } fn ptr_br_if(&mut self, opts: &LinearMemoryOptions, depth: u32) { - if opts.memory64 { + if opts.memory64() { self.instruction(I64Const(0)); self.instruction(I64Ne); } @@ -4141,7 +4131,7 @@ impl<'a, 'b> Compiler<'a, 'b> { } fn ptr_store(&mut self, mem: &Memory) { - if mem.mem_opts().memory64 { + if mem.mem_opts().memory64() { self.i64_store(mem); } else { self.i32_store(mem); @@ -4204,7 +4194,7 @@ impl<'a> Source<'a> { Source::Stack(s.slice(1..s.locals.len()).slice(0..flat_len)) } Source::Memory(mem) => { - let mem = if mem.mem_opts().memory64 { + let mem = if mem.mem_opts().memory64() { mem.bump(info.payload_offset64) } else { mem.bump(info.payload_offset32) @@ -4268,7 +4258,7 @@ impl<'a> Destination<'a> { Destination::Stack(&s[1..][..flat_len], opts) } Destination::Memory(mem) => { - let mem = if mem.mem_opts().memory64 { + let mem = if mem.mem_opts().memory64() { mem.bump(info.payload_offset64) } else { mem.bump(info.payload_offset32) @@ -4296,7 +4286,7 @@ fn next_field_offset<'a>( mem: &Memory<'a>, ) -> Memory<'a> { let abi = types.canonical_abi(field); - let offset = if mem.mem_opts().memory64 { + let offset = if mem.mem_opts().memory64() { abi.next_field64(offset) } else { abi.next_field32(offset) @@ -4309,7 +4299,7 @@ impl<'a> Memory<'a> { MemArg { offset: u64::from(self.offset), align, - memory_index: self.mem_opts().memory.unwrap().as_u32(), + memory_index: self.mem_opts().memory.unwrap().0.as_u32(), } }