diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 44d5da3bd690..fc5145058924 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -983,7 +983,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> { self.module.tables[index] } - /// Cast the value to I64 and sign extend if necessary. + /// Cast the value to I64 and zero extend if necessary. /// /// Returns the value casted to I64. fn cast_index_to_i64( @@ -6100,8 +6100,14 @@ impl FuncEnvironment<'_> { } } } - for segment in self.translation.table_initialization.segments.iter() { - self.module_initialize_table_with_segment(builder, segment)?; + for (segment_index, segment) in self + .translation + .table_initialization + .segments + .iter() + .enumerate() + { + self.module_initialize_table_with_segment(builder, segment, segment_index)?; } match &self.translation.memory_init { MemoryInit::Unprocessed(_) => unreachable!(), @@ -6237,12 +6243,13 @@ impl FuncEnvironment<'_> { /// /// Note that this isn't done for "precomputed" tables which won't have /// initializers present. Additionally note that this doesn't use - /// `table.init` because the element segment won't actually be present - /// anywhere at runtime. + /// `table.init` because active segments are unavailable to that instruction. + /// Eligible function-index lists are retained separately for lazy reads. fn module_initialize_table_with_segment( &mut self, builder: &mut FunctionBuilder, segment: &TableSegment, + segment_index: usize, ) -> WasmResult<()> { let offset = self.translate_const_expr(builder, &segment.offset)?; let segment_len = builder.ins().iconst( @@ -6267,6 +6274,31 @@ impl FuncEnvironment<'_> { .table_init_per_element; let fuel = self.pre_translate_bulk_op(builder, segment_len, cost); + if self.tunables.table_lazy_init + && self.module.is_imported_table(segment.table_index) + && matches!(segment.elements, TableSegmentElements::Functions(_)) + { + // Preserve bounds checks and element fuel charges at the original + // initialization boundary; defer only construction of VMFuncRefs. + let vmctx = self.vmctx_val(&mut builder.cursor()); + let table = builder + .ins() + .iconst(I32, i64::from(segment.table_index.as_u32())); + let segment_id = builder + .ins() + .iconst(I32, i64::from(u32::try_from(segment_index).unwrap())); + let offset = self.cast_index_to_i64( + &mut builder.cursor(), + offset, + self.module.tables[segment.table_index].idx_type, + ); + let libcall = self.builtin_functions.table_init_func_segment(builder.func); + builder + .ins() + .call(libcall, &[vmctx, table, segment_id, offset]); + return self.post_translate_bulk_op(builder, fuel); + } + // Re-use the `table.set` translation for making this a simple function // to define. That re-executes the bounds check which is a bit // inefficient, but can be optimized in the future. diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs index bc4e2f5554da..8798c188d07c 100644 --- a/crates/environ/src/builtin.rs +++ b/crates/environ/src/builtin.rs @@ -199,6 +199,9 @@ macro_rules! foreach_builtin_function { ); #[cfg(feature = "stack-switching")] asan_finish_switch_fiber(vmctx: vmctx, fake_stack: pointer, current_csi: pointer); + // Apply an active function segment without eagerly materializing + // all of its function records. Bounds and ordering remain eager. + table_init_func_segment(vmctx: vmctx, table: u32, segment: u32, offset: u64) -> bool; } }; } diff --git a/crates/environ/src/compile/module_environ.rs b/crates/environ/src/compile/module_environ.rs index 937cf27e7b59..92595dfbd913 100644 --- a/crates/environ/src/compile/module_environ.rs +++ b/crates/environ/src/compile/module_environ.rs @@ -1552,6 +1552,23 @@ impl ModuleTranslation<'_> { pub fn finalize_table_init(&mut self, tunables: &Tunables, types: &mut ModuleTypesBuilder) { if tunables.table_lazy_init { self.try_func_table_init(); + // try_func_table_init removes a prefix, so capture segment positions + // only afterwards to match startup translation. General expressions + // retain their existing eager evaluation and ordering. + self.module.deferred_table_segments = self + .table_initialization + .segments + .iter() + .map(|segment| match &segment.elements { + TableSegmentElements::Functions(indices) + if self.module.is_imported_table(segment.table_index) => + { + indices.iter().copied().try_collect().panic_on_oom() + } + _ => TryVec::new(), + }) + .try_collect() + .panic_on_oom(); } // If any table has a non-null initializers, or if there's any active diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index f15583615a07..f80f413e7b71 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -96,19 +96,18 @@ pub struct TableInitialization { /// Initial values for tables defined within the module itself. /// /// This contains the initial values and initializers for tables defined - /// within a wasm, so excluding imported tables. This initializer can - /// represent null-initialized tables, element-initialized tables (e.g. with - /// the function-references proposal), or precomputed images of table - /// initialization. For example table initializers to a table that are all - /// in-bounds will get removed from `segment` and moved into - /// `initial_values` here. + /// within a wasm, so excluding imported tables. Initial values are null or + /// constant expressions (e.g. with the function-references proposal). + /// When an initial value is precomputed in [`Module::table_initialization`], + /// its entry here becomes [`TableInitialValue::Null`]. pub initial_values: TryPrimaryMap, /// Element segments present in the initial wasm module which are executed /// at instantiation time. /// - /// These element segments are iterated over during instantiation to apply - /// any segments that weren't already moved into `initial_values` above. + /// Finalization removes segments folded into [`Module::table_initialization`]. + /// The remaining segments execute in order during startup; their positions + /// also index [`Module::deferred_table_segments`] when lazy init is enabled. pub segments: TryVec, } @@ -201,6 +200,12 @@ pub struct Module { /// to be infallible as part of [`ModuleTranslation::finalize_table_init`]. pub table_initialization: TryPrimaryMap>, + /// Shared function indices for active imported-table segments, indexed by + /// their position in `TableInitialization::segments` after finalization. + /// Ineligible segments have empty lists. Runtime bindings identify the + /// contributing instance, which its Store retains even if startup traps. + pub deferred_table_segments: TryVec>, + /// WebAssembly linear memory initializer. /// /// This will track how memory is initialized, either exclusively via @@ -335,6 +340,7 @@ impl Module { exports: Default::default(), startup: ModuleStartup::None, table_initialization: Default::default(), + deferred_table_segments: Default::default(), memory_initialization: Default::default(), passive_elements: Default::default(), runtime_data: Default::default(), @@ -682,6 +688,7 @@ impl TypeTrace for Module { exports: _, startup, table_initialization: _, + deferred_table_segments: _, memory_initialization: _, passive_elements: _, runtime_data: _, @@ -734,6 +741,7 @@ impl TypeTrace for Module { exports: _, startup, table_initialization: _, + deferred_table_segments: _, memory_initialization: _, passive_elements: _, runtime_data: _, diff --git a/crates/wasmtime/src/runtime/func.rs b/crates/wasmtime/src/runtime/func.rs index d7de03f1fb50..81a14ffa29ba 100644 --- a/crates/wasmtime/src/runtime/func.rs +++ b/crates/wasmtime/src/runtime/func.rs @@ -2439,7 +2439,7 @@ impl HostFunc { // SAFETY: this is an entrypoint of wasm which requires correct type // ascription of `T` itself, meaning that this should be safe to call // both `enter_host_from_wasm` as well as `unchecked_context_mut`. - unsafe { vm::Instance::enter_host_from_wasm(caller_vmctx, run) } + unsafe { vm::Instance::enter_host_from_wasm(caller_vmctx, true, run) } } /// Analog of [`Func::new_unchecked`] /// diff --git a/crates/wasmtime/src/runtime/vm/component.rs b/crates/wasmtime/src/runtime/vm/component.rs index e8a276411aae..2519fb47d676 100644 --- a/crates/wasmtime/src/runtime/vm/component.rs +++ b/crates/wasmtime/src/runtime/vm/component.rs @@ -270,7 +270,9 @@ impl ComponentInstance { let store = unsafe { &mut *reference.store.0.as_ptr() }; let instance = Instance::from_wasmtime(store, reference.id); - catch_unwind_and_record_trap(store, UncaughtException::Trap, |store| f(store, instance)) + catch_unwind_and_record_trap(store, UncaughtException::Trap, true, |store| { + f(store, instance) + }) } /// Returns the `InstanceId` associated with the `vmctx` provided. diff --git a/crates/wasmtime/src/runtime/vm/debug_builtins.rs b/crates/wasmtime/src/runtime/vm/debug_builtins.rs index 0d18ff4fb737..52315927d542 100644 --- a/crates/wasmtime/src/runtime/vm/debug_builtins.rs +++ b/crates/wasmtime/src/runtime/vm/debug_builtins.rs @@ -18,7 +18,7 @@ pub unsafe extern "C" fn resolve_vmctx_memory_ptr(p: *const u32) -> *const u8 { VMCTX_AND_MEMORY.0 != NonNull::dangling(), "must call `__vmctx->set()` before resolving Wasm pointers" ); - Instance::enter_host_from_wasm(VMCTX_AND_MEMORY.0, |store, instance| { + Instance::enter_host_from_wasm(VMCTX_AND_MEMORY.0, true, |store, instance| { let handle = store.instance_mut(instance); assert!( VMCTX_AND_MEMORY.1 < handle.env_module().memories.len(), diff --git a/crates/wasmtime/src/runtime/vm/instance.rs b/crates/wasmtime/src/runtime/vm/instance.rs index 7ecfa1eaf09f..3aa70276ee91 100644 --- a/crates/wasmtime/src/runtime/vm/instance.rs +++ b/crates/wasmtime/src/runtime/vm/instance.rs @@ -319,6 +319,7 @@ impl Instance { #[inline] pub(crate) unsafe fn enter_host_from_wasm( vmctx: NonNull, + call_hooks: bool, f: impl FnOnce(&mut dyn VMStore, InstanceId) -> R, ) -> R::Abi where @@ -337,7 +338,7 @@ impl Instance { // Thread the `store` and `instance` through panic/trap infrastructure // back into `f`. - catch_unwind_and_record_trap(store, UncaughtException::Propagate, |store| { + catch_unwind_and_record_trap(store, UncaughtException::Propagate, call_hooks, |store| { f(store, instance) }) } @@ -1051,6 +1052,41 @@ impl Instance { self.wasm_data(range) } + pub(crate) fn initialize_deferred_func_segment( + self: Pin<&mut Self>, + registry: &ModuleRegistry, + table_index: TableIndex, + segment: u32, + start: u64, + ) -> Result<()> { + let (module, mut instance) = self.module_and_self(); + let indices = &module.deferred_table_segments[segment as usize]; + let source = instance.vmctx().into(); + let deferred = { + let (idx, owner) = instance + .as_mut() + .defined_table_index_and_instance(table_index); + owner.get_defined_table(idx).defer_func_segment( + start, + indices.len() as u64, + source, + segment, + )? + }; + if !deferred { + for (offset, &function) in indices.iter().enumerate() { + let function = instance.as_mut().get_func_ref(registry, function); + let (idx, owner) = instance + .as_mut() + .defined_table_index_and_instance(table_index); + owner + .get_defined_table(idx) + .set_func(start + offset as u64, function)?; + } + } + Ok(()) + } + /// Get a table by index regardless of whether it is locally-defined /// or an imported, foreign table. Ensure that the given range of /// elements in the table is lazily initialized. We define this @@ -1076,6 +1112,8 @@ impl Instance { /// given the provided `idx`. /// /// The `range` specified is eagerly initialized for funcref tables. + /// Deferred segments can access sibling instances even though this table + /// is locally defined; callers must not hold independent instance borrows. pub fn get_defined_table_with_lazy_init( mut self: Pin<&mut Self>, registry: &ModuleRegistry, @@ -1097,19 +1135,29 @@ impl Instance { Err(_) => break, }; - // The table element `i` is uninitialized and is now being - // initialized. This must imply that a `precompiled` list of - // function indices is available for this table. The precompiled - // list is extracted and then it is consulted with `i` to - // determine the function that is going to be initialized. Note - // that `i` may be outside the limits of the static - // initialization so it's a fallible `get` instead of an index. - let module = self.env_module(); - let precomputed = &module.table_initialization[idx]; - // Panicking here helps catch bugs rather than silently truncating by accident. - let func_index = precomputed.get(usize::try_from(i).unwrap()).cloned(); - let func_ref = func_index - .and_then(|func_index| self.as_mut().get_func_ref(registry, func_index)); + // An imported-table segment can replace the owner's static + // initializer. Resolve its contributing instance first; outside + // those ranges, a missing static initializer means null. + let index = usize::try_from(i).unwrap(); + let deferred = self.tables[idx].1.deferred_func_segment(index); + let func_ref = if let Some(deferred) = deferred { + // SAFETY: startup runs after Store insertion, so the source + // remains alive even if a later segment or start traps. + // Imports are from the same Store. sibling_vmctx_mut ties + // the source borrow to our exclusive instance borrow; no + // table or owner access occurs until it ends. + let source = unsafe { + self.as_mut() + .sibling_vmctx_mut(deferred.source.as_non_null()) + }; + let function = source.env_module().deferred_table_segments + [deferred.segment as usize][index - deferred.start]; + source.get_func_ref(registry, function) + } else { + let precomputed = &self.env_module().table_initialization[idx]; + let func_index = precomputed.get(index).copied(); + func_index.and_then(|function| self.as_mut().get_func_ref(registry, function)) + }; self.as_mut().tables_mut()[idx] .1 .set_func(i, func_ref) diff --git a/crates/wasmtime/src/runtime/vm/interpreter.rs b/crates/wasmtime/src/runtime/vm/interpreter.rs index 39045ffb7666..8e08ac9a5eb3 100644 --- a/crates/wasmtime/src/runtime/vm/interpreter.rs +++ b/crates/wasmtime/src/runtime/vm/interpreter.rs @@ -530,7 +530,7 @@ impl InterpreterRef<'_> { // SAFETY: `caller` is valid and its store is not otherwise // borrowed, per this function's contract. unsafe { - Instance::enter_host_from_wasm(caller, |store, _instance| { + Instance::enter_host_from_wasm(caller, true, |store, _instance| { trap = store .store_opaque() .modules() diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs index 7bd010bc9f16..74740ef7523f 100644 --- a/crates/wasmtime/src/runtime/vm/libcalls.rs +++ b/crates/wasmtime/src/runtime/vm/libcalls.rs @@ -122,7 +122,7 @@ pub mod raw { ) $(-> libcall!(@ty $result))? { $(#[cfg($attr)])? unsafe { - Instance::enter_host_from_wasm(vmctx, |store, instance| { + Instance::enter_host_from_wasm(vmctx, libcall!(@call_hooks $name), |store, instance| { super::$name(store, instance, $($pname),*) }) } @@ -149,6 +149,11 @@ pub mod raw { )* }; + // Active segments previously used infallible ref_func calls without + // host transitions. Deferred initialization must retain that behavior. + (@call_hooks table_init_func_segment) => (false); + (@call_hooks $name:ident) => (true); + (@ty u32) => (u32); (@ty u64) => (u64); (@ty f32) => (f32); @@ -405,6 +410,22 @@ fn ref_func(store: &mut dyn VMStore, instance: InstanceId, func_index: u32) -> N .cast() } +fn table_init_func_segment( + store: &mut dyn VMStore, + instance: InstanceId, + table_index: u32, + segment: u32, + offset: u64, +) -> Result<()> { + let (instance, registry) = store.instance_and_module_registry_mut(instance); + instance.initialize_deferred_func_segment( + registry, + TableIndex::from_u32(table_index), + segment, + offset, + ) +} + // Returns a table entry after lazily initializing it. fn table_get_lazy_init_func_ref( store: &mut dyn VMStore, diff --git a/crates/wasmtime/src/runtime/vm/table.rs b/crates/wasmtime/src/runtime/vm/table.rs index 4bac89bedeb5..68b87eef3bd2 100644 --- a/crates/wasmtime/src/runtime/vm/table.rs +++ b/crates/wasmtime/src/runtime/vm/table.rs @@ -5,7 +5,7 @@ use crate::prelude::*; use crate::runtime::store::StoreResourceLimiter; use crate::runtime::vm::stack_switching::VMContObj; -use crate::runtime::vm::vmcontext::{VMFuncRef, VMTableDefinition}; +use crate::runtime::vm::vmcontext::{VMContext, VMFuncRef, VMTableDefinition}; use crate::runtime::vm::{GcStore, SendSyncPtr, VMGcRef, VmPtr}; use core::alloc::Layout; use core::cmp; @@ -132,6 +132,16 @@ impl From for StaticTable { } } +#[derive(Clone, Copy)] +pub(super) struct DeferredFuncSegment { + pub start: usize, + pub end: usize, + // The Store owns this instance, including after failed startup. Dropping a + // table must never dereference the source; Store teardown has no such order. + pub source: VmPtr, + pub segment: u32, +} + pub struct StaticFuncTable { /// Where data for this table is stored. The length of this list is the /// maximum size of the table. @@ -140,6 +150,7 @@ pub struct StaticFuncTable { size: usize, /// Whether elements of this table are initialized lazily. lazy_init: bool, + deferred_segments: TryVec, } pub struct StaticGcRefTable { @@ -190,6 +201,7 @@ pub struct DynamicFuncTable { maximum: Option, /// Whether elements of this table are initialized lazily. lazy_init: bool, + deferred_segments: TryVec, } pub struct DynamicGcRefTable { @@ -339,6 +351,7 @@ impl Table { elements: unsafe { alloc_dynamic_table_elements(minimum)? }, maximum, lazy_init: tunables.table_lazy_init, + deferred_segments: TryVec::new(), })), TableElementType::GcRef => Ok(Self::from(DynamicGcRefTable { elements: unsafe { alloc_dynamic_table_elements(minimum)? }, @@ -388,6 +401,7 @@ impl Table { data, size, lazy_init: tunables.table_lazy_init, + deferred_segments: TryVec::new(), })) } TableElementType::GcRef => { @@ -723,6 +737,69 @@ impl Table { Ok(Some(old_size)) } + /// Install a segment at its original startup boundary. The zero table + /// marker uses the existing lazy-read path; explicit writes produce tagged + /// values and therefore override this history without separate invalidation. + /// The source must already belong to the same Store as this table. + pub(super) fn defer_func_segment( + &mut self, + start: u64, + len: u64, + source: VmPtr, + segment: u32, + ) -> Result { + let Range { start, end } = self.validate_fill(start, len)?; + if start == end { + return Ok(true); + } + if !self.funcrefs().1 { + return Ok(false); + } + let deferred = match self { + Self::Dynamic(DynamicTable::Func(table)) => &mut table.deferred_segments, + Self::Static(StaticTable::Func(table)) => &mut table.deferred_segments, + _ => unreachable!(), + }; + // Bound both metadata and first-access lookup. Additional segments use + // ordinary eager initialization, so long-lived imported tables cannot + // accumulate unbounded history. Nothing is changed on that path. + const MAX_DEFERRED_SEGMENTS: usize = 128; + if deferred.len() == MAX_DEFERRED_SEGMENTS { + return Ok(false); + } + // Metadata allocation must not introduce a new startup failure after + // CoW memory images have been installed. Eager VMFuncRef initialization + // is allocation-free, and a failed push leaves the table unchanged. + if deferred + .push(DeferredFuncSegment { + start, + end, + source, + segment, + }) + .is_err() + { + return Ok(false); + } + self.funcrefs_mut().0[start..end].fill(MaybeTaggedFuncRef(None)); + Ok(true) + } + + pub(super) fn deferred_func_segment(&self, index: usize) -> Option { + let deferred = match self { + Self::Dynamic(DynamicTable::Func(table)) => &table.deferred_segments, + Self::Static(StaticTable::Func(table)) => &table.deferred_segments, + _ => unreachable!(), + }; + // Active segments apply in declaration order. The most recent range + // wins, including partial overlaps with earlier contributing instances. + deferred + .iter() + .rev() + .find(|s| s.start <= index && index < s.end) + .copied() + } + /// Get reference to the specified element. /// /// Returns `Ok(None)` if the element is null or uninitialized. @@ -855,6 +932,7 @@ impl Table { data, size, lazy_init, + .. })) => ( unsafe { slice::from_raw_parts(data.as_ptr().cast(), *size) }, *lazy_init, @@ -878,6 +956,7 @@ impl Table { data, size, lazy_init, + .. })) => ( unsafe { slice::from_raw_parts_mut(data.as_ptr().cast(), *size) }, *lazy_init, @@ -985,6 +1064,7 @@ impl Default for Table { data: SendSyncPtr::new(NonNull::from(&mut [])), size: 0, lazy_init: false, + deferred_segments: TryVec::new(), }) } } diff --git a/crates/wasmtime/src/runtime/vm/traphandlers.rs b/crates/wasmtime/src/runtime/vm/traphandlers.rs index 60ee369f686a..5f9b0bd37ec6 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers.rs @@ -139,9 +139,12 @@ pub(super) unsafe fn raise_preexisting_trap(store: &mut dyn VMStore) { /// propagate the exception to their wasm caller, but component host functions /// and intrinsics are a component boundary which exceptions cannot cross, so /// there the exception is instead turned into a trap. +/// `call_hooks` is false for internal initialization that must report traps +/// without introducing an observable host-call boundary. pub fn catch_unwind_and_record_trap( store: &mut dyn VMStore, uncaught_exception: UncaughtException, + call_hooks: bool, f: impl FnOnce(&mut dyn VMStore) -> R, ) -> R::Abi where @@ -151,7 +154,7 @@ where // return value is always provided and if unwind information is provided // (e.g. `ret` is a "false"-y value) then it's recorded in TLS for the // unwind operation that's about to happen from Cranelift-generated code. - let (ret, unwind) = R::maybe_catch_unwind(store, |store| f(store)); + let (ret, unwind) = R::maybe_catch_unwind(store, call_hooks, |store| f(store)); if let Some(mut unwind) = unwind { if let UnwindReason::Trap(Ok(trap)) = &mut unwind { uncaught_exception.apply(&mut trap.reason); @@ -190,6 +193,7 @@ pub trait HostResult { /// and initiate an unwind (currently through `raise_preexisting_trap`). fn maybe_catch_unwind( store: &mut dyn VMStore, + call_hooks: bool, f: impl FnOnce(&mut dyn VMStore) -> Self, ) -> (Self::Abi, Option); } @@ -207,6 +211,7 @@ macro_rules! host_result_no_catch { #[allow(unreachable_code, reason = "some types uninhabited on some platforms")] fn maybe_catch_unwind( store: &mut dyn VMStore, + _call_hooks: bool, f: impl FnOnce(&mut dyn VMStore) -> $t, ) -> ($t, Option) { (f(store), None) @@ -234,6 +239,7 @@ impl HostResult for NonNull { type Abi = *mut u8; fn maybe_catch_unwind( store: &mut dyn VMStore, + _call_hooks: bool, f: impl FnOnce(&mut dyn VMStore) -> Self, ) -> (*mut u8, Option) { (f(store).as_ptr(), None) @@ -264,8 +270,12 @@ where fn maybe_catch_unwind( store: &mut dyn VMStore, + call_hooks: bool, f: impl FnOnce(&mut dyn VMStore) -> Result, ) -> (T::Abi, Option) { + #[cfg(not(feature = "call-hook"))] + let _ = call_hooks; + // First wrap `f` in call hooks if that feature is enabled. This is used // as a "pretty far down in the stack" mechanism of ensuring that hooks // aren't forgotten. @@ -276,23 +286,26 @@ where // * host functions // * component versions of the above // - // This specifically is NOT handling libcalls where the result can't - // carry a result. This should be safe as anything which doesn't return - // a `Result` sort of has to be simple enough to not allow recursion so - // it's just a brief exit from the guest to the host. + // Infallible libcalls do not run these hooks. Internal initialization + // may also opt out to preserve the same observable boundaries as eager + // segment writes, while still catching errors and panics. // // Also note that this only happens with the `call-hook` feature because // this otherwise imposes a dynamic dispatch on the `store` trait object // which otherwise can't be optimized away. #[cfg(feature = "call-hook")] let f = move |store: &mut dyn VMStore| { - store.call_hook(crate::CallHook::CallingHost)?; + if call_hooks { + store.call_hook(crate::CallHook::CallingHost)?; + } let res = f(store); // Note that if this returns a trap then `ret` is discarded // entirely. - store.call_hook(crate::CallHook::ReturningFromHost)?; + if call_hooks { + store.call_hook(crate::CallHook::ReturningFromHost)?; + } res.map_err(|e| e.into()) }; diff --git a/tests/all/call_hook.rs b/tests/all/call_hook.rs index 63de3509b7ea..67988dfdbeeb 100644 --- a/tests/all/call_hook.rs +++ b/tests/all/call_hook.rs @@ -321,6 +321,26 @@ fn instantiate() -> Result<(), Error> { assert_eq!(store.data().calls_into_wasm, 1); assert_eq!(store.data().calls_into_host, 0); + let table = Table::new( + &mut store, + TableType::new(RefType::FUNCREF, 1, None)?, + Ref::Func(None), + )?; + let m = Module::new( + store.engine(), + r#"(module + (import "" "" (table 1 funcref)) + (func $f) + (elem (i32.const 0) func $f))"#, + )?; + // Internal segment initialization must not add a host transition, even + // when deferring function records requires a fallible runtime helper. + store.data_mut().trap_next_call_host = true; + Instance::new(&mut store, &m, &[table.into()])?; + assert!(matches!(table.get(&mut store, 0), Some(Ref::Func(Some(_))))); + assert_eq!(store.data().calls_into_wasm, 2); + assert_eq!(store.data().calls_into_host, 0); + Ok(()) } diff --git a/tests/all/table.rs b/tests/all/table.rs index b47670546ba2..207b2c4f5341 100644 --- a/tests/all/table.rs +++ b/tests/all/table.rs @@ -398,3 +398,167 @@ fn gc_store_with_table_initializers() -> Result<()> { Ok(()) } + +#[test] +#[cfg(all(feature = "cranelift", not(miri)))] +fn imported_function_segments_preserve_order_and_source_lifetimes() -> Result<()> { + let mut initialization_fuel = None; + for lazy in [false, true] { + let mut config = Config::new(); + config.table_lazy_init(lazy).consume_fuel(true); + let engine = Engine::new(&config)?; + let mut store = Store::new(&engine, ()); + store.set_fuel(100_000)?; + let table = Table::new( + &mut store, + TableType::new(RefType::FUNCREF, 6, None)?, + Ref::Func(None), + )?; + let mut instantiate = |offset: u32, value: i32| -> Result<()> { + let module = Module::new( + &engine, + format!( + r#"(module + (import "" "" (table 6 funcref)) + (global $value i32 (i32.const {value})) + (func $first (result i32) global.get $value) + (func $second (result i32) global.get $value i32.const 1 i32.add) + (elem (i32.const {offset}) func $first $first $second) + (elem (i32.const {}) func $second))"#, + offset + 1 + ), + )?; + // SAFETY: these bytes were just serialized by this Engine. This also + // exercises retained active-segment metadata after deserialization. + let module = unsafe { Module::deserialize(&engine, module.serialize()?)? }; + Instance::new(&mut store, &module, &[table.into()])?; + Ok(()) + }; + instantiate(0, 10)?; + instantiate(2, 20)?; + let remaining_fuel = store.get_fuel()?; + if let Some(expected) = initialization_fuel { + assert_eq!(remaining_fuel, expected); + } + initialization_fuel = Some(remaining_fuel); + // The caller holds no Instance handle for either contributing module. + table.set(&mut store, 1, Ref::Func(None))?; + Table::copy(&mut store, &table, 2, &table, 0, 3)?; + for (index, expected) in [Some(10), None, Some(10), None, Some(20), None] + .into_iter() + .enumerate() + { + match (table.get(&mut store, index as u64), expected) { + (Some(Ref::Func(Some(f))), Some(value)) => { + assert_eq!(f.typed::<(), i32>(&store)?.call(&mut store, ())?, value) + } + (Some(Ref::Func(None)), None) => {} + other => panic!("unexpected table element {index}: {other:?}"), + } + } + let failed = Module::new( + &engine, + r#"(module + (import "" "" (table 6 funcref)) + (func $f (result i32) i32.const 99) + (elem (i32.const 5) func $f) + (func $start unreachable) (start $start))"#, + )?; + assert_eq!( + Instance::new(&mut store, &failed, &[table.into()]) + .unwrap_err() + .downcast_ref::(), + Some(&Trap::UnreachableCodeReached) + ); + let Some(Ref::Func(Some(f))) = table.get(&mut store, 5) else { + panic!("missing failed-start contribution") + }; + assert_eq!(f.typed::<(), i32>(&store)?.call(&mut store, ())?, 99); + let out_of_bounds = Module::new( + &engine, + r#"(module + (import "" "" (table 6 funcref)) + (func $f (result i32) i32.const 101) + (elem (i32.const 0) func $f) + (elem (i32.const 6) func $f))"#, + )?; + assert_eq!( + Instance::new(&mut store, &out_of_bounds, &[table.into()]) + .unwrap_err() + .downcast_ref::(), + Some(&Trap::TableOutOfBounds) + ); + let Some(Ref::Func(Some(f))) = table.get(&mut store, 0) else { + panic!("missing earlier segment") + }; + assert_eq!(f.typed::<(), i32>(&store)?.call(&mut store, ())?, 101); + } + Ok(()) +} + +#[test] +#[cfg(all(feature = "cranelift", not(miri)))] +fn imported_function_segments_bound_history_and_preserve_explicit_writes() -> Result<()> { + let engine = Engine::default(); + let module = Module::new( + &engine, + r#"(module + (import "" "table" (table 2 funcref)) + (import "" "value" (global $value i32)) + (func $f (result i32) global.get $value) + (elem (i32.const 0) func $f))"#, + )?; + let mut store = Store::new(&engine, ()); + let table = Table::new( + &mut store, + TableType::new(RefType::FUNCREF, 2, None)?, + Ref::Func(None), + )?; + // Cross the bounded deferred-history limit while old sources remain alive. + for value in 0..132 { + let global = Global::new( + &mut store, + GlobalType::new(ValType::I32, Mutability::Const), + Val::I32(value), + )?; + Instance::new(&mut store, &module, &[table.into(), global.into()])?; + } + let Some(Ref::Func(Some(f))) = table.get(&mut store, 0) else { + panic!("missing function") + }; + assert_eq!(f.typed::<(), i32>(&store)?.call(&mut store, ())?, 131); + table.set(&mut store, 0, Ref::Func(None))?; + assert!(matches!(table.get(&mut store, 0), Some(Ref::Func(None)))); + Ok(()) +} + +#[test] +#[cfg(all(feature = "cranelift", feature = "pooling-allocator", not(miri)))] +fn imported_function_segments_do_not_survive_pooled_table_reuse() -> Result<()> { + let mut pooling = PoolingAllocationConfig::default(); + pooling + .total_core_instances(2) + .total_tables(2) + .max_tables_per_module(1) + .table_elements(6); + let mut config = Config::new(); + config.allocation_strategy(InstanceAllocationStrategy::Pooling(pooling)); + let engine = Engine::new(&config)?; + let owner = Module::new(&engine, "(module (table (export \"table\") 6 funcref))")?; + let source = Module::new( + &engine, + r#"(module + (import "" "" (table 6 funcref)) + (func $f (result i32) i32.const 42) + (elem (i32.const 0) func $f))"#, + )?; + for _ in 0..3 { + let mut store = Store::new(&engine, ()); + let owner = Instance::new(&mut store, &owner, &[])?; + let table = owner.get_table(&mut store, "table").unwrap(); + assert!(matches!(table.get(&mut store, 0), Some(Ref::Func(None)))); + Instance::new(&mut store, &source, &[table.into()])?; + // Drop the Store without materializing the deferred function. + } + Ok(()) +}