From 2de33782ec8c23612ad9bde958a1c37a2184bf00 Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 8 Apr 2026 12:55:14 -0500 Subject: [PATCH 1/2] Merge with_call_scope! into common_call --- crates/core/src/host/v8/mod.rs | 44 ++++++++++---------------- crates/core/src/host/v8/syscall/mod.rs | 2 +- crates/core/src/host/v8/syscall/v2.rs | 2 +- 3 files changed, 18 insertions(+), 30 deletions(-) diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 5dd5f7e2527..9fb6c92ff83 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -7,7 +7,7 @@ use self::ser::serialize_to_js; use self::string::{str_from_ident, IntoJsString}; use self::syscall::{ call_call_procedure, call_call_reducer, call_call_view, call_call_view_anon, call_describe_module, get_hooks, - get_registered_hooks, process_thrown_exception, resolve_sys_module, FnRet, HookFunctions, + process_thrown_exception, resolve_sys_module, FnRet, HookFunctions, }; use super::module_common::{build_common_module_from_raw, run_describer, ModuleCommon}; use super::module_host::{CallProcedureParams, CallReducerParams, ModuleInfo, ModuleWithInstance}; @@ -1653,16 +1653,6 @@ struct V8Instance<'a, 'scope, 'isolate> { args: &'a Global, } -macro_rules! with_call_scope { - ($scope:expr, |$call_scope:ident, $hooks:ident| $body:block) => {{ - // Open a fresh HandleScope for this invocation so call-local V8 handles - // are released when the reducer/view/procedure returns. - v8::scope!(let $call_scope, $scope); - let $hooks = get_registered_hooks($call_scope).expect("module hooks should be registered before invoking JS"); - $body - }}; -} - impl WasmInstance for V8Instance<'_, '_, '_> { fn extract_descriptions(&mut self) -> Result { extract_description(self.scope, self.hooks, self.replica_ctx) @@ -1681,26 +1671,22 @@ impl WasmInstance for V8Instance<'_, '_, '_> { } fn call_reducer(&mut self, op: ReducerOp<'_>, budget: FunctionBudget) -> ReducerExecuteResult { - with_call_scope!(self.scope, |scope, hooks| { - common_call(scope, &hooks, budget, op, |scope, op| { - let reducer_args_buf = Local::new(scope, self.args); - Ok(call_call_reducer(scope, &hooks, op, reducer_args_buf)?) - }) + common_call(self.scope, self.hooks, budget, op, |scope, op| { + let reducer_args_buf = Local::new(scope, self.args); + Ok(call_call_reducer(scope, self.hooks, op, reducer_args_buf)?) }) .map_result(|call_result| call_result.and_then(|res| res.map_err(ExecutionError::User))) } fn call_view(&mut self, op: ViewOp<'_>, budget: FunctionBudget) -> ViewExecuteResult { - with_call_scope!(self.scope, |scope, hooks| { - common_call(scope, &hooks, budget, op, |scope, op| call_call_view(scope, &hooks, op)) + common_call(self.scope, self.hooks, budget, op, |scope, op| { + call_call_view(scope, self.hooks, op) }) } fn call_view_anon(&mut self, op: AnonymousViewOp<'_>, budget: FunctionBudget) -> ViewExecuteResult { - with_call_scope!(self.scope, |scope, hooks| { - common_call(scope, &hooks, budget, op, |scope, op| { - call_call_view_anon(scope, &hooks, op) - }) + common_call(self.scope, self.hooks, budget, op, |scope, op| { + call_call_view_anon(scope, self.hooks, op) }) } @@ -1713,10 +1699,8 @@ impl WasmInstance for V8Instance<'_, '_, '_> { op: ProcedureOp, budget: FunctionBudget, ) -> (ProcedureExecuteResult, Option) { - let result = with_call_scope!(self.scope, |scope, hooks| { - common_call(scope, &hooks, budget, op, |scope, op| { - call_call_procedure(scope, &hooks, op) - }) + let result = common_call(self.scope, self.hooks, budget, op, |scope, op| { + call_call_procedure(scope, self.hooks, op) }) .map_result(|call_result| { call_result.map_err(|e| match e { @@ -1733,15 +1717,19 @@ impl WasmInstance for V8Instance<'_, '_, '_> { fn common_call<'scope, R, O, F>( scope: &mut PinScope<'scope, '_>, - hooks: &HookFunctions<'scope>, + hooks: &HookFunctions<'_>, budget: FunctionBudget, op: O, call: F, ) -> ExecutionResult where O: InstanceOp, - F: FnOnce(&mut PinTryCatch<'scope, '_, '_, '_>, O) -> Result>, + F: FnOnce(&mut PinTryCatch<'_, '_, '_, '_>, O) -> Result>, { + // Open a fresh HandleScope for this invocation so call-local V8 handles + // are released when the reducer/view/procedure returns. + v8::scope!(let scope, scope); + // TODO(v8): Start the budget timeout and long-running logger. let env = env_on_isolate_unwrap(scope); diff --git a/crates/core/src/host/v8/syscall/mod.rs b/crates/core/src/host/v8/syscall/mod.rs index ec4747bdce9..c0c619667a2 100644 --- a/crates/core/src/host/v8/syscall/mod.rs +++ b/crates/core/src/host/v8/syscall/mod.rs @@ -76,7 +76,7 @@ fn resolve_sys_module_inner<'scope>( /// This handles any (future) ABI version differences. pub(super) fn call_call_reducer<'scope>( scope: &mut PinTryCatch<'scope, '_, '_, '_>, - hooks: &HookFunctions<'scope>, + hooks: &HookFunctions<'_>, op: ReducerOp<'_>, reducer_args_buf: Local<'scope, ArrayBuffer>, ) -> ExcResult { diff --git a/crates/core/src/host/v8/syscall/v2.rs b/crates/core/src/host/v8/syscall/v2.rs index 3df67f547be..5be8526dd7a 100644 --- a/crates/core/src/host/v8/syscall/v2.rs +++ b/crates/core/src/host/v8/syscall/v2.rs @@ -417,7 +417,7 @@ fn hooks_symbol<'scope>(scope: &PinScope<'scope, '_>) -> Local<'scope, v8::Symbo /// Calls the `__call_reducer__` function `fun`. pub(super) fn call_call_reducer<'scope>( scope: &mut PinTryCatch<'scope, '_, '_, '_>, - hooks: &HookFunctions<'scope>, + hooks: &HookFunctions<'_>, op: ReducerOp<'_>, reducer_args_buf: Local<'scope, ArrayBuffer>, ) -> ExcResult { From 34be70ddfac54da49f30dbace9e6c6eb083b2902 Mon Sep 17 00:00:00 2001 From: Noa Date: Fri, 10 Apr 2026 12:55:52 -0500 Subject: [PATCH 2/2] Implement near-heap-limit termination --- crates/core/src/config.rs | 28 ++- crates/core/src/host/v8/error.rs | 67 ++++-- crates/core/src/host/v8/mod.rs | 227 ++++++++++++++------- crates/core/src/worker_metrics/mod.rs | 5 + crates/standalone/src/subcommands/start.rs | 2 +- 5 files changed, 227 insertions(+), 102 deletions(-) diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index beff6ac5867..be4c8629ff6 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -185,8 +185,12 @@ pub struct V8HeapPolicyConfig { pub heap_gc_trigger_fraction: f64, #[serde(default = "def_retire", deserialize_with = "de_fraction")] pub heap_retire_fraction: f64, - #[serde(default, rename = "heap-limit-mb", deserialize_with = "de_limit_mb")] - pub heap_limit_bytes: Option, + #[serde( + default = "def_heap_limit", + rename = "heap-limit-mb", + deserialize_with = "de_limit_mb" + )] + pub heap_limit_bytes: usize, } impl Default for V8HeapPolicyConfig { @@ -196,7 +200,7 @@ impl Default for V8HeapPolicyConfig { heap_check_time_interval: def_time_interval(), heap_gc_trigger_fraction: def_gc_trigger(), heap_retire_fraction: def_retire(), - heap_limit_bytes: None, + heap_limit_bytes: def_heap_limit(), } } } @@ -237,6 +241,12 @@ fn def_retire() -> f64 { 0.75 } +/// Default heap limit, in bytes +fn def_heap_limit() -> usize { + // 1 GiB + 1024 * 1024 * 1024 +} + fn de_nz_u64<'de, D>(deserializer: D) -> Result, D::Error> where D: serde::Deserializer<'de>, @@ -289,22 +299,20 @@ where } } -fn de_limit_mb<'de, D>(deserializer: D) -> Result, D::Error> +fn de_limit_mb<'de, D>(deserializer: D) -> Result where D: serde::Deserializer<'de>, { let value = u64::deserialize(deserializer)?; if value == 0 { - return Ok(None); + return Ok(def_heap_limit()); } let bytes = value .checked_mul(1024 * 1024) .ok_or_else(|| serde::de::Error::custom("heap-limit-mb is too large"))?; - usize::try_from(bytes) - .map(Some) - .map_err(|_| serde::de::Error::custom("heap-limit-mb does not fit in usize")) + usize::try_from(bytes).map_err(|_| serde::de::Error::custom("heap-limit-mb does not fit in usize")) } #[cfg(test)] @@ -420,7 +428,7 @@ mod tests { ); assert_eq!(config.v8_heap_policy.heap_gc_trigger_fraction, 0.67); assert_eq!(config.v8_heap_policy.heap_retire_fraction, 0.75); - assert_eq!(config.v8_heap_policy.heap_limit_bytes, None); + assert_eq!(config.v8_heap_policy.heap_limit_bytes, 1024 * 1024 * 1024); } #[test] @@ -443,6 +451,6 @@ mod tests { ); assert_eq!(config.v8_heap_policy.heap_gc_trigger_fraction, 0.6); assert_eq!(config.v8_heap_policy.heap_retire_fraction, 0.8); - assert_eq!(config.v8_heap_policy.heap_limit_bytes, Some(256 * 1024 * 1024)); + assert_eq!(config.v8_heap_policy.heap_limit_bytes, 256 * 1024 * 1024); } } diff --git a/crates/core/src/host/v8/error.rs b/crates/core/src/host/v8/error.rs index 451c33a72d7..53d9be0c50a 100644 --- a/crates/core/src/host/v8/error.rs +++ b/crates/core/src/host/v8/error.rs @@ -217,7 +217,7 @@ pub(crate) struct ExceptionThrown { impl ExceptionThrown { /// Turns a caught JS exception in `scope` into a [`JSError`]. - pub(crate) fn into_error(self, scope: &mut PinTryCatch) -> JsError { + pub(crate) fn into_error(self, scope: &mut PinTryCatch) -> Result { JsError::from_caught(scope) } } @@ -254,12 +254,15 @@ pub(super) enum ErrorOrException { Exception(Exc), } -impl ErrorOrException { - pub(super) fn map_exception(self, f: impl FnOnce(Exc) -> Exc2) -> ErrorOrException { - match self { +impl ErrorOrException { + pub(super) fn exc_into_error( + self, + scope: &mut PinTryCatch<'_, '_, '_, '_>, + ) -> Result, UnknownJsError> { + Ok(match self { ErrorOrException::Err(e) => ErrorOrException::Err(e), - ErrorOrException::Exception(exc) => ErrorOrException::Exception(f(exc)), - } + ErrorOrException::Exception(exc) => ErrorOrException::Exception(exc.into_error(scope)?), + }) } } @@ -275,6 +278,12 @@ impl From for ErrorOrException { } } +impl From for ErrorOrException { + fn from(e: JsError) -> Self { + Self::Exception(e) + } +} + impl From> for anyhow::Error { fn from(err: ErrorOrException) -> Self { match err { @@ -528,23 +537,41 @@ fn get_or_insert_slot(isolate: &mut v8::Isolate, default: impl FnOnc impl JsError { /// Turns a caught JS exception in `scope` into a [`JSError`]. - fn from_caught(scope: &mut PinTryCatch<'_, '_, '_, '_>) -> Self { - match scope.message() { - Some(message) => Self { - trace: message - .get_stack_trace(scope) - .map(|trace| JsStackTrace::from_trace(scope, trace)) - .unwrap_or_default(), - msg: message.get(scope).to_rust_string_lossy(scope), - }, - None => Self { - trace: JsStackTrace::default(), - msg: "unknown error".to_owned(), - }, + fn from_caught(scope: &mut PinTryCatch<'_, '_, '_, '_>) -> Result { + let message = scope.message().ok_or(UnknownJsError)?; + Ok(Self { + trace: message + .get_stack_trace(scope) + .map(|trace| JsStackTrace::from_trace(scope, trace)) + .unwrap_or_default(), + msg: message.get(scope).to_rust_string_lossy(scope), + }) + } +} + +pub(super) struct UnknownJsError; + +impl From for JsError { + fn from(_: UnknownJsError) -> Self { + Self { + trace: JsStackTrace::default(), + msg: "unknown error".to_owned(), } } } +impl From for ErrorOrException { + fn from(e: UnknownJsError) -> Self { + Self::Exception(e.into()) + } +} + +impl From for anyhow::Error { + fn from(e: UnknownJsError) -> Self { + JsError::from(e).into() + } +} + pub(super) fn log_traceback(replica_ctx: &ReplicaContext, func_type: &str, func: &str, e: &anyhow::Error) { log::info!("{func_type} \"{func}\" runtime error: {e:}"); if let Some(js_err) = e.downcast_ref::() { @@ -573,7 +600,7 @@ pub(super) fn catch_exception<'scope, T>( body: impl FnOnce(&mut PinTryCatch<'scope, '_, '_, '_>) -> Result>, ) -> Result> { tc_scope!(scope, scope); - body(scope).map_err(|e| e.map_exception(|exc| exc.into_error(scope))) + body(scope).map_err(|e| e.exc_into_error(scope).unwrap_or_else(Into::into)) } pub(super) type PinTryCatch<'scope, 'iso, 'x, 's> = PinnedRef<'x, TryCatch<'s, 'scope, HandleScope<'iso>>>; diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 9fb6c92ff83..8283aac686b 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -39,7 +39,7 @@ use futures::future::LocalBoxFuture; use futures::FutureExt; use itertools::Either; use parking_lot::RwLock; -use prometheus::IntGauge; +use prometheus::{IntCounter, IntGauge}; use spacetimedb_auth::identity::ConnectionAuthCtx; use spacetimedb_client_api_messages::energy::FunctionBudget; use spacetimedb_datastore::locking_tx_datastore::FuncCallType; @@ -50,6 +50,7 @@ use spacetimedb_schema::def::ModuleDef; use spacetimedb_schema::identifier::Identifier; use spacetimedb_table::static_assert_size; use std::cell::Cell; +use std::os::raw::c_void; use std::panic::AssertUnwindSafe; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, LazyLock}; @@ -1277,16 +1278,51 @@ fn startup_instance_worker<'scope>( /// Returns a new isolate. fn new_isolate(heap_policy: V8HeapPolicyConfig) -> OwnedIsolate { - let params = if let Some(heap_limit_bytes) = heap_policy.heap_limit_bytes { - v8::CreateParams::default().heap_limits(0, heap_limit_bytes) - } else { - v8::CreateParams::default() - }; + let params = v8::CreateParams::default().heap_limits(0, heap_policy.heap_limit_bytes); let mut isolate = Isolate::new(params); isolate.set_capture_stack_trace_for_uncaught_exceptions(true, 1024); isolate } +/// Run the closure `f` with `callback` set as `scope`'s near-heap-limit callback. +/// +/// Upon return, the callback will be unregistered and the heap limit will be set back down to as +/// close to `reset_heap_limit` as the runtime deems reasonable. +fn with_near_heap_limit_callback(scope: &mut I, reset_heap_limit: usize, mut callback: Cb, f: F) -> R +where + Cb: FnMut(usize, usize) -> usize, + I: AsMut, + F: FnOnce(&mut I) -> R, +{ + unsafe extern "C" fn callback_wrapper( + data: *mut c_void, + current_heap_limit: usize, + initial_heap_limit: usize, + ) -> usize + where + F: FnMut(usize, usize) -> usize, + { + let callback = data.cast::(); + unsafe { (*callback)(current_heap_limit, initial_heap_limit) } + } + + let data = std::ptr::from_mut(&mut callback).cast::(); + let raw_callback: v8::NearHeapLimitCallback = callback_wrapper::; + + scope.as_mut().add_near_heap_limit_callback(raw_callback, data); + + // Immediately set up a guard that will remove the callback when this scope exits, because + // `data` points to a stack-allocated object and it cannot be allowed to hang around after + // this stack frame exits. + let mut guard = scopeguard::guard(scope, |isolate| { + isolate + .as_mut() + .remove_near_heap_limit_callback(raw_callback, reset_heap_limit) + }); + + f(&mut guard) +} + /// Spawns an instance worker for `program` /// and returns on success the corresponding [`JsInstance`] /// that talks to the worker. @@ -1392,6 +1428,10 @@ async fn spawn_instance_worker( replica_ctx, hooks: &hooks, args: &args, + heap_limit_hit_metric: &WORKER_METRICS + .v8_heap_limit_hit + .with_label_values(&info.database_identity), + initial_heap_limit: heap_policy.heap_limit_bytes, }; if let Some(heap_metrics) = heap_metrics.as_mut() { let _initial_heap_stats = sample_heap_stats(inst.scope, heap_metrics); @@ -1651,6 +1691,9 @@ struct V8Instance<'a, 'scope, 'isolate> { replica_ctx: &'a Arc, hooks: &'a HookFunctions<'scope>, args: &'a Global, + /// Metric for the number of times the v8 heap limit has been hit. + heap_limit_hit_metric: &'a IntCounter, + initial_heap_limit: usize, } impl WasmInstance for V8Instance<'_, '_, '_> { @@ -1671,22 +1714,21 @@ impl WasmInstance for V8Instance<'_, '_, '_> { } fn call_reducer(&mut self, op: ReducerOp<'_>, budget: FunctionBudget) -> ReducerExecuteResult { - common_call(self.scope, self.hooks, budget, op, |scope, op| { - let reducer_args_buf = Local::new(scope, self.args); - Ok(call_call_reducer(scope, self.hooks, op, reducer_args_buf)?) + let args = self.args; + common_call(self, budget, op, |scope, hooks, op| { + let reducer_args_buf = Local::new(scope, args); + Ok(call_call_reducer(scope, hooks, op, reducer_args_buf)?) }) .map_result(|call_result| call_result.and_then(|res| res.map_err(ExecutionError::User))) } fn call_view(&mut self, op: ViewOp<'_>, budget: FunctionBudget) -> ViewExecuteResult { - common_call(self.scope, self.hooks, budget, op, |scope, op| { - call_call_view(scope, self.hooks, op) - }) + common_call(self, budget, op, |scope, hooks, op| call_call_view(scope, hooks, op)) } fn call_view_anon(&mut self, op: AnonymousViewOp<'_>, budget: FunctionBudget) -> ViewExecuteResult { - common_call(self.scope, self.hooks, budget, op, |scope, op| { - call_call_view_anon(scope, self.hooks, op) + common_call(self, budget, op, |scope, hooks, op| { + call_call_view_anon(scope, hooks, op) }) } @@ -1699,8 +1741,8 @@ impl WasmInstance for V8Instance<'_, '_, '_> { op: ProcedureOp, budget: FunctionBudget, ) -> (ProcedureExecuteResult, Option) { - let result = common_call(self.scope, self.hooks, budget, op, |scope, op| { - call_call_procedure(scope, self.hooks, op) + let result = common_call(self, budget, op, |scope, hooks, op| { + call_call_procedure(scope, hooks, op) }) .map_result(|call_result| { call_result.map_err(|e| match e { @@ -1715,77 +1757,120 @@ impl WasmInstance for V8Instance<'_, '_, '_> { } } -fn common_call<'scope, R, O, F>( - scope: &mut PinScope<'scope, '_>, - hooks: &HookFunctions<'_>, +fn common_call( + inst: &mut V8Instance<'_, '_, '_>, budget: FunctionBudget, op: O, call: F, ) -> ExecutionResult where O: InstanceOp, - F: FnOnce(&mut PinTryCatch<'_, '_, '_, '_>, O) -> Result>, + F: FnOnce(&mut PinTryCatch<'_, '_, '_, '_>, &HookFunctions<'_>, O) -> Result>, { - // Open a fresh HandleScope for this invocation so call-local V8 handles - // are released when the reducer/view/procedure returns. - v8::scope!(let scope, scope); + let scope = &mut *inst.scope; - // TODO(v8): Start the budget timeout and long-running logger. - let env = env_on_isolate_unwrap(scope); + let heap_limit_hit = Cell::new(0u32); + let terminated_because_of_heap_limit = Cell::new(false); - // Start the timer. - // We'd like this tightly around `call`. - env.start_funcall(op.name().clone(), op.timestamp(), op.call_type()); + let isolate_handle = scope.thread_safe_handle(); - // Wrap the call in `TryCatch`. - // - // `v8::tc_scope!` adds exception handling on top of the current scope; it - // does not create a new HandleScope. The fresh per-call HandleScope is - // opened by the caller before entering `common_call`. - v8::tc_scope!(let scope, scope); - let call_result = call(scope, op).map_err(|mut e| { - if let ErrorOrException::Exception(_) = e { - // If we're terminating execution, don't try to check `instanceof`. - if scope.can_continue() - && let Some(exc) = scope.exception() - { - match process_thrown_exception(scope, hooks, exc) { - Ok(Some(err)) => return err, - Ok(None) => {} - Err(exc) => e = ErrorOrException::Exception(exc), + let near_heap_limit_callback = |current_heap_limit: usize, _initial_heap_limit: usize| { + heap_limit_hit.update(|x| x + 1); + inst.heap_limit_hit_metric.inc(); + if !isolate_handle.is_execution_terminating() { + terminated_because_of_heap_limit.set(true); + isolate_handle.terminate_execution(); + } + current_heap_limit.saturating_mul(2) + }; + + with_near_heap_limit_callback(scope, inst.initial_heap_limit, near_heap_limit_callback, |scope| { + // Open a fresh HandleScope for this invocation so call-local V8 handles + // are released when the reducer/view/procedure returns. + v8::scope!(let scope, scope); + + // TODO(v8): Start the budget timeout and long-running logger. + let env = env_on_isolate_unwrap(scope); + + // Start the timer. + // We'd like this tightly around `call`. + env.start_funcall(op.name().clone(), op.timestamp(), op.call_type()); + + // Wrap the call in `TryCatch`. + // + // `v8::tc_scope!` adds exception handling on top of the current scope; it + // does not create a new HandleScope. The fresh per-call HandleScope is + // opened by the caller before entering `common_call`. + v8::tc_scope!(let scope, scope); + + let call_result = call(scope, inst.hooks, op).map_err(|mut e| { + if let ErrorOrException::Exception(_) = e { + // If we're terminating execution, don't try to check `instanceof`. + if scope.can_continue() + && let Some(exc) = scope.exception() + { + match process_thrown_exception(scope, inst.hooks, exc) { + Ok(Some(err)) => return err, + Ok(None) => {} + Err(exc) => e = ErrorOrException::Exception(exc), + } } } - } - let e = e.map_exception(|exc| exc.into_error(scope)).into(); - if scope.can_continue() { - // We can continue. - ExecutionError::Recoverable(e) - } else if scope.has_terminated() { - // We can continue if we do `Isolate::cancel_terminate_execution`. - scope.cancel_terminate_execution(); - ExecutionError::Recoverable(e) - } else { - // We cannot continue. - ExecutionError::Trap(e) - } - }); - // Finish timings. - let timings = env_on_isolate_unwrap(scope).finish_funcall(); + let e = e.exc_into_error(scope).map(anyhow::Error::from); + if scope.can_continue() { + // We can continue. + ExecutionError::Recoverable(e.unwrap_or_else(Into::into)) + } else if scope.has_terminated() { + // We can continue if we do `Isolate::cancel_terminate_execution`. + scope.cancel_terminate_execution(); + let e = e.unwrap_or_else(|unknown| { + if terminated_because_of_heap_limit.get() { + anyhow::Error::msg("JavaScript module exceeded memory limit") + } else { + // TODO(noa): check for other causes of termination and give a better error message + unknown.into() + } + }); + ExecutionError::Recoverable(e) + } else { + // We cannot continue. + ExecutionError::Trap(e.unwrap_or_else(Into::into)) + } + }); - // Derive energy stats. - let energy = energy_from_elapsed(budget, timings.total_duration); + let env = env_on_isolate_unwrap(scope); - // Fetch the currently used heap size in V8. - // The used size is ostensibly fairer than the total size. - let memory_allocation = scope.get_heap_statistics().used_heap_size(); + // Finish timings. + let timings = env.finish_funcall(); - let stats = ExecutionStats { - energy, - timings, - memory_allocation, - }; - ExecutionResult { stats, call_result } + // Derive energy stats. + let energy = energy_from_elapsed(budget, timings.total_duration); + + let database_identity = *env.instance_env.database_identity(); + + // Fetch the currently used heap size in V8. + + // The used size is ostensibly fairer than the total size. + let heap_stats = scope.get_heap_statistics(); + let memory_allocation = heap_stats.used_heap_size(); + + if heap_limit_hit.get() > 1 { + tracing::warn!( + %database_identity, + current_heap_limit = heap_stats.heap_size_limit(), + used_heap_size = memory_allocation, + "Module hit heap limit multiple times in single call, even after doubling!", + ) + } + + let stats = ExecutionStats { + energy, + timings, + memory_allocation, + }; + ExecutionResult { stats, call_result } + }) } /// Extracts the raw module def by running the registered `__describe_module__` hook. diff --git a/crates/core/src/worker_metrics/mod.rs b/crates/core/src/worker_metrics/mod.rs index 7f5492a7057..62d3c453533 100644 --- a/crates/core/src/worker_metrics/mod.rs +++ b/crates/core/src/worker_metrics/mod.rs @@ -312,6 +312,11 @@ metrics_group!( #[labels(database_identity: Identity, worker_kind: str)] pub v8_heap_size_limit_bytes: IntGaugeVec, + #[name = spacetime_worker_v8_heap_limit_hit] + #[help = "The number of times the V8 heap size limit for a has been hit"] + #[labels(database_identity: Identity)] + pub v8_heap_limit_hit: IntCounterVec, + #[name = spacetime_worker_v8_instance_lane_queue_length] #[help = "The number of queued requests waiting for a database's JS instance lane worker"] #[labels(database_identity: Identity)] diff --git a/crates/standalone/src/subcommands/start.rs b/crates/standalone/src/subcommands/start.rs index a54e3cb39e9..42a04d1e2e6 100644 --- a/crates/standalone/src/subcommands/start.rs +++ b/crates/standalone/src/subcommands/start.rs @@ -533,7 +533,7 @@ mod tests { ); assert_eq!(config.common.v8_heap_policy.heap_gc_trigger_fraction, 0.6); assert_eq!(config.common.v8_heap_policy.heap_retire_fraction, 0.8); - assert_eq!(config.common.v8_heap_policy.heap_limit_bytes, Some(128 * 1024 * 1024)); + assert_eq!(config.common.v8_heap_policy.heap_limit_bytes, 128 * 1024 * 1024); assert_eq!( config.websocket,