diff --git a/Cargo.lock b/Cargo.lock index a4a6d170..6fab69af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -89,6 +89,15 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -1047,6 +1056,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "1.0.0" @@ -2423,6 +2438,7 @@ dependencies = [ "alloy-hardforks 0.4.7", "axum 0.7.9", "backon", + "criterion", "eyre", "futures-util", "git-version", @@ -2563,6 +2579,12 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "castaway" version = "0.2.4" @@ -2625,6 +2647,33 @@ dependencies = [ "windows-link", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cipher" version = "0.4.4" @@ -2943,6 +2992,42 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools 0.13.0", + "num-traits", + "oorandom", + "page_size", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "tokio", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +dependencies = [ + "cast", + "itertools 0.13.0", +] + [[package]] name = "critical-section" version = "1.2.0" @@ -4334,6 +4419,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -6206,6 +6302,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "op-alloy" version = "0.23.1" @@ -6792,6 +6894,34 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "polyval" version = "0.6.2" @@ -11636,6 +11766,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.10.0" diff --git a/Cargo.toml b/Cargo.toml index f66517f9..3ab98973 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,8 +63,14 @@ url = "2.5.4" [dev-dependencies] alloy-hardforks = "0.4.0" alloy-chains = "0.2" +criterion = { version = "0.8.2", features = ["async_tokio"] } signet-bundle = "0.16.0-rc.11" +[[bench]] +name = "sim" +path = "benches/sim/main.rs" +harness = false + # comment / uncomment for local dev # [patch.crates-io] # signet-constants = { path = "../signet-sdk/crates/constants" } diff --git a/benches/sim/bundles.rs b/benches/sim/bundles.rs new file mode 100644 index 00000000..e065126c --- /dev/null +++ b/benches/sim/bundles.rs @@ -0,0 +1,111 @@ +//! Benchmarks for bundle simulation: varying counts, DB latency, and concurrency. + +use crate::fixture::{Fixture, LATENCIES, set_up_bundle_sim, set_up_sender_distribution_sim}; +use builder::test_utils::setup_test_config; +use criterion::{BatchSize, Bencher, BenchmarkId, Criterion}; +use std::time::Duration; + +pub fn bench_bundle_counts(criterion: &mut Criterion) { + const LATENCY: Duration = Duration::ZERO; + const CONCURRENCY: usize = 4; + + setup_test_config(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + + let mut group = criterion.benchmark_group("sim_varying_simple_bundle_counts_no_db_latency"); + group.sample_size(10); + + for bundle_count in [1, 10, 100, 1000] { + group.bench_with_input( + BenchmarkId::new("bundle_count", bundle_count), + &bundle_count, + |bench: &mut Bencher, &bundle_count| { + bench.to_async(&runtime).iter_batched( + || set_up_bundle_sim(bundle_count, LATENCY, CONCURRENCY), + Fixture::run, + BatchSize::PerIteration, + ); + }, + ); + } + group.finish(); +} + +pub fn bench_bundle_db_latency(criterion: &mut Criterion) { + const BUNDLE_COUNT: usize = 10; + const CONCURRENCY: usize = 4; + + setup_test_config(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + + let mut group = + criterion.benchmark_group(format!("sim_{BUNDLE_COUNT}_simple_bundles_varying_db_latency")); + group.sample_size(10); + + for (label, latency) in LATENCIES { + group.bench_with_input(BenchmarkId::new("latency", label), &latency, |bench, &lat| { + bench.to_async(&runtime).iter_batched( + || set_up_bundle_sim(BUNDLE_COUNT, lat, CONCURRENCY), + Fixture::run, + BatchSize::PerIteration, + ); + }); + } + group.finish(); +} + +pub fn bench_bundle_concurrency(criterion: &mut Criterion) { + const BUNDLE_COUNT: usize = 100; + const LATENCY: Duration = Duration::ZERO; + + setup_test_config(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + + let mut group = criterion.benchmark_group(format!( + "sim_{BUNDLE_COUNT}_simple_bundles_no_db_latency_varying_concurrency" + )); + group.sample_size(10); + + for concurrency in [1, 2, 4, 8, 16] { + group.bench_with_input( + BenchmarkId::new("thread_count", concurrency), + &concurrency, + |bench, &conc| { + bench.to_async(&runtime).iter_batched( + || set_up_bundle_sim(BUNDLE_COUNT, LATENCY, conc), + Fixture::run, + BatchSize::PerIteration, + ); + }, + ); + } + group.finish(); +} + +pub fn bench_bundle_sender_distribution(criterion: &mut Criterion) { + const BUNDLE_COUNT: usize = 100; + const CONCURRENCY: usize = 4; + + setup_test_config(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + + let mut group = criterion.benchmark_group(format!( + "sim_{BUNDLE_COUNT}_simple_bundles_no_db_latency_varying_sender_count" + )); + group.sample_size(10); + + for num_senders in [1, 5, 20, 100] { + group.bench_with_input( + BenchmarkId::new("sender_count", num_senders), + &num_senders, + |bench, &senders| { + bench.to_async(&runtime).iter_batched( + || set_up_sender_distribution_sim(BUNDLE_COUNT, senders, CONCURRENCY), + Fixture::run, + BatchSize::PerIteration, + ); + }, + ); + } + group.finish(); +} diff --git a/benches/sim/complex_bundles.rs b/benches/sim/complex_bundles.rs new file mode 100644 index 00000000..24ba2653 --- /dev/null +++ b/benches/sim/complex_bundles.rs @@ -0,0 +1,61 @@ +//! Benchmarks for complex bundle shapes: multi-tx bundles and bundles with host transactions. + +use crate::fixture::{Fixture, set_up_host_tx_bundle_sim, set_up_multi_tx_bundle_sim}; +use builder::test_utils::setup_test_config; +use criterion::{BatchSize, BenchmarkId, Criterion}; + +pub fn bench_multi_tx_bundles(criterion: &mut Criterion) { + const BUNDLE_COUNT: usize = 100; + const CONCURRENCY: usize = 4; + + setup_test_config(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + + let mut group = criterion.benchmark_group(format!( + "sim_{BUNDLE_COUNT}_bundles_no_db_latency_varying_txs_per_bundle" + )); + group.sample_size(10); + + for txs_per_bundle in [1, 3, 5, 10] { + group.bench_with_input( + BenchmarkId::new("txs_per_bundle", txs_per_bundle), + &txs_per_bundle, + |bench, &tpb| { + bench.to_async(&runtime).iter_batched( + || set_up_multi_tx_bundle_sim(BUNDLE_COUNT, tpb, CONCURRENCY), + Fixture::run, + BatchSize::PerIteration, + ); + }, + ); + } + group.finish(); +} + +pub fn bench_host_tx_bundles(criterion: &mut Criterion) { + const BUNDLE_COUNT: usize = 100; + const CONCURRENCY: usize = 4; + + setup_test_config(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + + let mut group = criterion.benchmark_group(format!( + "sim_{BUNDLE_COUNT}_bundles_no_db_latency_varying_host_txs_per_bundle" + )); + group.sample_size(10); + + for host_txs in [0, 1, 3, 5] { + group.bench_with_input( + BenchmarkId::new("host_txs_per_bundle", host_txs), + &host_txs, + |bench, &htpb| { + bench.to_async(&runtime).iter_batched( + || set_up_host_tx_bundle_sim(BUNDLE_COUNT, htpb, CONCURRENCY), + Fixture::run, + BatchSize::PerIteration, + ); + }, + ); + } + group.finish(); +} diff --git a/benches/sim/failed_bundles.rs b/benches/sim/failed_bundles.rs new file mode 100644 index 00000000..324adfac --- /dev/null +++ b/benches/sim/failed_bundles.rs @@ -0,0 +1,53 @@ +//! Benchmarks for bundles that fail: preflight validation failures and EVM execution reverts. + +use crate::fixture::{ + Fixture, LATENCIES, set_up_execution_failure_sim, set_up_preflight_failure_sim, +}; +use builder::test_utils::setup_test_config; +use criterion::{BatchSize, BenchmarkId, Criterion}; + +pub fn bench_preflight_failure(criterion: &mut Criterion) { + const COUNT: usize = 10; + const CONCURRENCY: usize = 4; + + setup_test_config(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + + let mut group = + criterion.benchmark_group(format!("sim_{COUNT}_simple_bundles_all_failing_preflight")); + group.sample_size(10); + + for (label, latency) in LATENCIES { + group.bench_with_input(BenchmarkId::new("latency", label), &latency, |bench, &lat| { + bench.to_async(&runtime).iter_batched( + || set_up_preflight_failure_sim(COUNT, lat, CONCURRENCY), + Fixture::run, + BatchSize::PerIteration, + ); + }); + } + group.finish(); +} + +pub fn bench_execution_failure(criterion: &mut Criterion) { + const COUNT: usize = 10; + const CONCURRENCY: usize = 4; + + setup_test_config(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + + let mut group = + criterion.benchmark_group(format!("sim_{COUNT}_simple_bundles_all_failing_execution")); + group.sample_size(10); + + for (label, latency) in LATENCIES { + group.bench_with_input(BenchmarkId::new("latency", label), &latency, |bench, &lat| { + bench.to_async(&runtime).iter_batched( + || set_up_execution_failure_sim(COUNT, lat, CONCURRENCY), + Fixture::run, + BatchSize::PerIteration, + ); + }); + } + group.finish(); +} diff --git a/benches/sim/fixture.rs b/benches/sim/fixture.rs new file mode 100644 index 00000000..472dce26 --- /dev/null +++ b/benches/sim/fixture.rs @@ -0,0 +1,429 @@ +//! Shared fixtures and setup functions for simulation benchmarks. + +use alloy::{ + primitives::{Address, Bytes, U256}, + serde::OtherFields, + signers::local::PrivateKeySigner, +}; +use builder::test_utils::{ + DEFAULT_BALANCE, DEFAULT_BASEFEE, TestDb, TestDbBuilder, TestHostEnv, TestRollupEnv, + TestSimEnvBuilder, TestStateSource, create_call_tx, create_transfer_tx, + scenarios_test_block_env, +}; +use signet_bundle::RecoveredBundle; +use signet_sim::{SharedSimEnv, SimCache}; +use std::time::Duration; +use tokio::time::Instant; +use trevm::revm::bytecode::Bytecode; +use trevm::revm::inspector::NoOpInspector; + +pub const BLOCK_NUMBER: u64 = 100; +pub const BLOCK_TIMESTAMP: u64 = 1_700_000_000; +pub const RU_CHAIN_ID: u64 = 88888; +pub const DEFAULT_PRIORITY_FEE: u128 = 10_000_000_000; +pub const MAX_GAS: u64 = 3_000_000_000; +pub const MAX_HOST_GAS: u64 = 24_000_000; +pub const REVERTING_CONTRACT: Address = Address::repeat_byte(0xBB); +/// Latency values based on real-world RPC provider benchmarks: +/// - 0ms: baseline, no network simulation +/// - 50ms: good provider, same-region (p50 for top providers) +/// - 200ms: cross-region or average provider +pub const LATENCIES: [(&str, Duration); 3] = [ + ("0ms", Duration::ZERO), + ("50ms", Duration::from_millis(50)), + ("200ms", Duration::from_millis(200)), +]; + +/// Type alias for the shared simulation environment used in benchmarks. +pub type BenchSimEnv = SharedSimEnv; + +/// Everything needed to run a benchmark iteration: the simulation environment +/// plus state sources for preflight validity checks. +pub struct Fixture { + env: BenchSimEnv, + rollup_source: TestStateSource, + host_source: TestStateSource, +} + +impl Fixture { + fn new(envs: Envs, concurrency: usize, cache: SimCache) -> Self { + let Envs { rollup_env, host_env, rollup_source, host_source } = envs; + // Set a deadline far in the future so that `sim_round()` never short-circuits. + let finish_by = Instant::now() + Duration::from_secs(3600); + let env = SharedSimEnv::new(rollup_env, host_env, finish_by, concurrency, cache); + Fixture { env, rollup_source, host_source } + } + + /// Drain all items from the sim env via repeated `sim_round()` calls. + pub async fn run(self) { + let Fixture { mut env, rollup_source, host_source } = self; + while env.sim_round(MAX_GAS, MAX_HOST_GAS, &rollup_source, &host_source).await.is_some() {} + } +} + +struct Envs { + rollup_env: TestRollupEnv, + host_env: TestHostEnv, + rollup_source: TestStateSource, + host_source: TestStateSource, +} + +impl Envs { + /// Build simulation environments and state sources from a shared database. + fn new(db: TestDb) -> Self { + let block_env = + scenarios_test_block_env(BLOCK_NUMBER, DEFAULT_BASEFEE, BLOCK_TIMESTAMP, MAX_GAS); + + let sim_env = TestSimEnvBuilder::new() + .with_rollup_db(db.clone()) + .with_host_db(db.clone()) + .with_block_env(block_env); + + let rollup_source = TestStateSource::new(db.clone()); + let host_source = TestStateSource::new(db); + let (rollup_env, host_env) = sim_env.build(); + Envs { rollup_env, host_env, rollup_source, host_source } + } +} + +/// Generate `count` random funded signers. +fn generate_signers(count: usize) -> Vec { + (0..count).map(|_| PrivateKeySigner::random()).collect() +} + +/// Create a `TestDbBuilder` with accounts funded from the given signers. +fn fund_accounts(signers: &[PrivateKeySigner]) -> TestDbBuilder { + let balance = U256::from(DEFAULT_BALANCE); + let mut builder = TestDbBuilder::new(); + for signer in signers { + builder = builder.with_account(signer.address(), balance, 0); + } + builder +} + +/// Create a `RecoveredBundle` with one transfer transaction. +fn make_bundle( + signer: &PrivateKeySigner, + to: Address, + nonce: u64, + uuid: String, + max_priority_fee: u128, +) -> RecoveredBundle { + let tx = + create_transfer_tx(signer, to, U256::from(1_000u64), nonce, RU_CHAIN_ID, max_priority_fee) + .unwrap(); + + RecoveredBundle::new_unchecked( + vec![tx], + vec![], + BLOCK_NUMBER, + Some(BLOCK_TIMESTAMP - 100), + Some(BLOCK_TIMESTAMP + 100), + vec![], + Some(uuid), + vec![], + None, + None, + vec![], + OtherFields::default(), + ) +} + +/// Build a [`SharedSimEnv`] and state sources with bundles in the cache. +/// +/// All databases use the same latency to model production RPC round-trips +/// (both simulation environments and preflight state sources hit the RPC). +pub fn set_up_bundle_sim(count: usize, latency: Duration, concurrency: usize) -> Fixture { + let signers = generate_signers(count); + let db = fund_accounts(&signers).with_latency(latency).build(); + let envs = Envs::new(db); + + let cache = SimCache::with_capacity(count); + let recipient = Address::repeat_byte(0xAA); + let bundles: Vec = signers + .iter() + .enumerate() + .map(|(idx, signer)| { + make_bundle(signer, recipient, 0, format!("bench-{idx}"), DEFAULT_PRIORITY_FEE) + }) + .collect(); + cache.add_bundles(bundles, DEFAULT_BASEFEE); + + Fixture::new(envs, concurrency, cache) +} + +/// Build a [`SharedSimEnv`] and state sources with standalone txs in the cache. +pub fn set_up_tx_sim(count: usize, latency: Duration, concurrency: usize) -> Fixture { + let signers = generate_signers(count); + let db = fund_accounts(&signers).with_latency(latency).build(); + let envs = Envs::new(db); + + let cache = SimCache::with_capacity(count); + let recipient = Address::repeat_byte(0xAA); + for signer in &signers { + let tx = create_transfer_tx( + signer, + recipient, + U256::from(1_000u64), + 0, + RU_CHAIN_ID, + DEFAULT_PRIORITY_FEE, + ) + .unwrap(); + cache.add_tx(tx, DEFAULT_BASEFEE); + } + + Fixture::new(envs, concurrency, cache) +} + +/// Build a [`SharedSimEnv`] with bundles that all fail preflight validation. +/// +/// Accounts are funded with nonce 1, but all bundles use nonce 0. Since +/// `state_nonce > tx_nonce`, every item is marked `Never` and removed from the +/// cache on the first `sim_round()` call. +pub fn set_up_preflight_failure_sim( + count: usize, + latency: Duration, + concurrency: usize, +) -> Fixture { + let signers = generate_signers(count); + + let balance = U256::from(DEFAULT_BALANCE); + let mut db_builder = TestDbBuilder::new(); + for signer in &signers { + db_builder = db_builder.with_account(signer.address(), balance, 1); + } + let db = db_builder.with_latency(latency).build(); + let envs = Envs::new(db); + + let cache = SimCache::with_capacity(count); + let recipient = Address::repeat_byte(0xAA); + let bundles: Vec = signers + .iter() + .enumerate() + .map(|(idx, signer)| { + make_bundle(signer, recipient, 0, format!("bench-{idx}"), DEFAULT_PRIORITY_FEE) + }) + .collect(); + cache.add_bundles(bundles, DEFAULT_BASEFEE); + + Fixture::new(envs, concurrency, cache) +} + +/// Build a [`SharedSimEnv`] with bundles that pass preflight but revert during +/// EVM execution. +/// +/// A contract at [`REVERTING_CONTRACT`] contains `PUSH0 PUSH0 REVERT` (always +/// reverts with empty data). All bundles call this contract, so they pass +/// preflight (valid nonce + sufficient balance) but fail during simulation. +pub fn set_up_execution_failure_sim( + count: usize, + latency: Duration, + concurrency: usize, +) -> Fixture { + let signers = generate_signers(count); + + // PUSH0 PUSH0 REVERT — always reverts with empty returndata. + let revert_bytecode = Bytecode::new_raw(Bytes::from_static(&[0x5F, 0x5F, 0xFD])); + + let balance = U256::from(DEFAULT_BALANCE); + let mut db_builder = TestDbBuilder::new().with_contract(REVERTING_CONTRACT, revert_bytecode); + for signer in &signers { + db_builder = db_builder.with_account(signer.address(), balance, 0); + } + let db = db_builder.with_latency(latency).build(); + let envs = Envs::new(db); + + let cache = SimCache::with_capacity(count); + let bundles: Vec = signers + .iter() + .enumerate() + .map(|(idx, signer)| { + let tx = create_call_tx( + signer, + REVERTING_CONTRACT, + Bytes::new(), + U256::ZERO, + 0, + RU_CHAIN_ID, + 50_000, + DEFAULT_PRIORITY_FEE, + ) + .unwrap(); + RecoveredBundle::new_unchecked( + vec![tx], + vec![], + BLOCK_NUMBER, + Some(BLOCK_TIMESTAMP - 100), + Some(BLOCK_TIMESTAMP + 100), + vec![], + Some(format!("bench-{idx}")), + vec![], + None, + None, + vec![], + OtherFields::default(), + ) + }) + .collect(); + cache.add_bundles(bundles, DEFAULT_BASEFEE); + + Fixture::new(envs, concurrency, cache) +} + +/// Build a [`SharedSimEnv`] with `bundle_count` bundles spread across `num_senders` signers. +/// +/// Each signer sends `bundle_count / num_senders` bundles with incrementing nonces. +pub fn set_up_sender_distribution_sim( + bundle_count: usize, + num_senders: usize, + concurrency: usize, +) -> Fixture { + let signers = generate_signers(num_senders); + let bundles_per_sender = bundle_count / num_senders; + + let balance = U256::from(DEFAULT_BALANCE); + let mut db_builder = TestDbBuilder::new(); + for signer in &signers { + db_builder = db_builder.with_account(signer.address(), balance, 0); + } + let db = db_builder.build(); + let envs = Envs::new(db); + + let cache = SimCache::with_capacity(bundle_count); + let recipient = Address::repeat_byte(0xAA); + let bundles: Vec = signers + .iter() + .enumerate() + .flat_map(|(sender_idx, signer)| { + (0..bundles_per_sender).map(move |nonce| { + make_bundle( + signer, + recipient, + nonce as u64, + format!("bench-{sender_idx}-{nonce}"), + DEFAULT_PRIORITY_FEE, + ) + }) + }) + .collect(); + cache.add_bundles(bundles, DEFAULT_BASEFEE); + + Fixture::new(envs, concurrency, cache) +} + +/// Build a [`SharedSimEnv`] with bundles containing multiple rollup transactions each. +/// +/// Each bundle has `txs_per_bundle` transfer transactions from the same sender with +/// incrementing nonces. This measures the cost of simulating larger bundles where +/// all transactions must succeed atomically. +pub fn set_up_multi_tx_bundle_sim( + bundle_count: usize, + txs_per_bundle: usize, + concurrency: usize, +) -> Fixture { + let signers = generate_signers(bundle_count); + let db = fund_accounts(&signers).build(); + let envs = Envs::new(db); + + let cache = SimCache::with_capacity(bundle_count); + let recipient = Address::repeat_byte(0xAA); + let bundles: Vec = signers + .iter() + .enumerate() + .map(|(idx, signer)| { + let txs: Vec<_> = (0..txs_per_bundle) + .map(|nonce| { + create_transfer_tx( + signer, + recipient, + U256::from(1_000u64), + nonce as u64, + RU_CHAIN_ID, + DEFAULT_PRIORITY_FEE, + ) + .unwrap() + }) + .collect(); + RecoveredBundle::new_unchecked( + txs, + vec![], + BLOCK_NUMBER, + Some(BLOCK_TIMESTAMP - 100), + Some(BLOCK_TIMESTAMP + 100), + vec![], + Some(format!("bench-{idx}")), + vec![], + None, + None, + vec![], + OtherFields::default(), + ) + }) + .collect(); + cache.add_bundles(bundles, DEFAULT_BASEFEE); + + Fixture::new(envs, concurrency, cache) +} + +/// Build a [`SharedSimEnv`] with bundles that include host-chain transactions. +/// +/// Each bundle has one rollup transfer plus `host_txs_per_bundle` host-chain transfers. +/// This measures the overhead of bundles that carry cross-chain transactions. +pub fn set_up_host_tx_bundle_sim( + bundle_count: usize, + host_txs_per_bundle: usize, + concurrency: usize, +) -> Fixture { + let signers = generate_signers(bundle_count); + let db = fund_accounts(&signers).build(); + let envs = Envs::new(db); + + let cache = SimCache::with_capacity(bundle_count); + let recipient = Address::repeat_byte(0xAA); + let bundles: Vec = signers + .iter() + .enumerate() + .map(|(idx, signer)| { + let rollup_tx = create_transfer_tx( + signer, + recipient, + U256::from(1_000u64), + 0, + RU_CHAIN_ID, + DEFAULT_PRIORITY_FEE, + ) + .unwrap(); + let host_txs: Vec<_> = (0..host_txs_per_bundle) + .map(|nonce| { + create_transfer_tx( + signer, + recipient, + U256::from(1_000u64), + (nonce + 1) as u64, + RU_CHAIN_ID, + DEFAULT_PRIORITY_FEE, + ) + .unwrap() + }) + .collect(); + RecoveredBundle::new_unchecked( + vec![rollup_tx], + host_txs, + BLOCK_NUMBER, + Some(BLOCK_TIMESTAMP - 100), + Some(BLOCK_TIMESTAMP + 100), + vec![], + Some(format!("bench-{idx}")), + vec![], + None, + None, + vec![], + OtherFields::default(), + ) + }) + .collect(); + cache.add_bundles(bundles, DEFAULT_BASEFEE); + + Fixture::new(envs, concurrency, cache) +} diff --git a/benches/sim/main.rs b/benches/sim/main.rs new file mode 100644 index 00000000..01aba87f --- /dev/null +++ b/benches/sim/main.rs @@ -0,0 +1,22 @@ +mod bundles; +mod complex_bundles; +mod failed_bundles; +mod fixture; +mod txs; + +use criterion::{criterion_group, criterion_main}; + +criterion_group!( + benches, + bundles::bench_bundle_counts, + bundles::bench_bundle_db_latency, + bundles::bench_bundle_concurrency, + bundles::bench_bundle_sender_distribution, + complex_bundles::bench_multi_tx_bundles, + complex_bundles::bench_host_tx_bundles, + txs::bench_tx_counts, + txs::bench_tx_db_latency, + failed_bundles::bench_preflight_failure, + failed_bundles::bench_execution_failure, +); +criterion_main!(benches); diff --git a/benches/sim/txs.rs b/benches/sim/txs.rs new file mode 100644 index 00000000..d124818c --- /dev/null +++ b/benches/sim/txs.rs @@ -0,0 +1,55 @@ +//! Benchmarks for standalone transaction simulation: varying counts and DB latency. + +use crate::fixture::{Fixture, LATENCIES, set_up_tx_sim}; +use builder::test_utils::setup_test_config; +use criterion::{BatchSize, Bencher, BenchmarkId, Criterion}; +use std::time::Duration; + +pub fn bench_tx_counts(criterion: &mut Criterion) { + const LATENCY: Duration = Duration::ZERO; + const CONCURRENCY: usize = 4; + + setup_test_config(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + + let mut group = criterion.benchmark_group("sim_varying_tx_counts_with_no_db_latency"); + group.sample_size(10); + + for tx_count in [1, 10, 100, 1000] { + group.bench_with_input( + BenchmarkId::new("tx_count", tx_count), + &tx_count, + |bench: &mut Bencher, &tx_count| { + bench.to_async(&runtime).iter_batched( + || set_up_tx_sim(tx_count, LATENCY, CONCURRENCY), + Fixture::run, + BatchSize::PerIteration, + ); + }, + ); + } + group.finish(); +} + +pub fn bench_tx_db_latency(criterion: &mut Criterion) { + const TX_COUNT: usize = 10; + const CONCURRENCY: usize = 4; + + setup_test_config(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + + let mut group = + criterion.benchmark_group(format!("sim_{TX_COUNT}_txs_with_varying_db_latency")); + group.sample_size(10); + + for (label, latency) in LATENCIES { + group.bench_with_input(BenchmarkId::new("latency", label), &latency, |bench, &lat| { + bench.to_async(&runtime).iter_batched( + || set_up_tx_sim(TX_COUNT, lat, CONCURRENCY), + Fixture::run, + BatchSize::PerIteration, + ); + }); + } + group.finish(); +} diff --git a/src/test_utils/db.rs b/src/test_utils/db.rs index 98f3332e..9b15bb35 100644 --- a/src/test_utils/db.rs +++ b/src/test_utils/db.rs @@ -4,17 +4,20 @@ use alloy::primitives::{Address, B256, U256}; use signet_sim::{AcctInfo, StateSource}; +use std::time::Duration; use trevm::revm::{ + bytecode::Bytecode, database::{CacheDB, EmptyDB}, database_interface::DatabaseRef, + primitives::{StorageKey, StorageValue}, state::AccountInfo, }; -/// In-memory database for testing (no network access required). -/// This is a type alias for revm's `CacheDB`, which stores all -/// blockchain state in memory. It implements `DatabaseRef` and can be used -/// with `RollupEnv` and `HostEnv` for offline simulation testing. -pub type TestDb = CacheDB; +/// Mirrors the production `CacheDB` stack: the outer [`CacheDB`] starts +/// empty and caches on the mutable `Database` path, while the inner [`LatencyDb`] +/// holds all state in-memory and applies configurable sleep on every read to +/// simulate RPC round-trip cost. +pub type TestDb = CacheDB; /// A [`StateSource`] for testing backed by an in-memory [`TestDb`]. /// Returns actual account info (nonce, balance) from the database, @@ -51,7 +54,7 @@ impl StateSource for TestStateSource { /// before running simulations. #[derive(Debug)] pub struct TestDbBuilder { - db: TestDb, + latency_db: LatencyDb, } impl Default for TestDbBuilder { @@ -63,7 +66,7 @@ impl Default for TestDbBuilder { impl TestDbBuilder { /// Create a new empty test database builder. pub fn new() -> Self { - Self { db: CacheDB::new(EmptyDB::default()) } + Self { latency_db: LatencyDb::default() } } /// Add an account with the specified balance and nonce. @@ -74,7 +77,9 @@ impl TestDbBuilder { /// * `balance` - The account balance in wei /// * `nonce` - The account nonce (transaction count) pub fn with_account(mut self, address: Address, balance: U256, nonce: u64) -> Self { - self.db.insert_account_info(address, AccountInfo { balance, nonce, ..Default::default() }); + self.latency_db + .in_mem_db + .insert_account_info(address, AccountInfo { balance, nonce, ..Default::default() }); self } @@ -87,10 +92,10 @@ impl TestDbBuilder { /// * `value` - The value to store pub fn with_storage(mut self, address: Address, slot: U256, value: U256) -> Self { // Ensure the account exists before setting storage - if !self.db.cache.accounts.contains_key(&address) { - self.db.insert_account_info(address, AccountInfo::default()); + if !self.latency_db.in_mem_db.cache.accounts.contains_key(&address) { + self.latency_db.in_mem_db.insert_account_info(address, AccountInfo::default()); } - let _ = self.db.insert_account_storage(address, slot, value); + let _ = self.latency_db.in_mem_db.insert_account_storage(address, slot, value); self } @@ -103,60 +108,78 @@ impl TestDbBuilder { /// * `number` - The block number /// * `hash` - The block hash pub fn with_block_hash(mut self, number: u64, hash: B256) -> Self { - self.db.cache.block_hashes.insert(U256::from(number), hash); + self.latency_db.in_mem_db.cache.block_hashes.insert(U256::from(number), hash); + self + } + + /// Add a contract account with the specified bytecode. + pub fn with_contract(mut self, address: Address, bytecode: Bytecode) -> Self { + self.latency_db.in_mem_db.insert_account_info( + address, + AccountInfo { code: Some(bytecode), ..Default::default() }, + ); + self + } + + /// Apply the given latency to every call to the wrapped, in-memory DB. + pub const fn with_latency(mut self, latency: Duration) -> Self { + self.latency_db.latency = Some(latency); self } /// Build the test database. pub fn build(self) -> TestDb { - self.db + CacheDB::new(self.latency_db) } } -#[cfg(test)] -mod tests { - use super::*; +/// In-memory database with configurable per-access latency. +/// +/// Holds all state (accounts, storage, block hashes) in plain hash maps and +/// calls [`std::thread::sleep`] before every read through [`DatabaseRef`]. +/// This simulates production conditions where the backing store (e.g. `AlloyDB`) +/// sends an RPC for each state lookup. +/// +/// Intended to be wrapped in a [`CacheDB`] so that the `CacheDB` provides real +/// mutable-path caching while this type represents the slow backing store behind it. +#[derive(Debug, Clone, Default)] +pub struct LatencyDb { + in_mem_db: CacheDB, + latency: Option, +} - #[test] - fn test_db_builder_creates_empty_db() { - let db = TestDbBuilder::new().build(); - assert!(db.cache.accounts.is_empty()); +impl LatencyDb { + fn sleep(&self) { + if let Some(latency) = self.latency { + std::thread::sleep(latency); + } } +} - #[test] - fn test_db_builder_adds_account() { - let address = Address::repeat_byte(0x01); - let balance = U256::from(1000u64); - let nonce = 5u64; - - let db = TestDbBuilder::new().with_account(address, balance, nonce).build(); +impl DatabaseRef for LatencyDb { + type Error = core::convert::Infallible; - let account = db.cache.accounts.get(&address).unwrap(); - assert_eq!(account.info.balance, balance); - assert_eq!(account.info.nonce, nonce); + fn basic_ref(&self, address: Address) -> Result, Self::Error> { + self.sleep(); + Ok(self.in_mem_db.basic_ref(address).unwrap()) } - #[test] - fn test_db_builder_adds_storage() { - let address = Address::repeat_byte(0x01); - let slot = U256::from(42u64); - let value = U256::from(123u64); - - let db = TestDbBuilder::new().with_storage(address, slot, value).build(); - - let account = db.cache.accounts.get(&address).unwrap(); - let stored = account.storage.get(&slot).unwrap(); - assert_eq!(*stored, value); + fn code_by_hash_ref(&self, code_hash: B256) -> Result { + self.sleep(); + Ok(self.in_mem_db.code_by_hash_ref(code_hash).unwrap()) } - #[test] - fn test_db_builder_adds_block_hash() { - let number = 100u64; - let hash = B256::repeat_byte(0xab); - - let db = TestDbBuilder::new().with_block_hash(number, hash).build(); + fn storage_ref( + &self, + address: Address, + index: StorageKey, + ) -> Result { + self.sleep(); + Ok(self.in_mem_db.storage_ref(address, index).unwrap()) + } - let stored = db.cache.block_hashes.get(&U256::from(number)).unwrap(); - assert_eq!(*stored, hash); + fn block_hash_ref(&self, number: u64) -> Result { + self.sleep(); + Ok(self.in_mem_db.block_hash_ref(number).unwrap()) } } diff --git a/src/test_utils/mod.rs b/src/test_utils/mod.rs index bd6a054b..3652d178 100644 --- a/src/test_utils/mod.rs +++ b/src/test_utils/mod.rs @@ -11,7 +11,7 @@ mod tx; // Re-export test harness components pub use block::{TestBlockBuild, TestBlockBuildBuilder, quick_build_block}; -pub use db::{TestDb, TestDbBuilder, TestStateSource}; +pub use db::{LatencyDb, TestDb, TestDbBuilder, TestStateSource}; pub use env::{TestHostEnv, TestRollupEnv, TestSimEnvBuilder}; pub use scenarios::{ DEFAULT_BALANCE, DEFAULT_BASEFEE, basic_scenario, custom_funded_scenario, funded_test_db,