From 246eab2e6fbfcbaa92bf78b470ea4eeb3bc53e4a Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 22 Feb 2026 04:55:17 -0800 Subject: [PATCH 1/4] fix(ci): derive DET-001 evidence from static-inspection.json status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, DET-001 was marked VERIFIED whenever the static-inspection/ artifact directory existed and was non-empty — ignoring the actual job outcome. Now parses static-inspection.json and checks the status field, so a FAILED inspection correctly produces UNVERIFIED evidence. --- scripts/generate_evidence.cjs | 55 +++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/scripts/generate_evidence.cjs b/scripts/generate_evidence.cjs index 685b7e74..1ebad161 100755 --- a/scripts/generate_evidence.cjs +++ b/scripts/generate_evidence.cjs @@ -22,12 +22,57 @@ function generateEvidence(gatheredArtifactsDir) { } }; + /** + * Parse static-inspection.json for DET-001 claim status. + * Returns VERIFIED only when the file exists, parses as valid JSON, + * contains claim_id "DET-001", and status "PASSED". + * All other conditions return UNVERIFIED with an error description. + */ + const checkStaticInspection = (artifactsDir) => { + const jsonPath = path.join(artifactsDir, 'static-inspection', 'static-inspection.json'); + let raw; + try { + raw = fs.readFileSync(jsonPath, 'utf8'); + } catch (e) { + console.error(`DET-001: static-inspection.json not found at ${jsonPath}`); + return { status: 'UNVERIFIED', error: 'static-inspection.json not found' }; + } + + let parsed; + try { + parsed = JSON.parse(raw); + } catch (e) { + console.error(`DET-001: invalid JSON in static-inspection.json: ${e.message}`); + return { status: 'UNVERIFIED', error: `invalid JSON: ${e.message}` }; + } + + if (parsed.claim_id !== 'DET-001' || typeof parsed.status !== 'string') { + console.error(`DET-001: unexpected structure in static-inspection.json: ${JSON.stringify(parsed)}`); + return { status: 'UNVERIFIED', error: 'missing or unexpected claim_id/status field' }; + } + + const verified = parsed.status === 'PASSED'; + if (!verified) { + console.error(`DET-001: static inspection reported status "${parsed.status}"`); + } + return { status: verified ? 'VERIFIED' : 'UNVERIFIED', source_status: parsed.status }; + }; + const claims = [ - { - id: 'DET-001', - status: checkArtifact('static-inspection') ? 'VERIFIED' : 'UNVERIFIED', - evidence: { workflow, run_id: runId, commit_sha: commitSha, artifact_name: 'static-inspection' } - }, + (() => { + const det001 = checkStaticInspection(gatheredArtifactsDir); + return { + id: 'DET-001', + status: det001.status, + evidence: { + workflow, run_id: runId, commit_sha: commitSha, + artifact_name: 'static-inspection', + source_file: 'static-inspection.json', + source_status: det001.source_status || null, + ...(det001.error ? { error: det001.error } : {}) + } + }; + })(), { id: 'DET-002', status: checkArtifact('det-linux-artifacts') ? 'VERIFIED' : 'UNVERIFIED', From b14b3c7a1b8a5e4d1b62f9d6cddaa900d2a4783b Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 22 Feb 2026 09:38:39 -0800 Subject: [PATCH 2/4] docs: rewrite ADR-0007-impl as proper ADR to fix docs build Replace 3185-line raw conversation transcript with a structured ADR document. The Vue template compiler was crashing on bare Rust generics outside fenced code blocks. The new document preserves all architectural knowledge in 13 sections covering module organization, data structures, tick pipeline, radix scheduler, conflict detection, BOAW parallel execution, state root hashing, WSC build process, design decisions, test strategy, and ADR-0007 alignment. --- docs/adr/ADR-0007-impl.md | 3735 +++++++------------------------------ 1 file changed, 678 insertions(+), 3057 deletions(-) diff --git a/docs/adr/ADR-0007-impl.md b/docs/adr/ADR-0007-impl.md index 3c7db14e..d0e41b89 100644 --- a/docs/adr/ADR-0007-impl.md +++ b/docs/adr/ADR-0007-impl.md @@ -1,3185 +1,806 @@ - - -Perfect! Now I have a comprehensive understanding. Let me create a summary document for you. - - Summary: warp-core Module Organization & Architecture - - Based on my thorough exploration of /Users/james/git/echo/crates/warp-core/src, here's a detailed breakdown of - the module structure, key components, and their responsibilities. - - --- - 1. OVERALL MODULE ORGANIZATION - - The crate is organized into several architectural layers: - - Core Modules (42 Rust files total) - - Public API Modules: - - math/ - Deterministic math subsystem (Vec3, Mat4, Quat, PRNG, trigonometry) - - wsc/ - Write-Streaming Columnar snapshot format - - inbox/ - Canonical inbox management for deterministic intent sequencing - - materialization/ - Channel-based output subsystem - - fixed/ - Q32.32 fixed-point helpers - - Private Infrastructure Modules: - - engine_impl.rs (1806 lines) - Core rewrite engine - - scheduler.rs (1304 lines) - Deterministic radix-sort scheduler - - tick_patch.rs (1694 lines) - Delta boundary artifacts (Paper III) - - graph.rs (568 lines) - In-memory graph store - - snapshot.rs (350 lines) - State snapshots and hash computation - - receipt.rs (241 lines) - Tick receipts (accepted/rejected rewrites) - - footprint.rs (250 lines) - Independence checks and read/write tracking - - rule.rs (84 lines) - Rewrite rule definitions - - attachment.rs (403 lines) - Polymorphic payload system - - payload.rs (304 lines) - Motion atom encoding/decoding - - warp_state.rs (116 lines) - Multi-instance WARP state (Stage B1) - - Supporting Modules: - - ident.rs (162 lines) - Identifier types and hashing - - record.rs - Node/edge record definitions - - tx.rs - Transaction identifiers - - telemetry.rs - Observability hooks - - sandbox.rs - Testing harness - - serializable.rs - Serde support (feature-gated) - - constants.rs - Constants and policy IDs - - --- - 2. GRAPHSTORE DEFINITION & USAGE - - Location: /Users/james/git/echo/crates/warp-core/src/graph.rs - - Purpose: In-memory graph storage for the rewrite executor and tests. The production engine will eventually swap - in a content-addressed store, but this structure keeps the spike self-contained. - - Key Structure: - pub struct GraphStore { - pub(crate) warp_id: WarpId, - pub(crate) nodes: BTreeMap, - pub(crate) edges_from: BTreeMap>, // Source → edges - pub(crate) edges_to: BTreeMap>, // Reverse: target ← edge ids - pub(crate) node_attachments: BTreeMap, - pub(crate) edge_attachments: BTreeMap, - pub(crate) edge_index: BTreeMap, // Reverse lookup: edge → from - pub(crate) edge_to_index: BTreeMap, // Reverse lookup: edge → to - } - - Key API Operations: - - insert_node(id, record) - Add/replace node - - insert_edge(from, edge) - Add/replace directed edge with upsert semantics - - delete_node_isolated(node) - Remove isolated node (rejects if edges exist) - - delete_node_cascade(node) - [INTERNAL] Remove node and all incident edges - - delete_edge_exact(from, edge_id) - Remove specific edge - - canonical_state_hash() - Deterministic BLAKE3 digest over full graph (V2 format) - - Iteration: iter_nodes(), iter_edges(), iter_node_attachments(), iter_edge_attachments() - - Determinism Contract: - - All maps use BTreeMap for stable iteration order - - Edge hashing sorts all edges by EdgeId before encoding - - Attachment hashing includes type_id + u64 LE length + bytes - - Node count and edge count encoded as u64 LE for WSC alignment - - Usage in Engine: - - Wrapped in WarpState for multi-instance support (Stage B1) - - Modified during rewrite execution via rule's ExecuteFn - - Snapshotted after tick commit to verify determinism - - Replayed via TickPatch for delta reconstruction - - --- - 3. EXECUTOR/REWRITE PIPELINE CODE - - Main Engine: /Users/james/git/echo/crates/warp-core/src/engine_impl.rs (1806 lines) - - High-Level Pipeline: - - ┌──────────────────────────────────────┐ - │ EngineBuilder (fluent constructor) │ - └──────────────┬──────────────────────┘ - │ - ▼ - ┌─────────────┐ - │ Engine │ Owns: - └─────────────┘ - WarpState (graph stores) - │ - Registered RewriteRules - │ - DeterministicScheduler - │ - TickHistory - │ - MaterializationBus - │ - ┌─────────┼─────────┐ - │ │ │ - ▼ ▼ ▼ - ┌─────────┐ ┌──────────┐ ┌────────────────┐ - │ Ingest │ │ Dispatch │ │ Tick Commit │ - │ Intent │ │ Intent │ │ + Finalize │ - └─────────┘ └──────────┘ └────────────────┘ - │ │ │ - └─────────┼──────────────┘ - │ - ▼ - ┌─────────────────────┐ - │ Rewrite Execution │ - │ (Matching & Apply) │ - └─────────────────────┘ - - Core Types: - - RewriteRule (/Users/james/git/echo/crates/warp-core/src/rule.rs): - pub struct RewriteRule { - pub id: Hash, // Deterministic rule ID - pub name: &'static str, // Human-readable name - pub left: PatternGraph, // Pattern metadata - pub matcher: MatchFn, // fn(&GraphView, &NodeId) -> bool - pub executor: ExecuteFn, // fn(&GraphView, &mut TickDelta, &NodeId) [Phase 5] - pub compute_footprint: FootprintFn, // fn(&GraphView, &NodeId) -> Footprint - pub factor_mask: u64, // O(1) spatial prefilter - pub conflict_policy: ConflictPolicy, // Abort, Retry, or Join - pub join_fn: Option, // Optional join strategy - } - - PendingRewrite (scheduler): - pub(crate) struct PendingRewrite { - pub rule_id: Hash, - pub compact_rule: CompactRuleId, // Hot-path handle - pub scope_hash: Hash, // Deterministic ordering key - pub scope: NodeKey, // Scope node reference - pub footprint: Footprint, // Read/write sets - pub phase: RewritePhase, // Matched → Reserved → Committed/Aborted - } - - Execution Flow: - - 1. Registration (register_rule): - - Validate rule uniqueness - - Assign compact rule ID - - Store in lookup maps - 2. Enqueueing (apply): - - Match rule against scope - - Compute footprint - - Enqueue PendingRewrite with last-wins semantics on (scope_hash, compact_rule) - 3. Scheduling (radix drain): - - Extract pending rewrites in deterministic order: (scope_hash, rule_id, nonce) - - Stable LSD radix sort (20 passes: 2 nonce + 2 rule + 16 scope) with 16-bit digits - - O(n) complexity, zero comparisons - 4. Independence Checks: - - Check footprint against ActiveFootprints (generation-stamped sets) - - O(1) conflict detection via GenSet lookups - - Apply conflict policy: Abort, Retry, or Join - 5. Execution: - - Invoke rule's executor callback with mutable GraphStore - - Create ScopedEmitter for materialization - - Collect emissions in order-independent BTreeMap> - 6. Commit: - - Snapshot graph state - - Compute state root hash - - Finalize materialization channels - - Record (Snapshot, TickReceipt, TickPatch) in tick history - - --- - 4. DELTA/TICK INFRASTRUCTURE - - Location: /Users/james/git/echo/crates/warp-core/src/tick_patch.rs (1694 lines) - - Purpose: - - Tick patches are prescriptive boundary records for one worldline step—sufficient to deterministically replay the - tick as a pure delta without re-running rule matching or scheduling. - - Key Types: - - TickCommitStatus: - pub enum TickCommitStatus { - Committed, // Successful commit - Aborted, // Reserved for future transactional semantics - } - - SlotId (unversioned slot identifiers): - pub enum SlotId { - Node(NodeKey), // Full node record (instance-scoped skeleton) - Edge(EdgeKey), // Full edge record (instance-scoped skeleton) - Attachment(AttachmentKey), // Node/edge plane payload + Descend links - Port(PortKey), // Boundary port value (opaque key) - } - - WarpOp (canonical delta operations): - pub enum WarpOp { - PortalInit { ... }, // Open descended attachment atomically (Stage B1.1) - UpsertNode { ... }, // Insert/replace node record - DeleteNode { ... }, // Delete node (cascade edges) - UpsertEdge { ... }, // Insert/replace edge record - DeleteEdge { ... }, // Delete specific edge - SetAttachment { ... }, // Set/clear attachment value - // ... more operations - } - - WarpTickPatchV1 (minimal intentional format): - pub struct WarpTickPatchV1 { - pub commit_status: TickCommitStatus, - pub ops: Vec, // Canonical graph edits - pub in_slots: Vec, // Unversioned input slots - pub out_slots: Vec, // Unversioned output slots - pub patch_digest: Hash, // BLAKE3 digest of patch bytes - } - - Determinism Properties: - - - Ops are prescriptive: execute without re-matching - - in_slots/out_slots are unversioned (Paper III-compatible) - - Value versioning recovered via (slot_id, tick_index) interpretation - - Patch digest committed to snapshot for boundary verification - - Usage in Engine: - - - diff_state() computes patch after commit - - Tick history stores: Vec<(Snapshot, TickReceipt, WarpTickPatchV1)> - - Used by jump_to_tick(index) for deterministic replay - - Future: replayed to recover intermediate state without re-running rules - - --- - 5. SCHEDULER & INDEPENDENCE CHECKING - - Location: /Users/james/git/echo/crates/warp-core/src/scheduler.rs (1304 lines) - - RadixScheduler: - - pub(crate) struct RadixScheduler { - pending: HashMap>, - pub(crate) active: HashMap, - } - - ActiveFootprints (generation-stamped conflict detection): - - pub(crate) struct ActiveFootprints { - nodes_written: GenSet, - nodes_read: GenSet, - edges_written: GenSet, - edges_read: GenSet, - attachments_written: GenSet, - attachments_read: GenSet, - ports: GenSet, - } - - Key Operations: - - - enqueue(tx, rewrite) - Last-wins on (scope_hash, compact_rule) - - drain_for_tx(tx) - Returns pending rewrites in deterministic order - - try_reserve(rewrite) - O(m) footprint independence check via GenSet - - Deterministic Ordering: - - Lexicographic order: (scope_hash, rule_id, nonce) - - scope_hash: Full 32-byte node hash - - rule_id: Compact rule identifier - - nonce: Insertion order tiebreaker - - --- - 6. MATERIALIZATION SUBSYSTEM - - Location: /Users/james/git/echo/crates/warp-core/src/materialization/ - - Architecture: - - External World - │ - ▲ (ViewOps frames) - │ - ┌────┴──────────────┐ - │ MaterializationPort │ ← Boundary API - └────┬──────────────┘ - │ - ┌────┴──────────────┐ - │ MaterializationBus │ ← Internal Runtime - └────┬──────────────┘ - │ - ┌────┴──────────────┐ - │ Engine │ commit → finalize - └───────────────────┘ - - Key Components: - - ChannelId (256-bit hash from domain-separated label): - pub fn make_channel_id(label: &str) -> ChannelId - - ChannelPolicy (finalization strategies): - pub enum ChannelPolicy { - Log, // All emissions in EmitKey order (default, confluence-safe) - StrictSingle, // Error if >1 emission (single-writer enforcement) - Reduce(ReduceOp), // Merge via commutative operation (Sum, First, etc.) - } - - EmitKey (order-independent emission identifiers): - pub struct EmitKey { - pub scope_hash: Hash, // Hash of scope node - pub rule_id: u32, // Compact rule ID - pub subkey: u32, // Optional differentiator (default 0) - } - Lexicographic ordering ensures deterministic finalization regardless of rule execution order. - - ScopedEmitter (context-aware emission adapter): - pub struct ScopedEmitter<'a> { - bus: &'a MaterializationBus, - scope_hash: Hash, // Auto-filled from execution context - rule_id: u32, // Prevents rule forgery - } - - emit(channel, data) - Emit with subkey 0 - - emit_with_subkey(channel, subkey, data) - Multi-emission support - - MaterializationBus (tick-scoped collection): - pub struct MaterializationBus { - pending: RefCell>>>, - policies: BTreeMap, - } - - Collects emissions during tick execution - - Finalizes post-commit according to policies - - Not thread-safe by design (single-threaded tick execution) - - MaterializationPort (external subscription interface): - pub struct MaterializationPort { - subscriptions: BTreeSet, - replay_cache: BTreeMap>, // Replay(1) semantics - pending_frames: Vec, - } - - Subscribe to channels - - Receive cached values (late-joiner support) - - Drain pending frames for transport - - Confluence Safety: - - - EmitKey ordering derived from scheduler's canonical order - - All policies preserve confluence (no silent "winner picks") - - Duplicate emissions rejected (catches non-deterministic iteration) - - --- - 7. TICK RECEIPTS & SNAPSHOTS - - TickReceipt (/Users/james/git/echo/crates/warp-core/src/receipt.rs): - pub struct TickReceipt { - tx: TxId, - entries: Vec, // Per-candidate outcomes - blocked_by: Vec>, // Blocking causality poset - digest: Hash, // Canonical commitment - } - - Records accepted vs rejected rewrites - - Captures blocking causality (which accepted rewrites blocked rejections) - - Digest commits to decision outcomes - - Snapshot (/Users/james/git/echo/crates/warp-core/src/snapshot.rs): - pub struct Snapshot { - pub root: NodeKey, - pub hash: Hash, // Commit ID (v2) - pub parents: Vec, // Parent snapshot hashes - pub plan_digest: Hash, // Candidate set + ordering - pub decision_digest: Hash, // Tick receipt outcomes - pub rewrites_digest: Hash, // Applied rewrites - pub patch_digest: Hash, // Tick patch boundary - pub policy_id: u32, // Aion policy version pin - pub tx: TxId, - } - - state_root: Graph-only BLAKE3 hash - - commit_id (v2): state_root + patch_digest + parents + policy_id - - Deterministic across node, edge, and attachment traversal - - --- - 8. KEY INVARIANTS & DETERMINISM - - Ordering Guarantees: - - All maps use BTreeMap for stable iteration - - Rewrites execute in (scope_hash, rule_id, nonce) order (O(n) radix sort) - - Channel emissions finalized in (channel, scope_hash, rule_id, subkey) order - - Determinism Boundaries: - - Wire Format: Canonical CBOR (via echo_wasm_abi) - - JSON: Forbidden for protocol; allowed only for debug/telemetry - - Float Math: Optimistic IEEE 754 by default; det_fixed feature for strict consensus - - Hashing: BLAKE3 with architecture-independent encoding (u64 LE lengths) - - Footprint Independence: - - O(1) conflict detection via generation-stamped sets (GenSet) - - Checks: node read/write, edge read/write, attachment, ports - - O(m) reserve time where m = footprint size - - --- - 9. EXAMPLE WORKFLOW: FULL TICK EXECUTION - - 1. ingest_intent(payload) → IngestDisposition - ↓ (queued in inbox) - - 2. dispatch_next_intent() → DispatchDisposition - ↓ (pending edge consumed) - - 3. apply(rule, scope) → ApplyResult - ├─ matcher(store, scope) → bool - ├─ compute_footprint(store, scope) → Footprint - └─ enqueue PendingRewrite - ↓ - - 4. commit_with_receipt() → (Snapshot, TickReceipt, Report) - ├─ drain_for_tx() in (scope_hash, rule_id, nonce) order - ├─ For each rewrite: - │ ├─ try_reserve(footprint) → bool - │ ├─ executor(store, scope) [calls emit() via ScopedEmitter] - │ └─ record outcome (Applied/Rejected) - │ - ├─ Finalize MaterializationBus: - │ ├─ Group emissions by (channel, EmitKey) - │ ├─ Apply ChannelPolicy to each channel - │ └─ Produce FinalizeReport (channels + errors) - │ - ├─ Snapshot graph: - │ ├─ canonical_state_hash() - │ ├─ Compute commit_id (v2) - │ └─ Store in tick_history - │ - ├─ Compute tick patch: - │ ├─ diff_state() against previous - │ └─ Generate WarpTickPatchV1 - │ - └─ Return to caller - ↓ - - 5. receive_finalized(channels) @ MaterializationPort - ├─ Update replay cache - └─ Queue frames for subscribed channels - - --- - 10. FILE SUMMARY TABLE - - ┌──────────────────┬───────┬───────────────────────────────────────────────────┐ - │ File │ Lines │ Purpose │ - ├──────────────────┼───────┼───────────────────────────────────────────────────┤ - │ engine_impl.rs │ 1806 │ Core rewrite engine, builder, tick execution │ - ├──────────────────┼───────┼───────────────────────────────────────────────────┤ - │ tick_patch.rs │ 1694 │ Delta operations and patch types (Paper III) │ - ├──────────────────┼───────┼───────────────────────────────────────────────────┤ - │ scheduler.rs │ 1304 │ Radix scheduler + footprint independence checks │ - ├──────────────────┼───────┼───────────────────────────────────────────────────┤ - │ graph.rs │ 568 │ In-memory graph store (nodes, edges, attachments) │ - ├──────────────────┼───────┼───────────────────────────────────────────────────┤ - │ snapshot.rs │ 350 │ State snapshots and hash computation │ - ├──────────────────┼───────┼───────────────────────────────────────────────────┤ - │ attachment.rs │ 403 │ Polymorphic payload system (Atom, Descend) │ - ├──────────────────┼───────┼───────────────────────────────────────────────────┤ - │ payload.rs │ 304 │ Motion atom encoding/decoding │ - ├──────────────────┼───────┼───────────────────────────────────────────────────┤ - │ receipt.rs │ 241 │ Tick receipts (accepted/rejected outcomes) │ - ├──────────────────┼───────┼───────────────────────────────────────────────────┤ - │ footprint.rs │ 250 │ Footprints and independence tracking │ - ├──────────────────┼───────┼───────────────────────────────────────────────────┤ - │ inbox.rs │ 210 │ Inbox management and intent queuing │ - ├──────────────────┼───────┼───────────────────────────────────────────────────┤ - │ materialization/ │ ~60 │ Channel emissions, ports, frames │ - ├──────────────────┼───────┼───────────────────────────────────────────────────┤ - │ warp_state.rs │ 116 │ Multi-instance WARP state (Stage B1) │ - ├──────────────────┼───────┼───────────────────────────────────────────────────┤ - │ rule.rs │ 84 │ Rewrite rule definitions │ - ├──────────────────┼───────┼───────────────────────────────────────────────────┤ - │ ident.rs │ 162 │ Identifier types and hashing │ - └──────────────────┴───────┴───────────────────────────────────────────────────┘ - --- - This is a sophisticated, determinism-first engine architecture with clear separation between execution (rules + - scheduler), storage (GraphStore), and output (materialization). The key innovation is the order-independent - materialization system backed by deterministic ordering of emissions. - -⎿  Done (23 tool uses · 85.5k tokens · 1m 21s) - -⏺ Explore(Explore WSC implementation) -⎿  Prompt: -Explore the WSC (WARP Snapshot Container) implementation in the codebase. Look for: 1. WscFile, WarpView, or similar snapshot-related code 2. How snapshots are read/written 3. Any existing zero-copy or mmap infrastructure 4. Segment or blob arena handling - - Search in crates/warp-core and any related locations. This is research only - do not modify any code. - -⎿  ❯ Explore the WSC (WARP Snapshot Container) implementation in the codebase. Look for: 1. WscFile, WarpView, or similar snapshot-related code 2. How snapshots are read/written 3. Any existing zero-copy or mmap infrastructure 4. Segment or blob arena handling - - Search in crates/warp-core and any related locations. This is research only - do not modify any code. - -⎿  Search(pattern: "**/warp-core/**/\*.rs") - -⎿  Search(pattern: "WscFile|WarpView|snapshot", type: "rs") - -⎿  Search(pattern: "zero.copy|mmap|MemoryMap", type: "rs") - -⎿  Search(pattern: "\*\*/warp-core/Cargo.toml") - -⎿    ⎿  Found 0 files - -⎿    ⎿  Found 0 files - -⎿    ⎿  Found 1 file -/Users/james/git/echo/crates/warp-core/Cargo.toml -⎿    ⎿  Found 81 files -/Users/james/git/echo/crates/warp-core/build.rs -/Users/james/git/echo/crates/warp-core/src/footprint.rs -/Users/james/git/echo/crates/warp-core/src/rule.rs -/Users/james/git/echo/crates/warp-core/src/sandbox.rs -/Users/james/git/echo/crates/warp-core/tests/atom_payload_digest_tests.rs -/Users/james/git/echo/crates/warp-core/tests/footprint_independence_tests.rs -/Users/james/git/echo/crates/warp-core/tests/mat4_mul_tests.rs -/Users/james/git/echo/crates/warp-core/tests/math_additional_tests.rs -/Users/james/git/echo/crates/warp-core/tests/math_convenience_tests.rs -/Users/james/git/echo/crates/warp-core/tests/nan_exhaustive_tests.rs -/Users/james/git/echo/crates/warp-core/tests/snapshot_reachability_tests.rs -/Users/james/git/echo/crates/warp-core/tests/vec3_ops_tests.rs -/Users/james/git/echo/crates/warp-core/tests/math_rotation_tests.rs -/Users/james/git/echo/crates/warp-core/src/math/mat4.rs -/Users/james/git/echo/crates/warp-core/src/math/trig_lut.rs -/Users/james/git/echo/crates/warp-core/src/attachment.rs -/Users/james/git/echo/crates/warp-core/src/cmd.rs -/Users/james/git/echo/crates/warp-core/src/record.rs -/Users/james/git/echo/crates/warp-core/src/serializable.rs -/Users/james/git/echo/crates/warp-core/src/snapshot.rs -/Users/james/git/echo/crates/warp-core/src/tx.rs -/Users/james/git/echo/crates/warp-core/src/warp_state.rs -/Users/james/git/echo/crates/warp-core/tests/determinism_audit.rs -/Users/james/git/echo/crates/warp-core/tests/ledger_tests.rs -/Users/james/git/echo/crates/warp-core/src/bin/gen_sin_qtr_lut.rs -/Users/james/git/echo/crates/warp-core/src/constants.rs -/Users/james/git/echo/crates/warp-core/src/inbox.rs -/Users/james/git/echo/crates/warp-core/src/math/fixed_q32_32.rs -/Users/james/git/echo/crates/warp-core/src/math/mod.rs -/Users/james/git/echo/crates/warp-core/src/math/prng.rs -/Users/james/git/echo/crates/warp-core/src/math/quat.rs -/Users/james/git/echo/crates/warp-core/src/math/scalar.rs -/Users/james/git/echo/crates/warp-core/src/math/trig.rs -/Users/james/git/echo/crates/warp-core/src/math/vec3.rs -/Users/james/git/echo/crates/warp-core/src/payload.rs -/Users/james/git/echo/crates/warp-core/src/receipt.rs -/Users/james/git/echo/crates/warp-core/src/scheduler.rs -/Users/james/git/echo/crates/warp-core/src/telemetry.rs -/Users/james/git/echo/crates/warp-core/src/tick_patch.rs -/Users/james/git/echo/crates/warp-core/tests/determinism_policy_tests.rs -/Users/james/git/echo/crates/warp-core/tests/deterministic_sin_cos_tests.rs -/Users/james/git/echo/crates/warp-core/tests/dfix64_tests.rs -/Users/james/git/echo/crates/warp-core/tests/dispatch_inbox.rs -/Users/james/git/echo/crates/warp-core/tests/dpo_concurrency_litmus.rs -/Users/james/git/echo/crates/warp-core/tests/duplicate_rule_registration_tests.rs -/Users/james/git/echo/crates/warp-core/tests/engine_motion_negative_tests.rs -/Users/james/git/echo/crates/warp-core/tests/engine_motion_tests.rs -/Users/james/git/echo/crates/warp-core/tests/inbox.rs -/Users/james/git/echo/crates/warp-core/tests/math_scalar_tests.rs -/Users/james/git/echo/crates/warp-core/tests/math_validation.rs -/Users/james/git/echo/crates/warp-core/tests/permutation_commute_tests.rs -/Users/james/git/echo/crates/warp-core/tests/property_commute_tests.rs -/Users/james/git/echo/crates/warp-core/tests/proptest_seed_pinning.rs -/Users/james/git/echo/crates/warp-core/tests/reserve_gate_tests.rs -/Users/james/git/echo/crates/warp-core/tests/tick_receipt_tests.rs -/Users/james/git/echo/crates/warp-core/tests/tx_lifecycle_tests.rs -/Users/james/git/echo/crates/warp-core/src/fixed.rs -/Users/james/git/echo/crates/warp-core/src/graph.rs -/Users/james/git/echo/crates/warp-core/src/ident.rs -/Users/james/git/echo/crates/warp-core/src/wsc/build.rs -/Users/james/git/echo/crates/warp-core/src/wsc/mod.rs -/Users/james/git/echo/crates/warp-core/src/wsc/read.rs -/Users/james/git/echo/crates/warp-core/src/wsc/types.rs -/Users/james/git/echo/crates/warp-core/src/wsc/validate.rs -/Users/james/git/echo/crates/warp-core/src/wsc/view.rs -/Users/james/git/echo/crates/warp-core/src/wsc/write.rs -/Users/james/git/echo/crates/warp-core/src/lib.rs -/Users/james/git/echo/crates/warp-core/src/materialization/emit_key.rs -/Users/james/git/echo/crates/warp-core/src/materialization/port.rs -/Users/james/git/echo/crates/warp-core/src/materialization/emission_port.rs -/Users/james/git/echo/crates/warp-core/src/materialization/channel.rs -/Users/james/git/echo/crates/warp-core/src/materialization/reduce_op.rs -/Users/james/git/echo/crates/warp-core/src/materialization/scoped_emitter.rs -/Users/james/git/echo/crates/warp-core/tests/materialization_engine_integration.rs -/Users/james/git/echo/crates/warp-core/src/materialization/mod.rs -/Users/james/git/echo/crates/warp-core/src/engine_impl.rs -/Users/james/git/echo/crates/warp-core/src/materialization/bus.rs -/Users/james/git/echo/crates/warp-core/tests/common/mod.rs -/Users/james/git/echo/crates/warp-core/tests/materialization_spec_police.rs -/Users/james/git/echo/crates/warp-core/tests/materialization_determinism.rs -/Users/james/git/echo/crates/warp-core/src/materialization/frame.rs -⎿  Read(/Users/james/git/echo/crates/warp-core/src/wsc) - -⎿  Read(/Users/james/git/echo/crates/warp-core/src/wsc/mod.rs) - -⎿  Read(/Users/james/git/echo/crates/warp-core/src/wsc/types.rs) - -⎿  Read(/Users/james/git/echo/crates/warp-core/src/wsc/view.rs) - -⎿  Error: EISDIR: illegal operation on a directory, read - -⎿  Read 63 lines - -⎿  Read 301 lines - -⎿  Read 499 lines - -⎿  Read(/Users/james/git/echo/crates/warp-core/src/wsc/read.rs) - -⎿  Read(/Users/james/git/echo/crates/warp-core/src/wsc/write.rs) - -⎿  Read(/Users/james/git/echo/crates/warp-core/src/wsc/build.rs) - -⎿  Read 312 lines - -⎿  Read 302 lines - -⎿  Read 581 lines - -⎿  Read(/Users/james/git/echo/crates/warp-core/src/wsc/validate.rs) - -⎿  Read(/Users/james/git/echo/crates/warp-core/tests/snapshot_reachability_tests.rs) - -⎿  Read(/Users/james/git/echo/crates/warp-core/Cargo.toml) - -⎿  Read 656 lines - -⎿  Read 72 lines - -⎿  Read 53 lines - -⎿  Search(pattern: "blob|arena|segment", path: "/Users/james/git/echo/crates/warp-core/src", type: "rs") - -⎿  Read(/Users/james/git/echo/crates/warp-core/src/attachment.rs · lines 1-200) - -⎿  Search(pattern: "mmap|MemoryMap|memory.map", path: "/Users/james/git/echo/crates", type: "rs") - -⎿  Read 200 lines - -⎿    ⎿  Found 0 files - -⎿    ⎿  Found 0 files - -⎿  Read(/Users/james/git/echo/crates/warp-core/src/graph.rs · lines 1-250) - -⎿  Read(/Users/james/git/echo/crates/warp-core/src/snapshot.rs · lines 1-150) - -⎿  Bash(grep -r "WscFile\|WarpView" /Users/james/git/echo/crates/warp-core --include="\*.rs" | head -20) -Waiting… -⎿  Read 250 lines - -⎿  Read 150 lines - -⎿  /Users/james/git/echo/crates/warp-core/src/wsc/mod.rs://! use warp*core::wsc::{WscFile, validate_wsc}; -/Users/james/git/echo/crates/warp-core/src/wsc/mod.rs://! let file = WscFile::open("state.wsc")?; -/Users/james/git/echo/crates/warp-core/src/wsc/mod.rs:pub use view::{AttachmentRef, WarpView, WscFile}; -/Users/james/git/echo/crates/warp-core/src/wsc/validate.rs:use super::view::WscFile; -/Users/james/git/echo/crates/warp-core/src/wsc/validate.rs:pub fn validate_wsc(file: &WscFile) -> Result<(), ReadError> -{ -/Users/james/git/echo/crates/warp-core/src/wsc/validate.rs:fn validate_warp_view(view: &super::view::WarpView<'*>) -> -Result<(), ReadError> { -/Users/james/git/echo/crates/warp-core/src/wsc/validate.rs: let file = WscFile::from_bytes(bytes).unwrap(); -/Users/james/git/echo/crates/warp-core/src/wsc/validate.rs: let file = WscFile::from_bytes(bytes).unwrap(); -/Users/james/git/echo/crates/warp-core/src/wsc/validate.rs: let file = WscFile::from_bytes(bytes).unwrap(); -/Users/james/git/echo/crates/warp-core/src/wsc/validate.rs: let file = WscFile::from_bytes(bytes).unwrap(); -/Users/james/git/echo/crates/warp-core/src/wsc/validate.rs: let file = WscFile::from_bytes(bytes).unwrap(); -/Users/james/git/echo/crates/warp-core/src/wsc/validate.rs: let file = WscFile::from_bytes(bytes).unwrap(); -/Users/james/git/echo/crates/warp-core/src/wsc/validate.rs: let file = WscFile::from_bytes(bytes).unwrap(); -/Users/james/git/echo/crates/warp-core/src/wsc/validate.rs: let file = WscFile::from_bytes(bytes).unwrap(); -/Users/james/git/echo/crates/warp-core/src/wsc/validate.rs: let file = WscFile::from_bytes(bytes).unwrap(); -/Users/james/git/echo/crates/warp-core/src/wsc/validate.rs: let file = WscFile::from_bytes(bytes).unwrap(); -/Users/james/git/echo/crates/warp-core/src/wsc/validate.rs: let file = WscFile::from_bytes(bytes).unwrap(); -/Users/james/git/echo/crates/warp-core/src/wsc/validate.rs: let file = WscFile::from_bytes(bytes).unwrap(); -/Users/james/git/echo/crates/warp-core/src/wsc/validate.rs: let file = WscFile::from_bytes(bytes).unwrap(); -/Users/james/git/echo/crates/warp-core/src/wsc/validate.rs: let file = WscFile::from_bytes(bytes).unwrap(); -⎿  Read(/Users/james/git/echo/crates/warp-core/src/materialization/frame.rs · lines 1-300) - -⎿  Bash(head -100 /Users/james/git/echo/crates/warp-core/src/materialization/bus.rs) -Waiting… -⎿  Read 255 lines - -⎿  // SPDX-License-Identifier: Apache-2.0 -// © James Ross Ω FLYING•ROBOTS -//! Internal materialization bus for tick-scoped channel emissions. -//! -//! The [`MaterializationBus`] collects emissions from rewrite rules during a tick. -//! Emissions are stored in an order-independent manner (keyed by [`EmitKey`]), -//! then finalized post-commit according to each channel's policy. -//! -//! # Order Independence -//! -//! The bus uses `BTreeMap>>` internally. -//! This ensures that: -//! 1. Insertion order doesn't affect the final result -//! 2. Finalization iterates in deterministic (canonical) order -//! 3. Confluence is preserved regardless of rewrite execution order -//! -//! # Usage -//! -//! Rules emit via `bus.emit(channel, emit_key, data)`. After commit, the engine -//! calls `bus.finalize()` to resolve each channel according to its policy and -//! produce the final output. - - use std::cell::RefCell; - use std::collections::btree_map::Entry; - use std::collections::BTreeMap; - - use super::channel::{ChannelConflict, ChannelId, ChannelPolicy, MaterializationErrorKind}; - use super::emit_key::EmitKey; - - /// Error returned when the same `(channel, EmitKey)` pair is emitted twice. - /// - /// This is a structural invariant: even if the payloads are identical, duplicate - /// emissions indicate a bug in the rule (e.g., iterating a non-deterministic source - /// like an unordered container without proper subkey differentiation). - /// - /// # Why Reject Identical Payloads? - /// - /// Allowing "identical payload = OK" encourages sloppy code that emits redundantly. - /// Then someone changes a field and tests fail mysteriously. Rejecting always forces - /// rule authors to think: "Am I iterating deterministically? Do I need unique subkeys?" - #[derive(Debug, Clone, PartialEq, Eq)] - pub struct DuplicateEmission { - /// The channel that received the duplicate. - pub channel: ChannelId, - /// The key that was duplicated. - pub key: EmitKey, - } - - impl core::fmt::Display for DuplicateEmission { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!( - f, - "duplicate emission for channel {:?} key {:?}", - self.channel, self.key - ) - } - } - - impl std::error::Error for DuplicateEmission {} - - /// Internal materialization bus for collecting and finalizing channel emissions. - /// - /// This is tick-scoped: emissions accumulate during a tick, then `finalize()` is - /// called post-commit to produce output. The bus is cleared after finalization. - /// - /// # Thread Safety - /// - /// `MaterializationBus` uses [`RefCell`] for interior mutability and is intentionally - /// **not thread-safe** (`!Sync`). This is by design: emissions occur within a single - /// tick's execution context, which is single-threaded. The rewrite engine processes - /// rules sequentially within a tick, so no synchronization is needed. - /// - /// If parallel emission were ever required (e.g., concurrent rule execution), the - /// design would need to change to use `RwLock>` or similar. - #[derive(Debug, Default)] - pub struct MaterializationBus { - /// Pending emissions: `channel -> (emit_key -> data)`. - /// Uses `BTreeMap` for deterministic iteration order. - pending: RefCell>>>, - - /// Channel policies (looked up during finalization). - policies: BTreeMap, - } - - /// Result of finalizing a single channel. - #[derive(Debug, Clone)] - pub struct FinalizedChannel { - /// The channel that was finalized. - pub channel: ChannelId, - /// The finalized data (format depends on policy). - pub data: Vec, - } - - /// Report from finalizing all channels. - /// - /// Unlike a `Result`, this type always succeeds. Channels that finalize - /// successfully appear in `channels`; channels that fail (e.g., `StrictSingle` - /// conflicts) appear in `errors`. This design ensures: - /// - /// 1. **No data loss**: A failing channel doesn't erase other channels' outputs - -⎿  Read(/Users/james/git/echo/docs/adr/ADR-0007-BOAW-Storage.md) - -⎿  Read 1129 lines - -⎿  Response: -Perfect! Now I have a comprehensive understanding. Let me create the final research report: - - WSC (WARP Snapshot Container) Implementation Research Report - - Executive Summary - - The codebase contains a mature, production-ready WSC (Write-Streaming Columnar) snapshot format implementation - located in /Users/james/git/echo/crates/warp-core/src/wsc/. The system is designed for deterministic - serialization and zero-copy deserialization of immutable WARP graph state with explicit infrastructure for - memory-mapped file access and columnar data organization. - - --- - 1. WSC Format Architecture - - File Structure (128-byte header + directory + sections) - - Location: /Users/james/git/echo/crates/warp-core/src/wsc/types.rs - - The format is built on fixed-size, repr(C) rows using bytemuck::Pod for zero-copy transmutation: - ┌──────────────┬───────────┬──────────────────────────────────────────────────────────────────────────────┐ - │ Component │ Size │ Purpose │ - ├──────────────┼───────────┼──────────────────────────────────────────────────────────────────────────────┤ - │ WscHeader │ 128 bytes │ Magic (v1), schema hash, tick, warp count, directory offset │ - ├──────────────┼───────────┼──────────────────────────────────────────────────────────────────────────────┤ - │ NodeRow │ 64 bytes │ Fixed-size node: node_id (32B) + node_type (32B) │ - ├──────────────┼───────────┼──────────────────────────────────────────────────────────────────────────────┤ - │ EdgeRow │ 128 bytes │ Fixed-size edge: edge_id + from_node_id + to_node_id + edge_type │ - ├──────────────┼───────────┼──────────────────────────────────────────────────────────────────────────────┤ - │ Range │ 16 bytes │ Index descriptor: start_le + len_le (u64 each) │ - ├──────────────┼───────────┼──────────────────────────────────────────────────────────────────────────────┤ - │ OutEdgeRef │ 40 bytes │ Outbound edge reference: edge_ix_le + edge_id │ - ├──────────────┼───────────┼──────────────────────────────────────────────────────────────────────────────┤ - │ AttRow │ 56 bytes │ Attachment row: tag (1B) + reserved (7B) + type/warp (32B) + blob offset/len │ - ├──────────────┼───────────┼──────────────────────────────────────────────────────────────────────────────┤ - │ WarpDirEntry │ 184 bytes │ Directory entry per WARP instance with all section offsets │ - └──────────────┴───────────┴──────────────────────────────────────────────────────────────────────────────┘ - Key design properties: - - All fields use little-endian byte order for cross-platform determinism - - All sections are 8-byte aligned for SIMD-friendly access - - Blobs are stored contiguously in an arena with (offset, length) references - - Index tables use Range structures to slice into data tables without copying - - Attachment Types (Tagged Union) - - Two attachment types via AttRow.tag: - - TAG_ATOM (1): Typed payload (type_id + blob location) - - TAG_DESCEND (2): Child WARP reference (WarpId, blob fields unused) - - --- - 2. Zero-Copy Read/Write Infrastructure - - Writing: write_wsc_one_warp (deterministic serialization) - - Location: /Users/james/git/echo/crates/warp-core/src/wsc/write.rs - - Input: OneWarpInput struct containing: - - Pre-sorted nodes/edges (canonical order) - - Index tables (out_index, node_atts_index, edge_atts_index) - - Blob arena (8-byte aligned) - - Process: - 1. Pre-calculate all section sizes and offsets - 2. Allocate buffer with capacity - 3. Write header with magic + metadata - 4. Write WARP directory entry with all offsets - 5. Sequentially write each section with padding to 8-byte boundaries - 6. Blobs written at end as concatenated arena - - Determinism guarantees: - - Input ordering is canonical (nodes by NodeId, edges by EdgeId, out_edges sorted per node) - - Little-endian encoding is architecture-independent - - Identical input produces identical bytes every run - - Reading: WscFile & WarpView (zero-copy views) - - Location: /Users/james/git/echo/crates/warp-core/src/wsc/view.rs - - pub struct WscFile { - data: Vec, // Entire file in memory (mmap not yet implemented) - } - - pub struct WarpView<'a> { - data: &'a [u8], - entry: &'a WarpDirEntry, - nodes: &'a [NodeRow], // Direct slice - no copy - edges: &'a [EdgeRow], // Direct slice - no copy - out_index: &'a [Range], // Index table - out_edges: &'a [OutEdgeRef], // Concatenated edge references - node_atts_index: &'a [Range], - node_atts: &'a [AttRow], - edge_atts_index: &'a [Range], - edge_atts: &'a [AttRow], - blobs: &'a [u8], // Blob arena - } - - Key features: - - No copying: Uses bytemuck::try_cast_slice() for safe type punning - - Lifetime-bound: Borrows prevent data invalidation - - Binary search indexing: node_ix() and edge_ix() use binary search on sorted tables - - Range-based access: Out-edges and attachments accessed via Range slices - - Public API: - impl<'a> WarpView<'a> { - pub fn out_edges_for_node(&self, node_ix: usize) -> &[OutEdgeRef] - pub fn node_attachments(&self, node_ix: usize) -> &[AttRow] - pub fn edge_attachments(&self, edge_ix: usize) -> &[AttRow] - pub fn blob_for_attachment(&self, att: &AttRow) -> Option<&[u8]> - } - - Reading Infrastructure: read.rs - - Location: /Users/james/git/echo/crates/warp-core/src/wsc/read.rs - - Error types: - pub enum ReadError { - Io(io::Error), - FileTooSmall { size, minimum }, - InvalidMagic { expected, actual }, - SectionOutOfBounds { name, offset, length, file_size }, - WarpIndexOutOfBounds { index, count }, - AlignmentViolation { name, offset, alignment }, - IndexRangeOutOfBounds { ... }, // Critical: catches corrupted index tables - InvalidAttachmentTag { tag, index }, - BlobOutOfBounds { offset, length, blob_size }, - OrderingViolation { kind, index }, // Nodes/edges must be sorted - NonZeroReservedBytes { field, index }, - NonAtomHasBlobFields { index, blob_off, blob_len }, - MissingRoot { root }, // Root must exist in nodes array - } - - Key functions: - pub fn validate_header(data: &[u8]) -> Result<&WscHeader, ReadError> - pub fn read_slice<'a, T: Pod>(data: &'a [u8], offset: u64, count: u64, name: &'static str) - -> Result<&'a [T], ReadError> - pub fn read_bytes<'a>(data: &'a [u8], offset: u64, length: u64, name: &'static str) - -> Result<&'a [u8], ReadError> - - --- - 3. Validation & Integrity - - Location: /Users/james/git/echo/crates/warp-core/src/wsc/validate.rs - - The validate_wsc() function performs comprehensive structural validation: - - 1. Index bounds checking: All Range references stay within data table boundaries - 2. Ordering validation: Nodes and edges must be sorted by ID - 3. Attachment validation: Tags must be ATOM (1) or DESCEND (2) - 4. Root node validation: root_node_id must exist in nodes array (or be zero for empty) - 5. Blob bounds: All blob references within the arena - 6. Reserved bytes: Ensure all reserved fields are zero - 7. Out-edge references: Verify all edge indices point to valid edges - - Critical safeguard: The validation catches corrupted index tables that accessors would otherwise mask with - empty slices. - - --- - 4. Build Infrastructure: GraphStore → WSC - - Location: /Users/james/git/echo/crates/warp-core/src/wsc/build.rs - - The build_one_warp_input() function converts in-memory GraphStore to OneWarpInput in canonical order: - - Canonical ordering rules: - - Nodes: BTreeMap iteration (already sorted by NodeId) - - Edges: Globally sorted by EdgeId (via explicit sort) - - Outbound edges: Sorted by EdgeId within each node's bucket - - Attachments: Follow same ordering as their owners - - Determinism tests included: - - "Repeated serialization produces identical bytes" (100x runs) - - "Different insertion orders produce identical bytes" (proves canonicalization) - - "Attachments serialized deterministically" regardless of build order - - Blob handling: - - Attachments are collected with 8-byte alignment (align8_vec()) - - Blobs stored as contiguous arena in order of collection - - Offset/length references stored in AttRow - - --- - 5. Existing Infrastructure for Zero-Copy/mmap - - Current State - - - No explicit mmap implementation yet - - Code comments state: "For very large files, consider using memory-mapped IO instead (not yet implemented)" - - File data currently read entirely into Vec - - Future-Ready Design - - The architecture is explicitly designed for mmap: - - 1. Alignment guarantees: All sections are 8-byte aligned - 2. Pod types: All row types are bytemuck::Pod (byte-identical layout) - 3. Lifetime bounds: WarpView<'a> borrows don't hold file handles - 4. No mutation: Snapshot is immutable by design - 5. Path independence: Reader doesn't care if data comes from memory or mmap - - Path to mmap: - // Pseudo-code: straightforward substitution - pub fn open_mmap(path: impl AsRef) -> Result { - let file = std::fs::File::open(path)?; - let mmap = unsafe { Mmap::map(&file)? }; // or memmap crate - Self::from_bytes(mmap.to_vec()) // or bind to mmap lifetime - } - - --- - 6. Blob Arena Handling - - Design: - - Single contiguous blob section at end of file - - Attachments reference via (offset: u64, length: u64) in AttRow - - Multiple attachments can share blob data if ranges don't overlap - - Blobs are 8-byte aligned for SIMD - - Access pattern: - pub fn blob_for_attachment(&self, att: &AttRow) -> Option<&[u8]> { - if !att.is_atom() { return None; } - let off = att.blob_off() as usize; - let len = att.blob_len() as usize; - self.blobs.get(off..off + len) - } - - Future enhancement (from ADR): Segment-level chunking for COW/structural sharing: - - Individual blob segments become content-addressed - - Commit builder references only changed segments - - Massive savings for append-only workloads - - --- - 7. Related Snapshot Infrastructure - - Snapshot Hash Computation - - Location: /Users/james/git/echo/crates/warp-core/src/snapshot.rs - - The system computes deterministic state_root hash via BLAKE3: - - 1. Performs reachability BFS (following skeleton edges + Descend attachments) - 2. Encodes reachable state in canonical order (by WarpId, then nodes by NodeId) - 3. For each node: encode ID, type, then sorted outbound edges - 4. Include attachment values (presence tag + type + length + bytes) - 5. Hash the canonical byte stream - - Includes: Snapshot struct with commit hash, parents, plan/decision/patch digests - - Materialization Infrastructure - - Location: /Users/james/git/echo/crates/warp-core/src/materialization/frame.rs - - Parallel binary frame format ("MBUS"): - - Magic: "MBUS" (4 bytes) - - Version: v1 (2 bytes LE) - - Payload length (4 bytes LE) - - Channel ID (32 bytes) + data - - Uses similar design principles: deterministic, binary, length-prefixed. - - --- - 8. BOAW Architecture (ADR-0007) - - Location: /Users/james/git/echo/docs/adr/ADR-0007-BOAW-Storage.md - - The accepted architecture road map includes: - - 1. Immutable Base: WSC-backed snapshots remain the canonical format - 2. TickDelta Overlay: Rewrites emit ops into thread-local deltas (no shared mutation) - 3. Deterministic Admission: Only independent rewrites admitted per tick - 4. Lockless Parallel Execute: Workers execute against read-only snapshot - 5. Deterministic Merge: Thread-local deltas canonically merged - 6. Canonical Commit: Apply patch → new snapshot → compute hashes - 7. Collapse/Merge: Multi-parent merges with typed rules - 8. Privacy: Mind mode forbids sensitive atoms in ledger - - Segment-level COW (Section 5.5): - - Snapshots stored as content-addressed segments (1–4MB blocks) - - Commit references segment hashes for: nodes, edges, indices, attachments, blobs - - Only changed segments written; unchanged segments reused - - --- - 9. Test Coverage - - Snapshot & reachability tests: - - /Users/james/git/echo/crates/warp-core/tests/snapshot_reachability_tests.rs - - Unreachable nodes don't affect hash - - Reachable edges do affect hash - - WSC format tests (integrated into modules): - - Header validation - - Magic number checks - - Alignment violations - - Index range bounds - - Ordering violations - - Attachment tag validation - - Blob reference validation - - Root node validation - - Roundtrip (write → read → verify) - - --- - 10. File Manifest - ┌─────────────────────┬───────┬─────────────────────────────────────┐ - │ File │ Lines │ Purpose │ - ├─────────────────────┼───────┼─────────────────────────────────────┤ - │ src/wsc/mod.rs │ 63 │ Module docs + re-exports │ - ├─────────────────────┼───────┼─────────────────────────────────────┤ - │ src/wsc/types.rs │ 301 │ Row type definitions (Pod) │ - ├─────────────────────┼───────┼─────────────────────────────────────┤ - │ src/wsc/view.rs │ 499 │ WscFile, WarpView zero-copy readers │ - ├─────────────────────┼───────┼─────────────────────────────────────┤ - │ src/wsc/read.rs │ 312 │ Low-level read primitives + errors │ - ├─────────────────────┼───────┼─────────────────────────────────────┤ - │ src/wsc/write.rs │ 302 │ Deterministic serialization │ - ├─────────────────────┼───────┼─────────────────────────────────────┤ - │ src/wsc/build.rs │ 581 │ GraphStore → OneWarpInput builder │ - ├─────────────────────┼───────┼─────────────────────────────────────┤ - │ src/wsc/validate.rs │ 656 │ Comprehensive file validation │ - └─────────────────────┴───────┴─────────────────────────────────────┘ - --- - Key Takeaways - - Strengths - - 1. Deterministic by design: Input order doesn't matter; output bytes are identical - 2. Zero-copy ready: Bytemuck Pod types enable safe pointer casting - 3. Validation-first: Catches corruption before use (not in accessors) - 4. Columnar + blob arena: SIMD-friendly, natural for batch operations - 5. Alignment aware: Explicit 8-byte boundaries throughout - 6. Immutable semantics: Snapshot ownership prevents accidental mutation - 7. Well-architected: Clear separation of read/write/validate/build concerns - - Future Work - - 1. mmap integration: Straightforward given Pod guarantees - 2. Segment sharing: ADR-0007 envisions content-addressed segments - 3. TickDelta overlay: Complement snapshots with deterministic delta merging - 4. Collapse/merge: Type-driven multi-parent reconciliation - 5. Privacy enforcement: Mind mode gates and claim records - - --- - Conclusion - - The WSC implementation is production-ready for deterministic snapshot I/O with explicit infrastructure for - zero-copy access and future memory-mapping. The columnar design with blob arena, combined with comprehensive - validation and deterministic ordering, makes it ideal for both research (proving determinism) and production - (high-performance state materialization). ADR-0007 charts a clear path to leveraging this foundation for - lockless parallel execution via thread-local deltas and canonical commits. - -⎿  Done (23 tool uses · 109.3k tokens · 1m 11s) - -⏺ Explore(Explore footprint infrastructure) -⎿  Prompt: -Search for any existing footprint, independence checking, or scheduling infrastructure in the codebase. Look -for: 1. Footprint struct or similar conflict detection code 2. Any sharding or virtual shard logic 3. Admission control or dependency checking 4. Worker queue or parallel execution infrastructure - - Search across all crates. This is research only - do not modify any code. - -⎿  ❯ Search for any existing footprint, independence checking, or scheduling infrastructure in the codebase. Look for: 1. Footprint struct or similar conflict detection code 2. Any sharding or virtual shard logic 3. Admission control or dependency checking 4. Worker queue or parallel execution infrastructure - - Search across all crates. This is research only - do not modify any code. - -⎿  Search(pattern: "footprint|Footprint", glob: "\*_/_.rs") - -⎿  Search(pattern: "shard|Shard|virtual._shard|sharding", glob: "\*\*/_.rs") - -⎿  Search(pattern: "admission|dependency.*check|conflict.*detect", glob: "\*_/_.rs") - -⎿  Search(pattern: "\*\*/src/lib.rs") - -⎿    ⎿  Found 1 file -crates/warp-benches/src/bin/sim\*sharded_store.rs -⎿    ⎿  Found 5 files -crates/warp-core/src/telemetry.rs -crates/warp-core/src/scheduler.rs -crates/warp-core/src/inbox.rs -crates/warp-core/tests/footprint_independence_tests.rs -crates/warp-core/src/rule.rs -⎿    ⎿  Found 18 files -crates/warp-core/src/engine_impl.rs -crates/warp-core/src/materialization/channel.rs -crates/warp-core/src/lib.rs -crates/echo-dind-tests/src/rules.rs -crates/warp-core/tests/tick_receipt_tests.rs -crates/warp-core/tests/dpo_concurrency_litmus.rs -crates/warp-core/tests/duplicate_rule_registration_tests.rs -crates/warp-core/src/tick_patch.rs -crates/warp-core/src/receipt.rs -crates/warp-core/src/scheduler.rs -crates/warp-core/src/inbox.rs -crates/warp-benches/benches/scheduler_drain.rs -crates/echo-dry-tests/src/demo_rules.rs -crates/echo-dry-tests/src/rules.rs -crates/warp-core/src/attachment.rs -crates/warp-core/tests/footprint_independence_tests.rs -crates/warp-core/src/rule.rs -crates/warp-core/src/footprint.rs -⎿    ⎿  Found 16 files -/Users/james/git/echo/crates/echo-app-core/src/lib.rs -/Users/james/git/echo/crates/echo-config-fs/src/lib.rs -/Users/james/git/echo/crates/echo-graph/src/lib.rs -/Users/james/git/echo/crates/echo-session-client/src/lib.rs -/Users/james/git/echo/crates/echo-session-proto/src/lib.rs -/Users/james/git/echo/crates/warp-geom/src/lib.rs -/Users/james/git/echo/specs/spec-000-rewrite/src/lib.rs -/Users/james/git/echo/crates/echo-registry-api/src/lib.rs -/Users/james/git/echo/crates/echo-wasm-bindings/src/lib.rs -/Users/james/git/echo/crates/echo-dind-harness/src/lib.rs -/Users/james/git/echo/crates/echo-dry-tests/src/lib.rs -/Users/james/git/echo/crates/warp-ffi/src/lib.rs -/Users/james/git/echo/crates/warp-wasm/src/lib.rs -/Users/james/git/echo/crates/echo-dind-tests/src/lib.rs -/Users/james/git/echo/crates/echo-wasm-abi/src/lib.rs -/Users/james/git/echo/crates/warp-core/src/lib.rs -⎿  Search(pattern: "worker|Worker|queue|Queue|executor|Executor", glob: "\*\*/\_.rs", head_limit: 30) - -⎿  Read(/Users/james/git/echo/crates/warp-core/src/footprint.rs) - -⎿  Read(/Users/james/git/echo/crates/warp-core/tests/footprint_independence_tests.rs) - -⎿  Read(/Users/james/git/echo/crates/warp-core/src/scheduler.rs) - -⎿  Read 251 lines - -⎿  Read 99 lines - -⎿  Read 1305 lines - -⎿    ⎿  Found 30 files -crates/warp-benches/src/bin/sim_sharded_store.rs -crates/warp-benches/src/bin/sim_parallel_executor.rs -crates/warp-core/tests/materialization_determinism.rs -crates/warp-core/tests/materialization_spec_police.rs -crates/warp-core/src/engine_impl.rs -crates/warp-core/src/materialization/emission_port.rs -crates/warp-core/src/materialization/port.rs -crates/warp-core/src/materialization/emit_key.rs -crates/warp-core/src/lib.rs -crates/warp-core/src/graph.rs -crates/echo-dind-tests/src/rules.rs -crates/echo-dind-harness/src/dind.rs -crates/warp-wasm/src/lib.rs -crates/warp-core/tests/reserve_gate_tests.rs -crates/warp-core/tests/tick_receipt_tests.rs -crates/warp-core/tests/permutation_commute_tests.rs -crates/warp-core/tests/property_commute_tests.rs -crates/warp-core/tests/dpo_concurrency_litmus.rs -crates/warp-core/tests/duplicate_rule_registration_tests.rs -crates/warp-core/tests/engine_motion_negative_tests.rs -crates/warp-core/tests/dispatch_inbox.rs -crates/warp-core/src/scheduler.rs -crates/warp-core/src/inbox.rs -crates/warp-benches/benches/scheduler_drain.rs -crates/echo-dry-tests/src/demo_rules.rs -crates/echo-dry-tests/src/lib.rs -crates/echo-dry-tests/src/rules.rs -crates/warp-core/src/snapshot.rs -crates/warp-core/src/attachment.rs -crates/warp-viewer/src/gpu.rs -⎿  Read(/Users/james/git/echo/crates/warp-benches/src/bin/sim_sharded_store.rs) - -⎿  Read(/Users/james/git/echo/crates/warp-benches/src/bin/sim_parallel_executor.rs) - -⎿  Search(pattern: "parallel|Parallel|concurrent|Concurrent|thread|Thread", glob: "\*_/_.rs", head_limit: 20) - -⎿  Read 103 lines - -⎿  Read 83 lines - -⎿    ⎿  Found 12 files -crates/warp-benches/src/bin/sim_sharded_store.rs -crates/warp-benches/src/bin/sim_parallel_executor.rs -crates/warp-core/src/materialization/bus.rs -crates/warp-core/src/materialization/mod.rs -crates/warp-core/src/materialization/emit_key.rs -crates/warp-core/src/wsc/write.rs -crates/warp-core/src/receipt.rs -crates/echo-session-client/examples/publish_pulse.rs -crates/warp-viewer/src/ui_effects.rs -crates/warp-core/src/sandbox.rs -crates/warp-benches/benches/scheduler_adversarial.rs -crates/echo-session-client/src/lib.rs -⎿  Read(/Users/james/git/echo/crates/warp-core/src/inbox.rs · lines 1-100) - -⎿  Search(pattern: "pending.*queue|work.*queue|task._queue|dispatch", glob: "\*\*/_.rs", head_limit: 15) - -⎿  Read(/Users/james/git/echo/crates/warp-core/src/lib.rs · lines 1-150) - -⎿  Read 100 lines - -⎿  Read 141 lines - -⎿    ⎿  Found 10 files -crates/warp-core/src/engine_impl.rs -crates/warp-core/src/materialization/port.rs -crates/echo-dind-tests/src/lib.rs -crates/echo-dind-harness/src/dind.rs -crates/warp-wasm/src/lib.rs -crates/warp-core/tests/dispatch_inbox.rs -crates/warp-core/src/payload.rs -crates/warp-core/src/scheduler.rs -crates/warp-core/src/inbox.rs -crates/echo-dry-tests/src/engine.rs -⎿  Read(/Users/james/git/echo/crates/warp-core/src/engine_impl.rs · lines 1-200) - -⎿  Search(pattern: "GenSet|ActiveFootprint|RadixScheduler|LegacyScheduler", glob: "\*_/_.rs") - -⎿  Read 200 lines - -⎿    ⎿  Found 2 files -crates/warp-core/src/engine\*impl.rs -crates/warp-core/src/scheduler.rs -⎿  Search(pattern: "\*\*/materialization/\_.rs") - -⎿  Read(/Users/james/git/echo/crates/warp-core/src/materialization/mod.rs · lines 1-150) - -⎿  Read 65 lines - -⎿    ⎿  Found 9 files -/Users/james/git/echo/crates/warp-core/src/materialization/emit_key.rs -/Users/james/git/echo/crates/warp-core/src/materialization/port.rs -/Users/james/git/echo/crates/warp-core/src/materialization/emission_port.rs -/Users/james/git/echo/crates/warp-core/src/materialization/channel.rs -/Users/james/git/echo/crates/warp-core/src/materialization/reduce_op.rs -/Users/james/git/echo/crates/warp-core/src/materialization/scoped_emitter.rs -/Users/james/git/echo/crates/warp-core/src/materialization/mod.rs -/Users/james/git/echo/crates/warp-core/src/materialization/bus.rs -/Users/james/git/echo/crates/warp-core/src/materialization/frame.rs -⎿  Bash(find /Users/james/git/echo -name "\*.rs" -type f | xargs grep -l "reserve\|conflict\|dispatch" | head -20) -Waiting… -⎿  Read(/Users/james/git/echo/crates/warp-benches/benches/scheduler_drain.rs · lines 1-100) - -⎿  Read 100 lines - -⎿  /Users/james/git/echo/crates/echo-session-ws-gateway/src/main.rs -/Users/james/git/echo/crates/echo-dry-tests/src/rules.rs -/Users/james/git/echo/crates/echo-dry-tests/src/demo_rules.rs -/Users/james/git/echo/crates/echo-dry-tests/src/engine.rs -/Users/james/git/echo/crates/warp-core/tests/materialization_spec_police.rs -/Users/james/git/echo/crates/warp-core/tests/reserve_gate_tests.rs -/Users/james/git/echo/crates/warp-core/tests/tick_receipt_tests.rs -/Users/james/git/echo/crates/warp-core/tests/duplicate_rule_registration_tests.rs -/Users/james/git/echo/crates/warp-core/tests/materialization_determinism.rs -/Users/james/git/echo/crates/warp-core/tests/footprint_independence_tests.rs -/Users/james/git/echo/crates/warp-core/tests/dispatch_inbox.rs -/Users/james/git/echo/crates/warp-core/tests/nan_exhaustive_tests.rs -/Users/james/git/echo/crates/warp-core/tests/dpo_concurrency_litmus.rs -/Users/james/git/echo/crates/warp-core/tests/materialization_engine_integration.rs -/Users/james/git/echo/crates/warp-core/src/telemetry.rs -/Users/james/git/echo/crates/warp-core/src/wsc/types.rs -/Users/james/git/echo/crates/warp-core/src/wsc/write.rs -/Users/james/git/echo/crates/warp-core/src/wsc/build.rs -/Users/james/git/echo/crates/warp-core/src/wsc/read.rs -/Users/james/git/echo/crates/warp-core/src/wsc/validate.rs -⎿  Read(/Users/james/git/echo/crates/warp-core/tests/reserve_gate_tests.rs · lines 1-150) - -⎿  Bash(grep -r "DeterministicScheduler\|RadixScheduler\|ActiveFootprint" /Users/james/git/echo/crates/warp-core/src ---include="\*.rs" -n | head -30) - - Waiting… - -⎿  Read 41 lines - -⎿  /Users/james/git/echo/crates/warp-core/src/scheduler.rs:26:pub(crate) struct ActiveFootprints { -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:43:impl ActiveFootprints { -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:59:pub(crate) struct RadixScheduler { -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:64: pub(crate) active: HashMap, -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:100:impl RadixScheduler { -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:135: let active = -self.active.entry(tx).or_insert_with(ActiveFootprints::new); -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:158: fn has_conflict(active: &ActiveFootprints, pr: -&PendingRewrite) -> bool { -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:239: fn mark_all(active: &mut ActiveFootprints, pr: -&PendingRewrite) { -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:610:pub(crate) struct DeterministicScheduler { -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:617:impl std::fmt::Debug for DeterministicScheduler { -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:619: f.debug_struct("DeterministicScheduler") -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:628: Radix(RadixScheduler), -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:632:impl Default for DeterministicScheduler { -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:641:impl DeterministicScheduler { -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:644: SchedulerKind::Radix => -SchedulerImpl::Radix(RadixScheduler::default()), -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:754: let mut sched = RadixScheduler::default(); -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:785: let mut sched = RadixScheduler::default(); -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:842: let mut sched = RadixScheduler::default(); -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:894: let mut sched = RadixScheduler::default(); -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:946: let mut sched = RadixScheduler::default(); -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:996: let mut sched = RadixScheduler::default(); -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:1049: let mut sched = RadixScheduler::default(); -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:1107: let mut sched = RadixScheduler::default(); -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:1178: let mut sched = RadixScheduler::default(); -/Users/james/git/echo/crates/warp-core/src/scheduler.rs:1265: let mut sched = RadixScheduler::default(); -/Users/james/git/echo/crates/warp-core/src/engine_impl.rs:19:use crate::scheduler::{DeterministicScheduler, -PendingRewrite, RewritePhase, SchedulerKind}; -/Users/james/git/echo/crates/warp-core/src/engine_impl.rs:316: scheduler: DeterministicScheduler, -/Users/james/git/echo/crates/warp-core/src/engine_impl.rs:477: scheduler: DeterministicScheduler::new(kind, -telemetry), -/Users/james/git/echo/crates/warp-core/src/engine_impl.rs:623: scheduler: DeterministicScheduler::new(kind, -telemetry), -/Users/james/git/echo/crates/warp-core/src/engine_impl.rs:1464: // predicate (`RadixScheduler::has_conflict` in -`scheduler.rs`). If one - -⎿  Response: -Perfect! Now let me create a comprehensive summary based on my research. - - Summary of Existing Footprint, Independence Checking, and Scheduling Infrastructure - - Based on my thorough search of the Echo codebase, here is what I found: - - 1. Footprint Struct and Conflict Detection - - Location: /Users/james/git/echo/crates/warp-core/src/footprint.rs - - A comprehensive footprint infrastructure exists: - - - Footprint struct (lines 131-152): Captures read/write sets for rewrites across: - - Nodes (read: n_read, write: n_write) - - Edges (read: e_read, write: e_write) - - Attachment slots (read: a_read, write: a_write) - - Boundary ports (input: b_in, output: b_out) - - Coarse factor_mask (u64) for O(1) prefiltering - - Set data structures: - - IdSet (lines 31-63): BTreeSet-backed ordered sets for nodes/edges with deterministic iteration - - PortSet (lines 65-97): BTreeSet-backed set for boundary port keys - - AttachmentSet (lines 99-129): BTreeSet-backed set for attachment slots - - Independence checking Footprint::independent() (lines 161-191): - - Fast path: checks disjoint factor_mask first - - Then checks boundary port intersections - - Then checks edge conflicts (write-write and write-read) - - Finally checks attachment and node conflicts (write-write and write-read) - - Symmetric conflict detection with early exits - - Port key packing (lines 206-214): Helper function pack_port_key() to create 64-bit stable port identifiers - - 2. Admission Control and Dependency Checking - - Location: /Users/james/git/echo/crates/warp-core/src/scheduler.rs - - Advanced scheduling infrastructure with two implementations: - - RadixScheduler (Primary) - - - PendingRewrite struct (lines 68-82): Represents pending rewrites with: - - Rule ID, compact rule handle - - Scope hash and scope node key - - Footprint for conflict checking - - RewritePhase state machine - - ActiveFootprints struct (lines 26-55): Generation-stamped sets for O(1) conflict detection: - - Separate tracking: nodes_written, nodes_read, edges_written, edges_read - - attachments_written, attachments_read, ports - - All use GenSet for O(1) lookups - - RadixScheduler struct (lines 59-65): - - pending: HashMap for O(1) enqueue, O(n) drain - - active: HashMap for O(m) independence checking - - Reserve method (lines 134-143): - - has_conflict() (lines 158-236): Checks full footprint independence - - Node write conflicts with prior writes OR reads - - Node read conflicts only with prior writes - - Same logic for edges and attachments - - Port conflicts: ANY intersection fails - - mark_all() (lines 239-278): Atomically marks all resources when reserved - - Returns bool: true=reserved, false=aborted on conflict - - No partial marking on conflict (atomic semantics) - - LegacyScheduler (Comparison) - - - Uses BTreeMap for pending queues - - Uses Vec for active frontier - - O(k) scan over reserved footprints using Footprint::independent() - - DeterministicScheduler wrapper (lines 610-712) - - - Swappable implementation (Radix vs Legacy via SchedulerKind enum) - - Telemetry integration with counters for reserved/conflict tracking - - Methods: enqueue(), drain_for_tx(), reserve(), finalize_tx() - - GenSet (lines 510-535) - - - Generation-stamped set for O(1) conflict detection without clearing - - Each transaction gets a new generation - - Tracks which generation last saw each key - 3. Deterministic Ordering/Scheduling +# ADR-0007-impl: warp-core Implementation of BOAW Storage + Parallel Execution - Location: /Users/james/git/echo/crates/warp-core/src/scheduler.rs (lines 287-480) +- **Status:** Accepted +- **Date:** 2026-01-19 +- **Parent:** ADR-0007 (BOAW Storage + Execution + Merge + Privacy) +- **Crate:** `crates/warp-core` (36 source modules) - High-performance O(n) radix-based scheduler: +--- - - PendingTx

struct (lines 301-327): - - index: FxHashMap for O(1) last-wins deduplication - - thin: Vec with (scope_be32, rule_id, nonce, handle) - - fat: Vec

{ + next_nonce: u32, + index: FxHashMap<([u8; 32], u32), usize>, // last-wins dedupe + thin: Vec, // 24B + 4B handle, radix-sortable + fat: Vec>, // payload storage + scratch: Vec, // radix sort double-buffer + counts16: Vec, // 65536-bucket histogram +} +``` + +### 2.4 WarpOp + WarpTickPatchV1 (`src/tick_patch.rs`) + +Eight operation variants define all state mutations: + +```rust +pub enum WarpOp { + OpenPortal { key, child_warp, child_root, init }, + UpsertWarpInstance { instance }, + DeleteWarpInstance { warp_id }, + UpsertNode { node, record }, + DeleteNode { node }, + UpsertEdge { warp_id, record }, + DeleteEdge { warp_id, from, edge_id }, + SetAttachment { key, value }, +} +``` -⎿  Read(/Users/james/git/echo/crates/warp-core/src/tick_patch.rs · lines 1-150) +Each op carries a `WarpOpKey` for canonical merge ordering. -⎿  Read 150 lines +```rust +pub enum SlotId { + Node(NodeKey), + Edge(EdgeKey), + Attachment(AttachmentKey), + Port(WarpScopedPortKey), +} +``` -⎿  Read 150 lines +### 2.5 MaterializationBus + EmitKey + ScopedEmitter (`src/materialization/`) -⎿  Read(/Users/james/git/echo/crates/warp-core/src/tick_patch.rs · lines 290-439) +Order-independent output collection using BTreeMap for deterministic +finalization. Duplicate detection rejects identical `(channel, EmitKey)` pairs. +Thread-unsafe by design (single-threaded tick execution). -⎿  Read(/Users/james/git/echo/crates/warp-core/src/engine_impl.rs · lines 820-899) +```rust +pub struct MaterializationBus { + pending: RefCell>>>, + policies: BTreeMap, +} -⎿  Read 150 lines +pub struct EmitKey { + pub scope_hash: Hash, // content hash of scope node + pub rule_id: u32, // compact rule ID + pub subkey: u32, // multi-emission differentiator +} +``` -⎿  Read 80 lines +`ScopedEmitter` prevents key forgery: the engine fills `scope_hash` and +`rule_id`; rules cannot construct `EmitKey` directly. -⎿  Read(/Users/james/git/echo/crates/warp-core/src/tick_patch.rs · lines 750-849) - -⎿  Search(pattern: "fn compute\*patch_digest", glob: "\*\*/\_.rs", output_mode: "content", head_limit: 20) - -⎿  Read 100 lines - -⎿    ⎿  Found 1 line -crates/warp-core/src/tick_patch.rs:755:fn compute_patch_digest_v2( -⎿  Read(/Users/james/git/echo/docs/rfc/mat-bus-finish.md) - -⎿  Read(/Users/james/git/echo/crates/warp-core/src/materialization/reduce_op.rs) - -⎿  Read 661 lines - -⎿  Read 385 lines - -⎿  Response: -Perfect! Now I have comprehensive information. Let me create a detailed research report. - - Research Summary: Materialization Bus Infrastructure in WARP - - Based on my thorough search of the warp-core codebase, here's what I found regarding materialization, the bus - infrastructure, and hash computation: - - 1. MaterializationBus Infrastructure - - Location: /Users/james/git/echo/crates/warp-core/src/materialization/bus.rs - - The MaterializationBus is a tick-scoped internal infrastructure that collects emissions from rewrite rules during - a tick: - - - Core Data Structure: Uses BTreeMap>> for order-independent storage - - Thread Safety: Intentionally not thread-safe (!Sync) using RefCell for interior mutability - by design, since - rewrite engine processes rules sequentially within a tick - - Key Methods: - - emit(channel, emit_key, data) - Collects emissions, rejects duplicate (channel, EmitKey) pairs - - finalize() - Post-commit resolution according to each channel's policy, returns FinalizeReport - - clear() - Clears pending emissions (for abort path) +### 2.6 WSC Format Types (`src/wsc/types.rs`) - Order Independence (Confluence Safety): The BTreeMap ordering ensures finalization iterates in deterministic - (canonical) order regardless of rewrite execution order, preserving confluence. +Fixed-size columnar rows, all `#[repr(C)]` + `Pod` + `Zeroable` for +zero-copy access via `bytemuck::try_cast_slice()`. - 2. Graph State Materialization +```rust +#[repr(C)] +pub struct WscHeader { // 128 bytes + pub magic: [u8; 8], // b"WSC\x00\x01\x00\x00\x00" + pub schema_hash: Hash, // 32 + pub tick_le: u64, + pub warp_count_le: u64, + pub warp_dir_off_le: u64, + pub reserved: [u8; 64], +} - Location: /Users/james/git/echo/crates/warp-core/src/materialization/mod.rs (architecture diagram) +#[repr(C)] +pub struct NodeRow { // 64 bytes + pub node_id: Hash, // 32 + pub node_type: Hash, // 32 +} - The materialization system follows a hexagonal architecture: +#[repr(C)] +pub struct EdgeRow { // 128 bytes + pub edge_id: Hash, // 32 + pub from_node_id: Hash, // 32 + pub to_node_id: Hash, // 32 + pub edge_type: Hash, // 32 +} - External World (ViewOps frames) - ▲ - │ - MaterializationPort (Boundary API) - │ - MaterializationBus (Internal Runtime) - │ - Engine (commit → finalize) +#[repr(C)] +pub struct AttRow { // 56 bytes + pub tag: u8, // 1=Atom, 2=Descend + pub reserved0: [u8; 7], + pub type_or_warp: Hash, // TypeId or WarpId + pub blob_off_le: u64, + pub blob_len_le: u64, +} +``` + +### 2.7 Snapshot + TickReceipt (`src/snapshot.rs`, `src/receipt.rs`) + +```rust +pub struct Snapshot { + pub root: NodeKey, + pub hash: Hash, + pub state_root: Hash, + pub parents: Vec, + pub plan_digest: Hash, + pub decision_digest: Hash, + pub rewrites_digest: Hash, + pub patch_digest: Hash, + pub policy_id: u32, + pub tx: TxId, +} +``` - Current Implementation: - - Emissions are stored in channels keyed by EmitKey - - Materialization frames are encoded as length-prefixed binary ("MBUS" format) - - Support for three channel policies: Log (default), StrictSingle, and Reduce(op) +`TickReceipt` records the causality poset witness for each tick: - 3. EmitKey: Order-Independent Emission Identification +```rust +pub struct TickReceipt { + tx: TxId, + entries: Vec, // canonical plan order + blocked_by: Vec>, // causality poset witness + digest: Hash, // commits to entries only +} +``` - Location: /Users/james/git/echo/crates/warp-core/src/materialization/emit_key.rs +--- - The EmitKey ensures deterministic ordering: +## 3) Tick Execution Pipeline - pub struct EmitKey { - pub scope_hash: Hash, // Hash of scope node where rewrite applied - pub rule_id: u32, // Compact rule identifier - pub subkey: u32, // Differentiator for multiple emissions per rule - } +The pipeline runs once per tick, transforming ingress into a committed +snapshot. - Ordering: Lexicographic (scope_hash, rule_id, subkey) matching scheduler's canonical ordering +```text + ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ + │ 1.Ingest │───►│ 2.Match │───►│ 3.Radix │───►│ 4.Reserve│ + │ + Dispatch│ │ + Plan │ │ Drain │ │ (GenSet) │ + └──────────┘ └──────────┘ └──────────┘ └──────────┘ + │ + ┌──────────┐ ┌──────────┐ ▼ + │ 6.Commit │◄───│ 5.Execute│◄────────────────────────┘ + │ + Hash │ │ Parallel │ + └──────────┘ └──────────┘ +``` - Subkey Usage: For rules emitting multiple items (e.g., iterating children), use stable subkey derived from item - hash via EmitKey::subkey_from_hash(h: &Hash) -> u32 +**Step 1 — Ingest + Dispatch.** Intents arrive via the inbox subsystem. +`ingest_inbox_event()` creates event nodes under the `sim/inbox` path and +attaches pending edges from opaque intent bytes. `dispatch_inbox_rule()` +drains pending intents and acks them. - 4. Channel Policies and Reduce Operations +**Step 2 — Rule Matching + Footprint.** For each candidate rule/scope pair, +the engine calls `MatchFn` to test applicability and `FootprintFn` to compute +the warp-scoped resource set. Each accepted match becomes a `PendingRewrite` +with fields `(scope_hash, rule_id, compact_rule, footprint, origin)`. - Location: /Users/james/git/echo/crates/warp-core/src/materialization/channel.rs and - /Users/james/git/echo/crates/warp-core/src/materialization/reduce_op.rs +**Step 3 — Radix Drain.** `PendingTx::drain_in_order()` produces candidates +in a canonical byte-lexicographic order. The ordering key is +`(scope_hash[32B BE], rule_id[4B], nonce[4B])` — see §4 for algorithm details. - Channel Policies: +**Step 4 — Independence Check + Reservation.** Candidates are tested against +`ActiveFootprints` using `GenSet`-based O(1) conflict detection. Independent +candidates are atomically reserved (mark-all-or-nothing). Conflicting +candidates are deferred or rejected per `ConflictPolicy`. +**Step 5 — Parallel Execution.** Admitted rewrites are dispatched to workers. +Each worker receives a `GraphView` (read-only snapshot projection) and emits +ops into a thread-local `&mut TickDelta`. There is no shared mutable state — +determinism is enforced by the merge step, not execution order. - ┌──────────────┬─────────────────────────────────────────────────┐ - │ Policy │ Behavior │ - ├──────────────┼─────────────────────────────────────────────────┤ - │ Log │ All emissions in EmitKey order, length-prefixed │ - ├──────────────┼─────────────────────────────────────────────────┤ - │ StrictSingle │ Error if >1 emission (catches bugs) │ - ├──────────────┼─────────────────────────────────────────────────┤ - │ Reduce(op) │ Merge via deterministic operation │ - └──────────────┴─────────────────────────────────────────────────┘ - ReduceOp Variants: +**Step 6 — Canonical Merge + Commit.** Thread-local deltas are merged +canonically (§6.3). The merged ops are applied to produce the next snapshot. +The engine computes `state_root`, `patch_digest`, and `commit_hash` +(§7), then stores the `Snapshot` and `TickReceipt`. - Commutative Monoids (permutation-invariant): - - Sum - u64 little-endian addition (empty→[0;8]) - - Max, Min - lexicographic comparison - - BitOr, BitAnd - bitwise operations with padding/truncation +--- - Order-Dependent (deterministic via EmitKey order, NOT commutative): - - First - take first by EmitKey order - - Last - take last by EmitKey order - - Concat - concatenate all in order +## 4) Radix Scheduler Algorithm - All operations include is_commutative() method for runtime classification. +**Ref:** `src/scheduler.rs` - 5. State Root and Commit Hash Computation +The scheduler uses LSD (least-significant-digit) radix sort to produce a +canonical ordering of pending rewrites. This is O(n) in the number of +candidates with zero comparisons. - Location: /Users/james/git/echo/crates/warp-core/src/snapshot.rs +### Sort Parameters - State Root (compute_state_root): - - BLAKE3 digest over canonical byte stream of entire reachable graph state - - Deterministic ordering: nodes visited in ascending NodeId order, edges sorted by EdgeId - - Includes reachability across instances via Descend attachments - - Fixed-size, architecture-independent encoding: - - 32-byte ids (NodeId, TypeId, EdgeId) - - 1 byte presence tag + 32 bytes type_id + 8-byte LE length + payload bytes +- **Key:** 40 bytes = `scope_hash[32B] || rule_id[4B] || nonce[4B]` +- **20 LSD passes** with 16-bit digits (65,536 buckets per pass) +- **Pass order:** nonce bytes [0,1], rule bytes [0,1], scope pairs [15..0] +- **Result:** byte-lexicographic order `(scope_be32, rule_id, nonce)` - Commit Hash (v2): compute_commit_hash_v2(state_root, parents, patch_digest, policy_id) - - Commits to: state_root + explicit parents + patch_digest + policy_id - - Uses BLAKE3 hashing +### Small-Batch Fallback - 6. Tick Patch and Patch Digest Computation +When the candidate count is below `SMALL_SORT_THRESHOLD = 1024`, the scheduler +falls back to comparison sort, which is faster for small n due to lower +constant factors. - Location: /Users/james/git/echo/crates/warp-core/src/tick_patch.rs +### Deduplication - WarpTickPatchV1: The boundary artifact for deterministic replay +`PendingTx` uses an `FxHashMap` keyed on `(scope_hash, compact_rule)` with +last-wins semantics. `FxHashMap` is safe here because it is internal to the +scheduler and never crosses a determinism boundary — iteration order over +the map is irrelevant since all entries are drained through the radix sort. - Patch Digest (compute_patch_digest_v2): Commits to: - - Patch format version (v2) - - policy_id - - rule_pack_id - - commit_status (Committed/Aborted) - - in_slots / out_slots (sorted, deduped) - - ops (canonical delta operations) +--- - Digest Encoding: BLAKE3 of structured format: - - Version as u16 LE - - Policy ID as u32 LE - - Rule pack ID (32 bytes) - - Commit status code (1 byte) - - Encoded slots (tag + ids for Node/Edge/Attachment/Port) - - Encoded ops (tag + operation-specific fields) +## 5) Conflict Detection - Operations (8 types): - - OpenPortal, UpsertWarpInstance, DeleteWarpInstance - - UpsertNode, DeleteNode - - UpsertEdge, DeleteEdge - - UpsertAttachment, DeleteAttachment +**Ref:** `src/footprint.rs`, `src/scheduler.rs`, `src/footprint_guard.rs` - 7. Engine Integration +### Independence Check - Location: /Users/james/git/echo/crates/warp-core/src/engine_impl.rs +`Footprint::independent()` performs 5 resource checks with early exit: - Commit Flow: - 1. engine_impl.rs:829-831 - Bus finalization: let mat_report = self.bus.finalize() - 2. engine_impl.rs:834-843 - Tick patch creation with ops from state diff - 3. engine_impl.rs:843-859 - Parallel computation: - - patch_digest = patch.digest() - - state_root = compute_state_root(&self.state, &self.current_root) - - hash = compute_commit_hash_v2(&state_root, &parents, &patch_digest, policy_id) - 4. Creates Snapshot with all digests and transaction tracking +1. **Factor mask (O(1)):** `(self.factor_mask & other.factor_mask) == 0` → + independent, skip remaining checks +2. **Boundary ports:** any `b_in`/`b_out` intersection → conflict +3. **Edges:** `e_write ∩ e_write`, `e_write ∩ e_read`, or + reverse → conflict +4. **Attachments:** `a_write ∩ a_write`, `a_write ∩ a_read`, or + reverse → conflict +5. **Nodes:** `n_write ∩ n_write`, `n_write ∩ n_read`, or + reverse → conflict - Abort Flow: engine_impl.rs:889-891 - - bus.clear() - clears pending emissions - - last_materialization.clear() - invalidates stale cached state +If all five checks pass, the rewrites are independent. - 8. Determinism Testing & Design Documents +### GenSet Mechanism - ADR-0003 (Determinism Philosophy): /Users/james/git/echo/docs/adr/ADR-0003-Materialization-Bus.md - - Establishes causality-first API (ingress-only writes) - - Bus-first UI via MaterializationPort - - Confluence-safe emission semantics with order-independent storage - - Rejects silent "winner picks" - requires explicit policy +`GenSet` provides O(1) per-key conflict detection without clearing hash +tables between transactions. Each `mark(key)` stamps the current generation; +`contains(key)` checks `seen[key] == gen`. Calling `advance()` increments the +generation counter, logically clearing all marks without touching the map. - RFC: MaterializationBus Completion: /Users/james/git/echo/docs/rfc/mat-bus-finish.md - - Specifies EmissionPort trait (hexagonal boundary) - - ScopedEmitter adapter (auto-fills EmitKey) - - Duplicate (channel, EmitKey) rejection policy - - Comprehensive cross-platform determinism test suite +### Atomic Reservation - 9. Frame Encoding Format +Admission is all-or-nothing per candidate: all resource keys in the footprint +are marked, or none are (if any conflict is found, the reservation is aborted). - Location: /Users/james/git/echo/crates/warp-core/src/materialization/frame.rs +### FootprintGuard (Runtime Enforcement) - Binary frame format ("MBUS"): - magic[4] = "MBUS" (0x4D, 0x42, 0x55, 0x53) - version[2] = 0x0001 (little-endian) - reserved[2] = 0x0000 - length[4] = payload byte length (LE) - payload[len] = channel_id[32] || data[len-32] +`FootprintGuard` validates that executors stay within their declared footprint. +Active in debug builds and when `footprint_enforce_release` feature is enabled +(disabled by `unsafe_graph`). Violations produce a `FootprintViolation` with +one of 9 `ViolationKind` variants: - All multi-byte integers are little-endian. Max payload ≈4GB (u32 limitation). +```rust +pub enum ViolationKind { + NodeReadNotDeclared(NodeId), + EdgeReadNotDeclared(EdgeId), + AttachmentReadNotDeclared(AttachmentKey), + NodeWriteNotDeclared(NodeId), + EdgeWriteNotDeclared(EdgeId), + AttachmentWriteNotDeclared(AttachmentKey), + CrossWarpEmission { op_warp: WarpId }, + UnauthorizedInstanceOp, + OpWarpUnknown, +} +``` - --- - Key Insights: +--- - 1. No Hash Collision Risk at Materialization Level - EmitKey uses scope_hash + rule_id + subkey, making - collisions astronomically unlikely - 2. Determinism by Construction - BTreeMaps enforce canonical ordering at every level; same emissions always - produce same bytes - 3. Confluence-First Design - Silent data loss is prohibited; use explicit policies (Log/StrictSingle/Reduce) - 4. Tick-Scoped Lifecycle - Bus is cleared after finalization, preventing cross-tick contamination - 5. Paper III Integration - Tick patches are replayable delta artifacts sufficient for deterministic state - recovery +## 6) BOAW Parallel Execution (Phase 6B) -⎿  Done (20 tool uses · 94.3k tokens · 53s) +**Ref:** `src/boaw/shard.rs`, `src/boaw/exec.rs`, `src/boaw/merge.rs` -❯ please checkout a git branch called graph-boaw for this work and be sure to git -A and git commit before ending each -of your turns +### 6.1 Virtual Shard Partitioning -∴ Thinking… +```rust +pub const NUM_SHARDS: usize = 256; +const SHARD_MASK: u64 = 255; -The user wants me to: +pub fn shard_of(scope: &NodeId) -> usize { + let first_8 = &scope.0[0..8]; + let val = u64::from_le_bytes(first_8.try_into().unwrap()); + (val & SHARD_MASK) as usize +} +``` -1. Create a new git branch called graph-boaw for the BOAW work -2. Commit any changes before ending each turn +`NUM_SHARDS = 256` is a **frozen protocol constant** — changing it requires a +version bump. The formula takes the first 8 bytes of the NodeId's 32-byte hash, +interprets them as a little-endian u64, and masks to 255. This provides: -Let me first create the branch and then continue with my analysis and implementation plan. +- **Load balance:** 256 virtual shards vs 8–64 typical workers +- **Cache locality:** scope-adjacent rewrites land in the same shard +- **Platform independence:** LE interpretation is explicit -⏺ Let me create the branch for this work first. +### 6.2 Worker Execution -⏺ Bash(git checkout -b graph-boaw) -⎿  Switched to a new branch 'graph-boaw' +`execute_parallel_sharded()` dispatches `ExecItem` work units across workers. +Workers atomically claim shards and execute all items within a claimed shard. -∴ Thinking… +```rust +pub struct ExecItem { + pub exec: ExecuteFn, + pub scope: NodeId, + pub origin: OpOrigin, +} -Good, branch created. Now let me compile my analysis into a comprehensive implementation plan. I have a lot of context from -the subagent explorations. +pub enum WorkerResult { + Success(TickDelta), + Poisoned(PoisonedDelta), + MissingStore(WarpId), +} +``` -Let me summarize what I've learned: +If a rule panics during execution, `catch_unwind` captures the panic and wraps +it as `PoisonedDelta`. The panic is surfaced at merge time as a typed error, +not an uncontrolled crash. -Current State +When `footprint_enforce_release` is enabled, `GraphView::new_guarded(store, guard)` +wraps reads, and `check_op()` validates each emitted op post-execution. -1. GraphStore (graph.rs, 568 lines): In-memory BTreeMap-based store with nodes, edges, attachments. Supports canonical - hashing via canonical_state_hash(). -2. WSC Format (wsc/): Mature zero-copy snapshot format with: - - WscFile / WarpView for reading - - write_wsc_one_warp for writing - - build_one_warp_input() for GraphStore → WSC conversion - - All Pod types, 8-byte aligned, ready for mmap +### 6.3 Canonical Merge -3. Footprint Infrastructure (footprint.rs, 250 lines): Already has: - - Footprint struct with read/write sets for nodes, edges, attachments, ports - - factor_mask for O(1) prefiltering - - independent() method for conflict detection +The merge algorithm guarantees deterministic output regardless of worker count +or execution order: -4. Scheduler (scheduler.rs, 1304 lines): RadixScheduler with: - - PendingRewrite with footprint and phase tracking - - ActiveFootprints with GenSet for O(1) conflict detection - - O(n) radix sort for deterministic ordering - - try_reserve() for admission control +1. **Flatten:** Collect all ops from all worker deltas with their `OpOrigin` +2. **Sort:** Order by `(WarpOpKey, OpOrigin)` for canonical ordering +3. **Validate new warps:** Collect `OpenPortal` ops with `PortalInit::Empty`; + reject same-tick writes to newly created warps +4. **Dedupe:** Identical ops from different origins are collapsed to one +5. **Conflict detection:** Divergent ops on the same `WarpOpKey` → + `MergeConflict` -5. TickPatch (tick_patch.rs, 1694 lines): Has: - - WarpOp enum (UpsertNode, DeleteNode, UpsertEdge, DeleteEdge, SetAttachment, etc.) - - WarpTickPatchV1 with ops, in_slots, out_slots, patch_digest - - **[Phase 5 Complete]**: Now used for execution emissions via TickDelta +```rust +pub enum MergeError { + Conflict(Box), + PoisonedDelta(PoisonedDelta), + WriteToNewWarp { warp_id: WarpId, op_origin: OpOrigin, op_kind: &'static str }, +} +``` -6. Engine (engine_impl.rs, 1806 lines): **[Phase 5 Complete]**: Current flow: - - Executors receive `&GraphView` (read-only) and emit ops to `&mut TickDelta` - - No GraphStore mutation during execution - - State updated post-execution via `apply_to_state()` +**Policy:** Merge conflicts are bugs — they mean the footprint model lied. +There is no recovery path; the tick is aborted. -7. Materialization (materialization/): Mature bus system for output emissions with deterministic ordering. +--- -What ADR-0007 Requires +## 7) State Root Hashing -The ADR outlines a migration plan (status as of 2026-01-18): +**Ref:** `src/snapshot.rs`, `src/graph.rs`, `src/tick_patch.rs` -1. Step 1: Introduce TickDelta + canonical merge alongside existing GraphStore **[COMPLETE]** -2. Step 2: Change executors to emit ops into TickDelta (stop writing GraphStore) **[COMPLETE - Phase 5]** -3. Step 3: Implement SnapshotBuilder that applies TickDelta to produce next snapshot **[COMPLETE - Phase 4]** -4. Step 4: Wire WSC writer as canonical snapshot output **[COMPLETE]** -5. Step 5: Read-only execution - GraphView instead of &mut GraphStore **[COMPLETE - Phase 5]** -6. Step 6: Add collapse/merge phase with typed registry **[PENDING]** -7. Step 7: Add mind/diagnostics enforcement gates **[PENDING]** +All hashing uses BLAKE3. -My Implementation Plan +### State Root (`canonical_state_hash()`) -Based on the ADR's sequencing section (lines 1062-1105), here's what I'll propose: +Deterministic BFS to collect the reachable graph across warp instances: -Phase 1: ADR + Test Skeletons (First Commit) +- Header: `b"DIND_STATE_HASH_V2\0"` +- Per-instance: `warp_id`, `root_node_id`, parent attachment key +- Nodes in ascending `NodeId` order, each contributing `(node_id, type_id)` +- Edges per source node, sorted by `EdgeId`, filtered to reachable targets +- Attachments: ATOM → `(type_id, blob_len as u64, bytes)`, DESC → `(warp_id)` -- The ADR is already written - it's the file I read -- Add the test skeletons from the ADR (lines 604-1043) -- These tests will fail initially - that's intentional +Node and edge counts are encoded as `u64` for version stability. - Phase 2: Wire TestHarness (Second Commit) +### Commit Hash -- Implement TestHarness against the test structure, following the ADR's guidance on how tests should be organized and what - they should validate. +```rust +pub fn compute_commit_hash_v2( + state_root: &Hash, + parents: &[Hash], + patch_digest: &Hash, + policy_id: u32, +) -> Hash +``` - The next phases build on this foundation: TickDelta collects operations during execution rather than mutating state - directly, then SnapshotBuilder applies those operations to generate the next snapshot and write WSC bytes. After that, I'll - add per-worker delta collection and prove the system produces consistent results regardless of worker count through - targeted tests. - Now let me synthesize all the exploration results into a comprehensive implementation plan. +The commit hash is: +`H(version_tag || parents_len || parents || state_root || patch_digest || policy_id)` - *** +This commits to the full causal chain: the prior state (parents), the +resulting state (state_root), and the operations that produced it +(patch_digest), under a specific policy. - BOAW Implementation Plan +### Patch Digest - ★ Insight ───────────────────────────────────── - Architecture Context: The codebase already has strong foundations for BOAW: +Computed from the versioned encoding of merged ops + slots + metadata in the +`WarpTickPatchV1`. This is separate from `state_root` so that two commits +can be compared either by "did they reach the same state?" or "did they apply +the same operations?" -- GraphStore uses BTreeMaps for deterministic iteration -- Footprint infrastructure already tracks read/write sets with factor_mask prefiltering -- RadixScheduler provides O(n) deterministic ordering and O(1) conflict detection via GenSet -- TickPatch has WarpOp enum for delta operations (now used for execution emissions via TickDelta) -- WSC is production-ready with zero-copy Pod types, ready for mmap +--- - The key pivot was: move from "mutate GraphStore → diff after" to "emit ops into TickDelta → apply to snapshot" **[COMPLETE - Phase 5]** - ───────────────────────────────────────────────── +## 8) WSC Build Process - *** +**Ref:** `src/wsc/build.rs`, `src/wsc/write.rs`, `src/wsc/view.rs` - Phase 1: Test Skeletons (This Commit) +### Canonical Ordering - Goal: Establish the contract + drill sergeant tests +The build process in `build_one_warp_input()` enforces strict ordering: - Create test files from ADR-0007 Section 14: +1. **Nodes** — iterated from `GraphStore`'s `BTreeMap`, already sorted by + `NodeId` +2. **Edges (global)** — all edges collected, then sorted by `EdgeId` via + `.sort_by_key(|e| e.id)` +3. **Per-node outbound edges** — for each node (in `NodeId` order), outgoing + edges are sorted by `EdgeId` and appended with an index `Range { start, len }` +4. **Attachments** — node attachments follow node order; edge attachments + follow global edge order -- tests/common/mod.rs - XorShift64 RNG, TestHarness trait, Scenario enum -- tests/boaw_end_to_end.rs - The "god test" for permutation invariance -- tests/boaw_determinism.rs - Serial vs parallel equivalence -- tests/boaw_cow.rs - Unlink semantics, overlay precedence -- tests/boaw_footprints.rs - Independence checks, bucket enforcement -- tests/boaw_merge.rs - Collapse/merge determinism -- tests/boaw_privacy.rs - Mind mode enforcement +### Blob Arena Alignment - These tests will intentionally panic until we wire the BOAW APIs. +All blob payloads are 8-byte aligned before appending: `(len + 7) & !7`. +This prepares the blob section for mmap and SIMD consumers. All `Range`, +`AttRow`, and index fields are stored little-endian. - *** +### Zero-Copy Read - Phase 2: Wire TestHarness to Current Engine +`WscFile` owns the raw `Vec`. `WarpView<'a>` holds lifetime-bound +slices directly into that buffer — no heap allocations at access time: - Goal: Tests compile and run against current pipeline +```rust +pub struct WarpView<'a> { + pub nodes: &'a [NodeRow], + pub edges: &'a [EdgeRow], + pub out_index: &'a [Range], + pub out_edges: &'a [OutEdgeRef], + pub node_atts_index: &'a [Range], + pub node_atts: &'a [AttRow], + pub edge_atts_index: &'a [Range], + pub edge_atts: &'a [AttRow], + pub blobs: &'a [u8], +} +``` - Implement TestHarness to: +All slice casts use `bytemuck::try_cast_slice()`, which is a safe transmutation +requiring `T: Pod`. Node/edge lookup is O(log n) via `binary_search_by_key`. -- Build base snapshots from current GraphStore -- Execute via current engine (both serial and parallel call serial for now) -- Return real hashes so test runner is alive +### Structural Validation (`src/wsc/validate.rs`) - *** +`validate_wsc()` runs 6 checks per warp instance: - Phase 3: TickDelta for Execution (The Real Pivot) +1. Index range bounds (all three index tables within data table bounds) +2. Node ordering (strict ascending by `NodeId`) +3. Edge ordering (strict ascending by `EdgeId`) +4. Root node present (binary search; zero root OK when nodes is empty) +5. Attachment validity (tag, reserved bytes, blob bounds, DESCEND invariants) +6. Out-edge reference validity (edge index within bounds, safe u64→usize cast) - Goal: Executors emit ops instead of mutating shared state +--- - Key Changes: - 1. Create TickDelta struct in src/tick_delta.rs: - pub struct TickDelta { - pub upsert_nodes: Vec<(NodeId, NodeRecord)>, - pub delete_nodes: Vec, - pub upsert_edges: Vec<(EdgeId, EdgeRecord)>, - pub delete_edges: Vec, - pub set_attachments: Vec<(AttachmentKey, Option)>, - } - 2. Modify ExecuteFn signature to take: - - &GraphView (read-only snapshot view) instead of &mut GraphStore - - &mut TickDelta (append-only ops collector) - 3. Implement canonical merge - sort ops by key for deterministic ordering - 4. Add DeltaView - overlay that resolves reads from (snapshot + delta) +## 9) Design Decisions - **Status: COMPLETE** (2026-01-18) +Nine implementation-level decisions with rationale. - *** +### 9.1 BTreeMap Everywhere - Phase 4: SnapshotBuilder +`GraphStore` uses `BTreeMap` for all collections. This gives deterministic +iteration order at ~2x lookup cost vs `HashMap`. The trade-off is accepted +because iteration order affects `canonical_state_hash()`, WSC build ordering, +and snapshot construction. Using `HashMap` would require a separate sort pass +at every materialization point. - Goal: Apply merged TickDelta to produce next WSC snapshot - 1. SnapshotBuilder takes base snapshot + merged ops - 2. Produces next reachable-only snapshot tables - 3. Writes WSC bytes via existing write_wsc_one_warp - 4. Computes state_root from materialized snapshot +### 9.2 RefCell in MaterializationBus - **Status: COMPLETE** (2026-01-18) - See `docs/memorials/2026-01-18-phase4-rubicon.md` +`MaterializationBus` uses `RefCell>` for pending emissions. +This is intentionally single-threaded: tick execution on a single warp is +sequential by design, and `RefCell` makes the borrow rules visible at compile +time without the overhead of `Mutex`. - *** +### 9.3 Factor Mask as u64 Coarse Prefilter - Phase 5: Read-Only Execution +`Footprint::factor_mask` is a 64-bit bloom-like field. One bitwise AND +(`mask_a & mask_b == 0`) can short-circuit the entire independence check +before examining per-resource sets. On workloads with many independent +rewrites touching disjoint partitions, this eliminates >90% of set +intersection work. - Goal: Pure execution - executors read from snapshot, write to delta only - 1. ExecuteFn signature changed: `fn(&GraphView, &mut TickDelta, &NodeId)` - 2. No GraphStore mutations during execution - emit ops only - 3. State updated after execution via `apply_to_state()` - 4. Legacy `&mut GraphStore` path removed from executor signature +### 9.4 FxHashMap in PendingTx - **Status: COMPLETE** (2026-01-18) +`PendingTx` uses Rustc's `FxHashMap` (non-deterministic iteration order) for +deduplication. This is safe because the map is internal to the scheduler and +all entries are drained through the radix sort before any +determinism-sensitive operation. The map's iteration order is never observed. - *** +### 9.5 20-Pass LSD Radix Sort - Phase 6: Parallel Execution with Per-Worker Deltas +A 40-byte key requires 20 passes with 16-bit digits (65,536 buckets). This +is O(n) with zero comparisons, ideal for the scheduling workload where n can +be large and keys are uniformly distributed (BLAKE3 hashes). The fallback to +comparison sort below 1,024 items avoids the per-pass overhead for small +batches. - Goal: Lockless parallelism without compromising determinism - 1. Per-worker TickDelta - each worker has thread-local delta - 2. Canonical merge - concatenate then sort by (op_kind, key, tie_breaker) - 3. Prove worker-count invariance - the "god test" passes for all worker counts +### 9.6 Pod + bytemuck for WSC - *** +All WSC row types are `#[repr(C)]` + `Pod` + `Zeroable`. This allows +zero-copy deserialization via `bytemuck::try_cast_slice()` — a safe +transmutation that requires no `unsafe` in user code. The constraint is +that all fields must be plain-old-data (no pointers, no padding with +uninitialized bytes). + +### 9.7 Separate GraphView Type + +`GraphView` is a compile-time read-only projection of `GraphStore`. Executors +receive `GraphView<'a>` instead of `&GraphStore`, preventing accidental +writes. When `FootprintGuard` is active, `GraphView::new_guarded()` also +validates reads against the declared footprint. - Phase 7+ (Future) +### 9.8 PoisonedDelta on Panic -- Segment-level structural sharing (content-addressed chunks) -- Collapse/merge with typed registry -- Privacy enforcement gates (mind vs diagnostics mode) +When a rule panics during parallel execution, `catch_unwind` captures the +panic payload and wraps it as `PoisonedDelta`. This ensures the panic is +surfaced as a typed `MergeError::PoisonedDelta` at merge time rather than +crashing the entire tick. The delta's ops are never applied. - *** +### 9.9 NUM_SHARDS = 256 Frozen Protocol Constant + +The shard count is a protocol constant, not a tuning knob. Changing it would +alter `shard_of()` routing and break determinism for any graph built under the +old constant. 256 provides a good balance: enough shards for load distribution +across 8–64 workers, small enough that the shard metadata overhead is trivial. + +--- -Files to Create/Modify -┌──────────────────────────────────────────┬────────┬───────────────────────────────────┐ -│ Path │ Action │ Purpose │ -├──────────────────────────────────────────┼────────┼───────────────────────────────────┤ -│ crates/warp-core/tests/boaw/mod.rs │ Create │ Test module organization │ -├──────────────────────────────────────────┼────────┼───────────────────────────────────┤ -│ crates/warp-core/tests/boaw/common.rs │ Create │ XorShift64, TestHarness, Scenario │ -├──────────────────────────────────────────┼────────┼───────────────────────────────────┤ -│ crates/warp-core/tests/boaw\_\*.rs │ Create │ 6 test files from ADR │ -├──────────────────────────────────────────┼────────┼───────────────────────────────────┤ -│ crates/warp-core/src/tick_delta.rs │ Create │ TickDelta struct + merge │ -├──────────────────────────────────────────┼────────┼───────────────────────────────────┤ -│ crates/warp-core/src/delta_view.rs │ Create │ Read-through overlay │ -├──────────────────────────────────────────┼────────┼───────────────────────────────────┤ -│ crates/warp-core/src/snapshot_builder.rs │ Create │ Apply delta → next WSC │ -├──────────────────────────────────────────┼────────┼───────────────────────────────────┤ -│ crates/warp-core/src/lib.rs │ Modify │ Export new modules │ -├──────────────────────────────────────────┼────────┼───────────────────────────────────┤ -│ crates/warp-core/src/engine_impl.rs │ Modify │ New ExecuteFn signature │ -├──────────────────────────────────────────┼────────┼───────────────────────────────────┤ -│ crates/warp-core/src/rule.rs │ Modify │ Update ExecuteFn type │ -└──────────────────────────────────────────┴────────┴───────────────────────────────────┘ +## 10) Test Strategy + +### 10.1 Integration Test Categories + +warp-core has 50+ integration tests in `crates/warp-core/tests/`. Key +categories: + +| Category | Representative Files | What They Validate | +| ------------------ | ------------------------------------------------------------------ | --------------------------------------------------------------------------- | +| Determinism | `boaw_determinism.rs`, `boaw_end_to_end.rs` | Snapshot hash invariance under permutations, serial-vs-parallel equivalence | +| Worker invariance | `boaw_engine_worker_invariance.rs` | Identical results for 1/2/4/8/16/32 workers | +| Parallel execution | `boaw_parallel_exec.rs` | Sharded partitioning, canonical merge ordering | +| Footprints | `boaw_footprints.rs`, `boaw_footprint_warp_scoping.rs` | Independence symmetry, warp-scoped conflict isolation | +| Merge safety | `boaw_merge_tripwire.rs`, `boaw_merge_warpopkey.rs` | Footprint violations caught at merge, cross-warp key isolation | +| Portal rules | `boaw_openportal_rules.rs` | No same-tick writes to newly created warps | +| Materialization | `materialization_determinism.rs`, `materialization_spec_police.rs` | Order-independence, permutation invariance, wire stability | +| Snapshot | `snapshot_reachability_tests.rs` | Unreachable nodes excluded from hash | +| Slice theorem | `slice_theorem_proof.rs` | Seven-phase executable proof of parallel correctness | +| Math determinism | `deterministic_sin_cos_tests.rs`, `nan_exhaustive_tests.rs` | ULP-budget trig, NaN canonicalization to `0x7fc00000` | +| Playback | `outputs_playback_tests.rs`, `playback_cursor_tests.rs` | Cursor seek, corrupt-hash detection | + +### 10.2 Testing Patterns + +- **Exhaustive permutation:** `materialization_spec_police.rs` uses Heap's + algorithm to test all N! orderings (N ≤ 6) and assert byte-identical output +- **Pinned seeds:** Proptest with deterministic seeds for reproducible CI + (`proptest_seed_pinning.rs`) +- **1-ULP float sensitivity:** `determinism_audit.rs` checks whether f32 + bit-flips affect canonical hashes +- **Serial-vs-parallel equivalence:** `boaw_determinism.rs` runs the same + workload with 1 worker and N workers, asserts identical `state_root`, + `patch_digest`, and `commit_hash` +- **Worker-count invariance:** `boaw_engine_worker_invariance.rs` iterates + over `{1, 2, 4, 8, 16, 32}` workers on independent workloads +- **Byte-identical assertion:** All hash comparisons use exact `[u8; 32]` + equality, not approximate matching + +### 10.3 Future Tests (Marked `#[ignore]`) + +Several test files contain `#[ignore]` tests for features not yet +implemented: + +- `boaw_cow.rs` — COW overlay semantics (pending overlay implementation) +- `boaw_merge.rs` — Multi-parent commutative merge (pending collapse/merge) +- `boaw_privacy.rs` — Mind-mode enforcement (pending privacy implementation) + +--- + +## 11) ADR-0007 Alignment + +Mapping from parent ADR sections to concrete implementation modules. + +| ADR-0007 Section | Implementation | Status | +| ---------------------------------- | ----------------------------------------------------------------------- | ------------------------------------------------------- | +| §3 Atom & Attachment policy | `attachment.rs` — `AttachmentPlane`, `AttachmentOwner`, `AttachmentKey` | Partial — typed storage exists, no registry enforcement | +| §4 WARP snapshots (reachable-only) | `snapshot.rs` — reachable BFS in state-root computation | Done | +| §4.2 WSC zero-copy IO | `wsc/` — full read/write/validate/view pipeline | Done | +| §5 COW mechanics | `tick_delta.rs`, `graph_view.rs` — delta-based writes, read-only views | Partial — no segment-level structural sharing | +| §6 Footprints & enforcement | `footprint.rs`, `footprint_guard.rs` — warp-scoped sets + guard | Done | +| §7 Scheduling (virtual shards) | `boaw/shard.rs`, `scheduler.rs` — `NUM_SHARDS=256`, radix sort | Done | +| §7.1 Shard routing (frozen) | `boaw/shard.rs::shard_of()` | Done — frozen protocol constant | +| §8 Tick pipeline | `engine_impl.rs`, `scheduler.rs`, `boaw/exec.rs`, `boaw/merge.rs` | Done | +| §8.4 Parallel execute | `boaw/exec.rs` — `execute_parallel_sharded()` | Done (Phase 6B) | +| §8.5 Merge deltas | `boaw/merge.rs` — canonical sort + conflict detection | Done | +| §8.6 Commit | `snapshot.rs` — `compute_commit_hash_v2()`, `compute_state_root()` | Done | +| §9 Collapse/merge | — | Not started — `boaw_merge.rs` tests are `#[ignore]` | +| §10 Privacy (mind/diagnostics) | — | Not started — `boaw_privacy.rs` tests are `#[ignore]` | + +### Not Yet Implemented + +- **Collapse/merge with typed registry:** ADR-0007 §9 defines multi-parent + merge with typed merge rules (commutative, LWW, ConflictOnly). No merge + implementation exists beyond single-tick delta merge. +- **Privacy enforcement:** ADR-0007 §10 defines mind mode (no secrets in + ledger) and diagnostics mode. The `ClaimRecord` type and enforcement gates + are not yet implemented. +- **Segment-level COW:** ADR-0007 §5.5 defines segment-level structural + sharing for WSC snapshots. Current implementation rebuilds full snapshots + each tick. + +--- + +## 12) Consequences + +### Benefits + +- **Lockless parallel execution** — workers emit to thread-local deltas; + no shared mutable state during execution +- **Deterministic across platforms** — canonical ordering (radix sort), + fixed shard topology (256), explicit LE encoding, BLAKE3 hashing +- **Zero-copy snapshots** — WSC format with `bytemuck` provides read access + without deserialization +- **Typed conflict handling** — `PoisonedDelta`, `MergeConflict`, and + `FootprintViolation` surface errors as structured data, not panics +- **Comprehensive test coverage** — 50+ integration tests including + exhaustive permutation, worker-count invariance, and executable proofs + +### Costs + +- **BTreeMap overhead** — ~2x lookup cost vs HashMap, accepted for + deterministic iteration +- **Radix sort memory** — 65,536-bucket histogram + scratch buffer per sort + pass, amortized across ticks +- **Full snapshot rebuild** — each tick rebuilds the entire snapshot (no + segment-level sharing yet) +- **Feature-gated enforcement** — `FootprintGuard` is debug-only by default; + release enforcement requires opt-in feature flag + +### Open Items + +- Implement collapse/merge with typed registry (ADR-0007 §9) +- Implement privacy enforcement / mind mode (ADR-0007 §10) +- Add segment-level COW for snapshot structural sharing (ADR-0007 §5.5) +- Migrate `GraphStore` from BTreeMap to immutable snapshot + overlay model +- Add benchmark suites (snapshot hashing, motion throughput, scheduler drain) From c4d1ddb7e9b837f97be860aa0ae4155d331e7022 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 22 Feb 2026 09:41:15 -0800 Subject: [PATCH 3/4] docs: add CHANGELOG entry for ADR-0007-impl rewrite --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88f96016..6b9a81f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ ## Unreleased +### Fixed (Docs) + +- **Docs Build:** Rewrote `ADR-0007-impl.md` from a 3185-line raw conversation + transcript into a proper 13-section ADR document. The Vue template compiler + was crashing on bare Rust generics (`BTreeMap`, etc.) + outside fenced code blocks. The new document preserves all architectural + knowledge as a structured implementation companion to ADR-0007. + ## [0.1.3] — 2026-02-21 ### Fixed (Sprint S1) From 896c205cd1294c90b8783b72b0f3c6bbaab3c8ed Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 24 Feb 2026 11:56:47 -0800 Subject: [PATCH 4/4] chore: Remove auditing script --- scripts/generate_evidence.cjs | 143 ---------------------------------- 1 file changed, 143 deletions(-) delete mode 100755 scripts/generate_evidence.cjs diff --git a/scripts/generate_evidence.cjs b/scripts/generate_evidence.cjs deleted file mode 100755 index 1ebad161..00000000 --- a/scripts/generate_evidence.cjs +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env node -const fs = require('fs'); -const path = require('path'); - -/** - * Generates an evidence JSON pack for CI claims. - * Maps specific claim IDs to immutable CI artifacts if they exist. - * - * @param {string} gatheredArtifactsDir - Path to the directory where all artifacts were downloaded. - */ -function generateEvidence(gatheredArtifactsDir) { - const workflow = process.env.GITHUB_WORKFLOW || 'det-gates'; - const runId = process.env.GITHUB_RUN_ID || 'local'; - const commitSha = process.env.GITHUB_SHA || 'local'; - - const checkArtifact = (name) => { - const fullPath = path.join(gatheredArtifactsDir, name); - try { - return fs.existsSync(fullPath) && fs.readdirSync(fullPath).length > 0; - } catch (e) { - return false; - } - }; - - /** - * Parse static-inspection.json for DET-001 claim status. - * Returns VERIFIED only when the file exists, parses as valid JSON, - * contains claim_id "DET-001", and status "PASSED". - * All other conditions return UNVERIFIED with an error description. - */ - const checkStaticInspection = (artifactsDir) => { - const jsonPath = path.join(artifactsDir, 'static-inspection', 'static-inspection.json'); - let raw; - try { - raw = fs.readFileSync(jsonPath, 'utf8'); - } catch (e) { - console.error(`DET-001: static-inspection.json not found at ${jsonPath}`); - return { status: 'UNVERIFIED', error: 'static-inspection.json not found' }; - } - - let parsed; - try { - parsed = JSON.parse(raw); - } catch (e) { - console.error(`DET-001: invalid JSON in static-inspection.json: ${e.message}`); - return { status: 'UNVERIFIED', error: `invalid JSON: ${e.message}` }; - } - - if (parsed.claim_id !== 'DET-001' || typeof parsed.status !== 'string') { - console.error(`DET-001: unexpected structure in static-inspection.json: ${JSON.stringify(parsed)}`); - return { status: 'UNVERIFIED', error: 'missing or unexpected claim_id/status field' }; - } - - const verified = parsed.status === 'PASSED'; - if (!verified) { - console.error(`DET-001: static inspection reported status "${parsed.status}"`); - } - return { status: verified ? 'VERIFIED' : 'UNVERIFIED', source_status: parsed.status }; - }; - - const claims = [ - (() => { - const det001 = checkStaticInspection(gatheredArtifactsDir); - return { - id: 'DET-001', - status: det001.status, - evidence: { - workflow, run_id: runId, commit_sha: commitSha, - artifact_name: 'static-inspection', - source_file: 'static-inspection.json', - source_status: det001.source_status || null, - ...(det001.error ? { error: det001.error } : {}) - } - }; - })(), - { - id: 'DET-002', - status: checkArtifact('det-linux-artifacts') ? 'VERIFIED' : 'UNVERIFIED', - evidence: { workflow, run_id: runId, commit_sha: commitSha, artifact_name: 'det-linux-artifacts' } - }, - { - id: 'DET-003', - status: checkArtifact('det-macos-artifacts') ? 'VERIFIED' : 'UNVERIFIED', - evidence: { workflow, run_id: runId, commit_sha: commitSha, artifact_name: 'det-macos-artifacts' } - }, - { - id: 'SEC-001', - status: checkArtifact('sec-artifacts') ? 'VERIFIED' : 'UNVERIFIED', - evidence: { workflow, run_id: runId, commit_sha: commitSha, artifact_name: 'sec-artifacts' } - }, - { - id: 'SEC-002', - status: checkArtifact('sec-artifacts') ? 'VERIFIED' : 'UNVERIFIED', - evidence: { workflow, run_id: runId, commit_sha: commitSha, artifact_name: 'sec-artifacts' } - }, - { - id: 'SEC-003', - status: checkArtifact('sec-artifacts') ? 'VERIFIED' : 'UNVERIFIED', - evidence: { workflow, run_id: runId, commit_sha: commitSha, artifact_name: 'sec-artifacts' } - }, - { - id: 'SEC-004', - status: checkArtifact('sec-artifacts') ? 'VERIFIED' : 'UNVERIFIED', - evidence: { workflow, run_id: runId, commit_sha: commitSha, artifact_name: 'sec-artifacts' } - }, - { - id: 'SEC-005', - status: checkArtifact('sec-artifacts') ? 'VERIFIED' : 'UNVERIFIED', - evidence: { workflow, run_id: runId, commit_sha: commitSha, artifact_name: 'sec-artifacts' } - }, - { - id: 'REPRO-001', - status: checkArtifact('build-repro-artifacts') ? 'VERIFIED' : 'UNVERIFIED', - evidence: { workflow, run_id: runId, commit_sha: commitSha, artifact_name: 'build-repro-artifacts' } - }, - { - id: 'PRF-001', - status: checkArtifact('perf-artifacts') ? 'VERIFIED' : 'UNVERIFIED', - evidence: { workflow, run_id: runId, commit_sha: commitSha, artifact_name: 'perf-artifacts' } - } - ]; - - const evidence = { - claims, - metadata: { - generated_at: new Date().toISOString(), - workflow, - run_id: runId, - commit_sha: commitSha - } - }; - - const outputPath = path.join(gatheredArtifactsDir, 'evidence.json'); - fs.writeFileSync(outputPath, JSON.stringify(evidence, null, 2)); - console.log(`Generated evidence.json at ${outputPath}`); -} - -module.exports = { generateEvidence }; - -if (require.main === module) { - const gatheredArtifactsDir = process.argv[2] || '.'; - generateEvidence(gatheredArtifactsDir); -}