Add S3 archive write-path foundation

.env.example

@@ -55,3 +55,44 @@ ENABLE_DA_TRACKING=false
 # FAUCET_PRIVATE_KEY=0x...
 # FAUCET_AMOUNT=0.01
 # FAUCET_COOLDOWN_MINUTES=30
+
+# Optional: archive indexed block bundles to an S3-compatible object store.
+#
+# When enabled, atlas-server writes a compressed copy of every indexed block
+# (block header + transaction receipts) to an S3 bucket as it indexes.
+# Objects are stored under:  {S3_PREFIX}/v1/blocks/{bucket_start}/{block_number}.json.zst
+# A manifest file at:        {S3_PREFIX}/v1/manifest.json  tracks the latest
+# contiguous uploaded block so consumers know how far the archive has progressed.
+#
+# Works with AWS S3 and any S3-compatible store (MinIO, Cloudflare R2, etc.).
+# Credentials are read from AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY below.
+# This only adds the write path; a restore command will be added later.
+#
+# S3_ENABLED=false
+#
+# -- Where to store files --
+# S3_BUCKET=atlas-archive         # Name of the bucket (must exist before starting)
+# S3_REGION=us-east-1             # AWS region, or any non-empty string for non-AWS stores
+# S3_PREFIX=                      # Optional path prefix inside the bucket, e.g. "mainnet".
+#                                 # Useful when sharing one bucket across multiple chains.
+#                                 # Leave empty to write directly at the bucket root.
+# S3_ENDPOINT=                    # Override the S3 server URL. Leave empty for AWS S3.
+#                                 # Set to your MinIO / R2 / etc. URL, e.g. http://minio:9000
+# S3_FORCE_PATH_STYLE=false       # Set to true for MinIO and most self-hosted stores.
+#                                 # AWS S3 uses subdomain-style (false); MinIO requires path-style (true).
+#
+# -- Performance & reliability --
+# S3_UPLOAD_CONCURRENCY=4         # Number of blocks uploaded in parallel. Higher = faster
+#                                 # catch-up after downtime, at the cost of more memory/network.
+# S3_RETRY_BASE_SECONDS=30        # Base delay before retrying a failed upload. Uses exponential
+#                                 # backoff: 30s → 60s → 120s … capped at 1 hour.
+#
+# -- AWS credentials (also used for MinIO) --
+# AWS_ACCESS_KEY_ID=              # Access key ID (IAM user key for AWS, root user for MinIO)
+# AWS_SECRET_ACCESS_KEY=          # Secret access key
+# AWS_SESSION_TOKEN=              # Only needed for temporary IAM credentials (e.g. assumed roles)
+
+# Optional MinIO admin credentials — only used by docker-compose to initialise the
+# MinIO container itself (create buckets, etc.). Not read by atlas-server.
+# MINIO_ROOT_USER=minioadmin
+# MINIO_ROOT_PASSWORD=minioadmin

CLAUDE.md

@@ -91,6 +91,16 @@ pub struct AppState {
 ### DA tracking (optional)
 When `ENABLE_DA_TRACKING=true`, a background DA worker queries ev-node for Celestia inclusion heights per block. `EVNODE_URL` is required only in that mode. Updates are pushed to SSE clients via an in-process `broadcast::Sender<Vec<DaSseUpdate>>`. The SSE handler streams `da_batch` events for incremental updates and emits `da_resync` when a client falls behind and should refetch visible DA state.
 
+### S3 archive (optional)
+When `S3_ENABLED=true`, a background `ArchiveUploader` task uploads every indexed block to an S3-compatible object store as a zstd-compressed JSON bundle (block header + receipts). The write path uses a transactional outbox pattern:
+1. The indexer writes archive entries to `archive_blocks` (with compressed payload) inside the same DB transaction as the block data.
+2. The uploader claims rows via `SELECT … FOR UPDATE SKIP LOCKED`, uploads to S3, then clears the payload column.
+3. `archive_state` tracks the latest *contiguous* uploaded block and triggers a manifest refresh at `{prefix}/v1/manifest.json` so consumers know how far the archive has progressed.
+
+Object key layout: `{prefix}/v1/blocks/{bucket_start:012}/{block_number:012}.json.zst` where `bucket_start = (block_number / 10_000) * 10_000`.
+
+Works with AWS S3 and any S3-compatible store (MinIO, Cloudflare R2, etc.). The URL is constructed automatically from `S3_BUCKET` + `S3_REGION` + optional `S3_ENDPOINT` — there is no single URL config. Set `S3_FORCE_PATH_STYLE=true` for MinIO and other self-hosted stores; leave it `false` for AWS.
+
 ### Frontend API client
 - Base URL: `/api` (proxied by nginx to `atlas-server:3000`)
 - Fast polling endpoint: `GET /api/height` → `{ block_height, indexed_at, features: { da_tracking } }` — serves from `head_tracker` first and falls back to `indexer_state` when the in-memory head is empty. Used by the navbar as a polling fallback when SSE is disconnected and by feature-flag consumers.
@@ -127,6 +137,16 @@ Key vars (see `.env.example` for full list):
 | `EVNODE_URL` | server | none |
 | `DA_RPC_REQUESTS_PER_SECOND` | DA worker | `50` |
 | `DA_WORKER_CONCURRENCY` | DA worker | `50` |
+| `S3_ENABLED` | archive | `false` |
+| `S3_BUCKET` | archive | required if enabled |
+| `S3_REGION` | archive | required if enabled |
+| `S3_PREFIX` | archive | `""` (bucket root) |
+| `S3_ENDPOINT` | archive | none (AWS S3) |
+| `S3_FORCE_PATH_STYLE` | archive | `false` |
+| `S3_UPLOAD_CONCURRENCY` | archive | `4` |
+| `S3_RETRY_BASE_SECONDS` | archive | `30` |
+| `AWS_ACCESS_KEY_ID` | archive | env / IAM role |
+| `AWS_SECRET_ACCESS_KEY` | archive | env / IAM role |
 
 ## Running Locally
 

backend/Cargo.toml

@@ -32,10 +32,13 @@ serde_json = "1.0"
 
 # HTTP client
 reqwest = { version = "0.13", features = ["json", "rustls"], default-features = false }
+aws-config = "1"
+aws-sdk-s3 = "1"
 
 # Error handling
 thiserror = "2.0"
 anyhow = "1.0"
+async-trait = "0.1"
 
 # Logging
 tracing = "0.1"
@@ -53,6 +56,7 @@ async-stream = "0.3"
 bigdecimal = { version = "0.4", features = ["serde"] }
 hex = "0.4"
 chrono = { version = "0.4", features = ["serde"] }
+zstd = "0.13"
 
 # Testing
 testcontainers = "0.27"

backend/crates/atlas-server/Cargo.toml

@@ -19,14 +19,18 @@ alloy = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 reqwest = { workspace = true }
+aws-config = { workspace = true }
+aws-sdk-s3 = { workspace = true }
 thiserror = { workspace = true }
 anyhow = { workspace = true }
+async-trait = { workspace = true }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
 dotenvy = { workspace = true }
 bigdecimal = { workspace = true }
 hex = { workspace = true }
 chrono = { workspace = true }
+zstd = { workspace = true }
 tokio-stream = { workspace = true }
 futures = { workspace = true }
 async-stream = { workspace = true }