feat: sqlite indexer

Cargo.toml

@@ -204,6 +204,7 @@ smallvec = "1"
 smart-default = "0.7"
 sonic-rs = "0.5"
 spire_enum = "1"
+sqlx = { version = "0.8", default-features = false, features = ["sqlite", "runtime-tokio", "macros"] }
 stacker = "0.1"
 static_assertions = "1"
 statrs = "0.18"
@@ -232,7 +233,6 @@ zstd = "0.13"
 
 # optional dependencies
 console-subscriber = { version = "0.5", features = ["parking_lot"], optional = true }
-sqlx = { version = "0.8", default-features = false, features = ["sqlite", "runtime-tokio", "macros"], optional = true }
 tikv-jemallocator = { version = "0.6", optional = true }
 tracing-loki = { version = "0.2", default-features = false, features = ["compat-0-2-1", "rustls"], optional = true }
 
@@ -324,14 +324,13 @@ lto = "fat"
 
 # These should be refactored (probably removed) in #2984
 [features]
-default = ["jemalloc", "tracing-loki", "sqlite"]
-test = []                                        # default feature set for unit tests
+default = ["jemalloc", "tracing-loki"]
+test = []                              # default feature set for unit tests
 slim = ["rustalloc"]
-cargo-test = []                                  # group of tests that is recommended to run with `cargo test` instead of `nextest`
-doctest-private = []                             # see lib.rs::doctest_private
-benchmark-private = ["dep:criterion"]            # see lib.rs::benchmark_private
-interop-tests-private = []                       # see lib.rs::interop_tests_private
-sqlite = ["dep:sqlx"]
+cargo-test = []                        # group of tests that is recommended to run with `cargo test` instead of `nextest`
+doctest-private = []                   # see lib.rs::doctest_private
+benchmark-private = ["dep:criterion"]  # see lib.rs::benchmark_private
+interop-tests-private = []             # see lib.rs::interop_tests_private
 
 # Allocator. Use at most one of these.
 rustalloc = []

scripts/tests/api_compare/docker-compose.yml

@@ -58,7 +58,7 @@ services:
           --halt-after-import
 
         SNAPSHOT_EPOCH="$(ls /data/*.car.zst | tail -n 1 | grep -Eo '[0-9]+' | tail -n 1)"
-        # backfill the index db
+        # backfill the index db (nosql)
         forest-tool index backfill --from $$SNAPSHOT_EPOCH --n-tipsets 200 --chain ${CHAIN}
 
         forest --chain ${CHAIN} --encrypt-keystore false --no-gc \
@@ -90,6 +90,10 @@ services:
         export FULLNODE_API_INFO="$(cat /data/forest-token):/dns/forest/tcp/${FOREST_RPC_PORT}/http"
         echo "Waiting till Forest is ready"
         forest-cli healthcheck ready --healthcheck-port ${FOREST_HEALTHZ_RPC_PORT} --wait
+
+        # backfill the index db (sql)
+        SNAPSHOT_EPOCH="$(ls /data/*.car.zst | tail -n 1 | grep -Eo '[0-9]+' | tail -n 1)"
+        forest-cli index validate-backfill --from $$SNAPSHOT_EPOCH --to $(($$SNAPSHOT_EPOCH - 200))
   forest-wallet-import:
     depends_on:
       forest:

scripts/tests/api_compare/filter-list-gateway

@@ -7,6 +7,7 @@
 !Filecoin.ChainSetHead
 !Filecoin.ChainStatObj
 !Filecoin.ChainTipSetWeight
+!Filecoin.ChainValidateIndex
 !Filecoin.EthGetBlockReceiptsLimited
 !Filecoin.F3
 !Filecoin.GasEstimateGasLimit

scripts/tests/api_compare/filter-list-offline

@@ -1,5 +1,7 @@
 # This list contains potentially broken methods (or tests) that are ignored.
 # They should be considered bugged, and not used until the root cause is resolved.
+# Indexer disabled
+!Filecoin.ChainValidateIndex
 !Filecoin.EthSyncing
 !eth_syncing
 !Filecoin.NetAddrsListen

src/chain/store/chain_store.rs

@@ -126,7 +126,7 @@ where
         chain_config: Arc<ChainConfig>,
         genesis_block_header: CachingBlockHeader,
     ) -> anyhow::Result<Self> {
-        let (publisher, _) = broadcast::channel(SINK_CAP);
+        let (head_changes_tx, _) = broadcast::channel(SINK_CAP);
         let chain_index = Arc::new(ChainIndex::new(Arc::clone(&db)));
         let validated_blocks = Mutex::new(HashSet::default());
         let head = if let Some(head_tsk) = heaviest_tipset_key_provider
@@ -141,7 +141,7 @@ where
             Tipset::from(&genesis_block_header)
         };
         let cs = Self {
-            head_changes_tx: publisher,
+            head_changes_tx,
             chain_index,
             tipset_tracker: TipsetTracker::new(Arc::clone(&db), chain_config.clone()),
             db,