Introduce LanternStore and move vote storage

.gitmodules

@@ -15,8 +15,8 @@
 	url = https://github.com/Pier-Two/leanSpec.git
 [submodule "tools/lean-quickstart"]
 	path = tools/lean-quickstart
-	url = https://github.com/Pier-Two/lean-quickstart.git
-	branch = devnet3
+	url = https://github.com/blockblaz/lean-quickstart.git
+	branch = main
 [submodule "external/c-hash-sig"]
 	path = external/c-hash-sig
 	url = https://github.com/Pier-Two/c-hash-sig.git

CMakeLists.txt

@@ -34,6 +34,7 @@ add_library(lantern STATIC
     src/consensus/slot_clock.c
     src/consensus/signature.c
     src/consensus/state.c
+    src/consensus/store.c
     src/consensus/ssz.c
     src/core/client.c
     src/core/client_debug.c
@@ -246,10 +247,12 @@ if(LANTERN_BUILD_TESTS)
     add_test(NAME lantern_consensus_runtime COMMAND lantern_consensus_runtime_test)
 
     add_executable(lantern_state_test tests/unit/test_state.c)
+    target_sources(lantern_state_test PRIVATE tests/support/state_store_adapter.c)
     target_link_libraries(lantern_state_test PRIVATE lantern)
     add_test(NAME lantern_state COMMAND lantern_state_test)
 
     add_executable(lantern_off_head_replay_test tests/unit/test_off_head_replay.c)
+    target_sources(lantern_off_head_replay_test PRIVATE tests/support/state_store_adapter.c)
     target_link_libraries(lantern_off_head_replay_test PRIVATE lantern)
     add_test(NAME lantern_off_head_replay COMMAND lantern_off_head_replay_test)
     set_tests_properties(lantern_off_head_replay PROPERTIES TIMEOUT 120)
@@ -265,6 +268,7 @@ if(LANTERN_BUILD_TESTS)
 
     add_executable(lantern_networking_messages_test
         tests/unit/test_networking_messages.c
+        tests/support/state_store_adapter.c
         tests/support/fixture_loader.c
     )
     target_link_libraries(lantern_networking_messages_test PRIVATE lantern)
@@ -290,6 +294,7 @@ if(LANTERN_BUILD_TESTS)
     add_test(NAME lantern_fork_choice COMMAND lantern_fork_choice_test)
 
     add_executable(lantern_storage_test tests/unit/test_storage.c)
+    target_sources(lantern_storage_test PRIVATE tests/support/state_store_adapter.c)
     target_link_libraries(lantern_storage_test PRIVATE lantern)
     add_test(NAME lantern_storage COMMAND lantern_storage_test)
 
@@ -308,6 +313,7 @@ if(LANTERN_BUILD_TESTS)
     add_executable(lantern_consensus_vectors_test
         tests/integration/test_consensus_vectors.c
         tests/integration/consensus_fixture_runner.c
+        tests/support/state_store_adapter.c
         tests/support/fixture_loader.c
     )
     target_link_libraries(lantern_consensus_vectors_test PRIVATE lantern)
@@ -330,6 +336,7 @@ if(LANTERN_BUILD_TESTS)
 
     add_executable(lantern_verify_signatures_vectors_test
         tests/integration/test_verify_signatures_vectors.c
+        tests/support/state_store_adapter.c
         tests/support/fixture_loader.c
     )
     target_link_libraries(lantern_verify_signatures_vectors_test PRIVATE lantern)
@@ -350,6 +357,7 @@ if(LANTERN_BUILD_TESTS)
     add_executable(lantern_genesis_vectors_test
         tests/integration/test_genesis_vectors.c
         tests/integration/consensus_fixture_runner.c
+        tests/support/state_store_adapter.c
         tests/support/fixture_loader.c
     )
     target_link_libraries(lantern_genesis_vectors_test PRIVATE lantern)
@@ -412,5 +420,6 @@ if(LANTERN_BUILD_TESTS)
     )
     set_property(TEST ${_lantern_ctest_targets} PROPERTY TIMEOUT 30)
     set_tests_properties(lantern_client_vote PROPERTIES TIMEOUT 300)
+    set_tests_properties(lantern_state PROPERTIES TIMEOUT 120)
 
 endif()

include/lantern/consensus/fork_choice.h

@@ -6,6 +6,7 @@
 #include <stdint.h>
 
 #include "lantern/consensus/containers.h"
+#include "lantern/consensus/state.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -14,8 +15,12 @@ extern "C" {
 struct lantern_fork_choice_vote_entry {
     bool has_checkpoint;
     LanternCheckpoint checkpoint;
+    uint64_t slot;
 };
 
+struct lantern_aggregated_payload_pool;
+struct lantern_attestation_data_by_root;
+
 struct lantern_fork_choice_block_entry {
     LanternRoot root;
     LanternRoot parent_root;
@@ -29,6 +34,11 @@ struct lantern_fork_choice_block_entry {
     LanternCheckpoint latest_finalized;
 };
 
+struct lantern_fork_choice_state_entry {
+    bool has_state;
+    LanternState state;
+};
+
 struct lantern_fork_choice_root_index_entry {
     LanternRoot root;
     size_t value;
@@ -55,13 +65,27 @@ typedef struct lantern_fork_choice {
     size_t block_len;
     size_t block_cap;
 
+    struct lantern_fork_choice_state_entry *states;
+    size_t state_cap;
+
     struct lantern_fork_choice_root_index_entry *index_entries;
     size_t index_cap;
     size_t index_len;
 
     struct lantern_fork_choice_vote_entry *known_votes;
     struct lantern_fork_choice_vote_entry *new_votes;
     size_t validator_count;
+
+    /*
+     * Attached views into store-owned attestation material.
+     *
+     * Aggregated gossip updates the shared store first. Fork-choice consumers
+     * read these pointers to observe the same proof pools and attestation-data
+     * map without maintaining a duplicate cache.
+     */
+    const struct lantern_aggregated_payload_pool *new_aggregated_payloads;
+    const struct lantern_aggregated_payload_pool *known_aggregated_payloads;
+    const struct lantern_attestation_data_by_root *attestation_data_by_root;
 } LanternForkChoice;
 
 void lantern_fork_choice_init(LanternForkChoice *store);
@@ -75,6 +99,13 @@ int lantern_fork_choice_set_anchor(
     const LanternCheckpoint *latest_justified,
     const LanternCheckpoint *latest_finalized,
     const LanternRoot *block_root_hint);
+int lantern_fork_choice_set_anchor_with_state(
+    LanternForkChoice *store,
+    const LanternBlock *anchor_block,
+    const LanternCheckpoint *latest_justified,
+    const LanternCheckpoint *latest_finalized,
+    const LanternRoot *block_root_hint,
+    const LanternState *anchor_state);
 
 int lantern_fork_choice_add_block(
     LanternForkChoice *store,
@@ -83,6 +114,14 @@ int lantern_fork_choice_add_block(
     const LanternCheckpoint *post_justified,
     const LanternCheckpoint *post_finalized,
     const LanternRoot *block_root_hint);
+int lantern_fork_choice_add_block_with_state(
+    LanternForkChoice *store,
+    const LanternBlock *block,
+    const LanternSignedVote *proposer_attestation,
+    const LanternCheckpoint *post_justified,
+    const LanternCheckpoint *post_finalized,
+    const LanternRoot *block_root_hint,
+    const LanternState *post_state);
 
 int lantern_fork_choice_add_vote(
     LanternForkChoice *store,
@@ -128,6 +167,13 @@ int lantern_fork_choice_set_block_validator_count(
     LanternForkChoice *store,
     const LanternRoot *root,
     uint64_t validator_count);
+int lantern_fork_choice_set_block_state(
+    LanternForkChoice *store,
+    const LanternRoot *root,
+    const LanternState *state);
+const LanternState *lantern_fork_choice_block_state(
+    const LanternForkChoice *store,
+    const LanternRoot *root);
 const LanternCheckpoint *lantern_fork_choice_latest_justified(const LanternForkChoice *store);
 const LanternCheckpoint *lantern_fork_choice_latest_finalized(const LanternForkChoice *store);
 const LanternRoot *lantern_fork_choice_safe_target(const LanternForkChoice *store);

include/lantern/consensus/state.h

@@ -7,8 +7,7 @@
 
 #include "lantern/consensus/containers.h"
 
-struct lantern_vote_record;
-struct lantern_fork_choice;
+typedef struct lantern_store LanternStore;
 
 struct lantern_root_list {
     LanternRoot *items;
@@ -23,18 +22,12 @@ typedef struct {
     LanternCheckpoint latest_justified;
     LanternCheckpoint latest_finalized;
     struct lantern_root_list historical_block_hashes;
-    uint64_t historical_roots_offset;
     struct lantern_bitlist justified_slots;
-    uint64_t justified_slots_offset;
     struct lantern_root_list justification_roots;
     struct lantern_bitlist justification_validators;
-    struct lantern_vote_record *validator_votes;
-    size_t validator_votes_len;
     LanternValidator *validators;
     size_t validator_count;
     size_t validator_capacity;
-    struct lantern_fork_choice *fork_choice;
-    LanternRoot validator_registry_root;
 } LanternState;
 
 void lantern_root_list_init(struct lantern_root_list *list);
@@ -44,44 +37,35 @@ int lantern_root_list_resize(struct lantern_root_list *list, size_t new_length);
 void lantern_state_init(LanternState *state);
 void lantern_state_reset(LanternState *state);
 int lantern_state_clone(const LanternState *source, LanternState *dest);
-void lantern_state_attach_fork_choice(LanternState *state, struct lantern_fork_choice *fork_choice);
 int lantern_state_generate_genesis(LanternState *state, uint64_t genesis_time, uint64_t num_validators);
 int lantern_state_process_slot(LanternState *state);
 int lantern_state_process_slots(LanternState *state, uint64_t target_slot);
 int lantern_state_process_block_header(LanternState *state, const LanternBlock *block);
 int lantern_state_process_attestations(
     LanternState *state,
+    LanternStore *store,
     const LanternAttestations *attestations,
     const LanternSignatureList *signatures);
 int lantern_state_process_block(
     LanternState *state,
+    LanternStore *store,
     const LanternBlock *block,
     const LanternBlockSignatures *signatures,
     const LanternSignedVote *proposer_attestation);
 bool lantern_state_slot_in_justified_window(const LanternState *state, uint64_t slot);
 int lantern_state_get_justified_slot_bit(const LanternState *state, uint64_t slot, bool *out_value);
 int lantern_state_mark_justified_slot(LanternState *state, uint64_t slot);
-int lantern_state_transition(LanternState *state, const LanternSignedBlock *signed_block);
-int lantern_state_prepare_validator_votes(LanternState *state, uint64_t validator_count);
-size_t lantern_state_validator_capacity(const LanternState *state);
-bool lantern_state_validator_has_vote(const LanternState *state, size_t index);
-int lantern_state_get_signed_validator_vote(
-    const LanternState *state,
-    size_t index,
-    LanternSignedVote *out_vote);
-int lantern_state_get_validator_vote(const LanternState *state, size_t index, LanternVote *out_vote);
-int lantern_state_set_signed_validator_vote(
-    LanternState *state,
-    size_t index,
-    const LanternSignedVote *vote);
-int lantern_state_set_validator_vote(LanternState *state, size_t index, const LanternVote *vote);
-void lantern_state_clear_validator_vote(LanternState *state, size_t index);
+int lantern_state_transition(LanternState *state, LanternStore *store, const LanternSignedBlock *signed_block);
 int lantern_state_set_validator_pubkeys(LanternState *state, const uint8_t *pubkeys, size_t count);
 size_t lantern_state_validator_count(const LanternState *state);
 const uint8_t *lantern_state_validator_pubkey(const LanternState *state, size_t index);
-int lantern_state_select_block_parent(LanternState *state, LanternRoot *out_parent_root);
+int lantern_state_select_block_parent(
+    LanternState *state,
+    const LanternStore *store,
+    LanternRoot *out_parent_root);
 int lantern_state_collect_attestations_for_block(
     const LanternState *state,
+    const LanternStore *store,
     uint64_t block_slot,
     uint64_t proposer_index,
     const LanternRoot *parent_root,
@@ -90,11 +74,13 @@ int lantern_state_collect_attestations_for_block(
     LanternSignatureList *out_signatures);
 int lantern_state_compute_vote_checkpoints(
     const LanternState *state,
+    const LanternStore *store,
     LanternCheckpoint *out_head,
     LanternCheckpoint *out_target,
     LanternCheckpoint *out_source);
 int lantern_state_preview_post_state_root(
     const LanternState *state,
+    const LanternStore *store,
     const LanternSignedBlock *block,
     LanternRoot *out_state_root);