Support tiered data storage

benches/payments.rs

@@ -127,6 +127,7 @@ fn payment_benchmark(c: &mut Criterion) {
 		true,
 		false,
 		common::TestStoreType::Sqlite,
+		common::TestStoreType::Sqlite,
 	);
 
 	let runtime =

bindings/ldk_node.udl

@@ -32,6 +32,8 @@ interface LogWriter {
 	void log(LogRecord record);
 };
 
+typedef interface FfiDynStore;
+
 interface Builder {
 	constructor();
 	[Name=from_config]
@@ -58,6 +60,8 @@ interface Builder {
 	void set_tor_config(TorConfig tor_config);
 	[Throws=BuildError]
 	void set_node_alias(string node_alias);
+	void set_backup_store(FfiDynStore backup_store);
+	void set_ephemeral_store(FfiDynStore ephemeral_store);
 	[Throws=BuildError]
 	void set_async_payments_role(AsyncPaymentsRole? role);
 	void set_wallet_recovery_mode();
@@ -73,6 +77,8 @@ interface Builder {
 	Node build_with_vss_store_and_fixed_headers(NodeEntropy node_entropy, string vss_url, string store_id, record<string, string> fixed_headers);
 	[Throws=BuildError]
 	Node build_with_vss_store_and_header_provider(NodeEntropy node_entropy, string vss_url, string store_id, VssHeaderProvider header_provider);
+	[Throws=BuildError]
+	Node build_with_store(NodeEntropy node_entropy, FfiDynStore store);
 };
 
 interface Node {

bindings/python/src/ldk_node/kv_store.py

@@ -0,0 +1,115 @@
+import threading
+
+from abc import ABC, abstractmethod
+from typing import List
+
+from ldk_node import IoError
+
+class AbstractKvStore(ABC):
+    @abstractmethod
+    async def read_async(self, primary_namespace: "str",secondary_namespace: "str",key: "str") -> "typing.List[int]":
+        pass
+
+    @abstractmethod
+    async def write_async(self, primary_namespace: "str",secondary_namespace: "str",key: "str",buf: "typing.List[int]") -> None:
+        pass
+
+    @abstractmethod
+    async def remove_async(self, primary_namespace: "str",secondary_namespace: "str",key: "str",lazy: "bool") -> None:
+        pass
+
+    @abstractmethod
+    async def list_async(self, primary_namespace: "str",secondary_namespace: "str") -> "typing.List[str]":
+        pass
+
+    @abstractmethod
+    def read(self, primary_namespace: "str",secondary_namespace: "str",key: "str") -> "typing.List[int]":
+        pass
+
+    @abstractmethod
+    def write(self, primary_namespace: "str",secondary_namespace: "str",key: "str",buf: "typing.List[int]") -> None:
+        pass
+
+    @abstractmethod
+    def remove(self, primary_namespace: "str",secondary_namespace: "str",key: "str",lazy: "bool") -> None:
+        pass
+
+    @abstractmethod
+    def list(self, primary_namespace: "str",secondary_namespace: "str") -> "typing.List[str]":
+        pass
+
+class TestKvStore(AbstractKvStore):
+    def __init__(self, name: str):
+        self.name = name
+        # Storage structure: {(primary_ns, secondary_ns): {key: [bytes]}}
+        self.storage = {}
+        self._lock = threading.Lock()
+
+    def dump(self):
+        print(f"\n[{self.name}] Store contents:")
+        for (primary_ns, secondary_ns), keys_dict in self.storage.items():
+            print(f"  Namespace: ({primary_ns!r}, {secondary_ns!r})")
+            for key, data in keys_dict.items():
+                print(f"    Key: {key!r} -> {len(data)} bytes")
+                # Optionally show first few bytes
+                preview = data[:20] if len(data) > 20 else data
+                print(f"      Data preview: {preview}...")
+
+    def read(self, primary_namespace: str, secondary_namespace: str, key: str) -> List[int]:
+        with self._lock:
+            print(f"[{self.name}] READ: {primary_namespace}/{secondary_namespace}/{key}")
+            namespace_key = (primary_namespace, secondary_namespace)
+
+            if namespace_key not in self.storage:
+                print(f"  -> namespace not found, keys: {list(self.storage.keys())}")
+                raise IoError.NotFound()
+
+            if key not in self.storage[namespace_key]:
+                print(f"  -> key not found, keys: {list(self.storage[namespace_key].keys())}")
+                raise IoError.NotFound()
+
+            data = self.storage[namespace_key][key]
+            print(f"  -> returning {len(data)} bytes")
+            return data
+
+    def write(self, primary_namespace: str, secondary_namespace: str, key: str, buf: List[int]) -> None:
+        with self._lock:
+            namespace_key = (primary_namespace, secondary_namespace)
+            if namespace_key not in self.storage:
+                self.storage[namespace_key] = {}
+
+            self.storage[namespace_key][key] = buf
+
+    def remove(self, primary_namespace: str, secondary_namespace: str, key: str, lazy: bool) -> None:
+        with self._lock:
+            namespace_key = (primary_namespace, secondary_namespace)
+            if namespace_key not in self.storage:
+                raise IoError.NotFound()
+
+            if key not in self.storage[namespace_key]:
+                raise IoError.NotFound()
+
+            del self.storage[namespace_key][key]
+
+            if not self.storage[namespace_key]:
+                del self.storage[namespace_key]
+
+    def list(self, primary_namespace: str, secondary_namespace: str) -> List[str]:
+        with self._lock:
+            namespace_key = (primary_namespace, secondary_namespace)
+            if namespace_key in self.storage:
+                return list(self.storage[namespace_key].keys())
+            return []
+
+    async def read_async(self, primary_namespace: str, secondary_namespace: str, key: str) -> List[int]:
+        return self.read(primary_namespace, secondary_namespace, key)
+
+    async def write_async(self, primary_namespace: str, secondary_namespace: str, key: str, buf: List[int]) -> None:
+        self.write(primary_namespace, secondary_namespace, key, buf)
+
+    async def remove_async(self, primary_namespace: str, secondary_namespace: str, key: str, lazy: bool) -> None:
+        self.remove(primary_namespace, secondary_namespace, key, lazy)
+
+    async def list_async(self, primary_namespace: str, secondary_namespace: str) -> List[str]:
+        return self.list(primary_namespace, secondary_namespace)
+