feat: add streaming export of events to JSON Lines

Author: Anshumancanrock

README.md

@@ -543,6 +543,17 @@ To see the integration test coverage report open `.coverage/integration/lcov-rep
   open .coverage/integration/lcov-report/index.html
   ```
 
+## Export Events
+
+Export all stored events to a [JSON Lines](https://jsonlines.org/) (`.jsonl`) file. Each line is a valid NIP-01 Nostr event JSON object. The export streams rows from the database using cursors, so it works safely on relays with millions of events without loading them into memory.
+
+```
+npm run export                            # writes to events.jsonl
+npm run export -- backup-2024-01-01.jsonl # custom filename
+```
+
+The script reads the same `DB_*` environment variables used by the relay (see [CONFIGURATION.md](CONFIGURATION.md)).
+
 ## Configuration
 
 You can change the default folder by setting the `NOSTR_CONFIG_DIR` environment variable to a different path.

src/scripts/export-events.ts

@@ -40,43 +40,52 @@ async function exportEvents(): Promise<void> {
     const output = fs.createWriteStream(outputPath)
     let exported = 0
 
-    const dbStream = db('events')
-      .select(
-        'event_id',
-        'event_pubkey',
-        'event_kind',
-        'event_created_at',
-        'event_content',
-        'event_tags',
-        'event_signature',
-      )
-      .whereNull('deleted_at')
-      .orderBy('event_created_at', 'asc')
-      .stream()
+    const trx = await db.transaction(null, { isolationLevel: 'repeatable read' })
+    try {
+      await trx.raw('SET TRANSACTION READ ONLY')
+
+      const dbStream = trx('events')
+        .select(
+          'event_id',
+          'event_pubkey',
+          'event_kind',
+          'event_created_at',
+          'event_content',
+          'event_tags',
+          'event_signature',
+        )
+        .whereNull('deleted_at')
+        .orderBy('event_created_at', 'asc')
+        .stream()
 
-    const toJsonLine = new Transform({
-      objectMode: true,
-      transform(row: any, _encoding, callback) {
-        const event = {
-          id: row.event_id.toString('hex'),
-          pubkey: row.event_pubkey.toString('hex'),
-          created_at: row.event_created_at,
-          kind: row.event_kind,
-          tags: row.event_tags || [],
-          content: row.event_content,
-          sig: row.event_signature.toString('hex'),
-        }
+      const toJsonLine = new Transform({
+        objectMode: true,
+        transform(row: any, _encoding, callback) {
+          const event = {
+            id: row.event_id.toString('hex'),
+            pubkey: row.event_pubkey.toString('hex'),
+            created_at: row.event_created_at,
+            kind: row.event_kind,
+            tags: row.event_tags || [],
+            content: row.event_content,
+            sig: row.event_signature.toString('hex'),
+          }
 
-        exported++
-        if (exported % 10000 === 0) {
-          console.log(`Exported ${exported}/${total} events...`)
-        }
+          exported++
+          if (exported % 10000 === 0) {
+            console.log(`Exported ${exported}/${total} events...`)
+          }
 
-        callback(null, JSON.stringify(event) + '\n')
-      },
-    })
+          callback(null, JSON.stringify(event) + '\n')
+        },
+      })
 
-    await pipeline(dbStream, toJsonLine, output)
+      await pipeline(dbStream, toJsonLine, output)
+      await trx.commit()
+    } catch (err) {
+      await trx.rollback()
+      throw err
+    }
 
     console.log(`Export complete: ${exported} events written to ${outputPath}`)
   } finally {