diff --git a/rfcs/2026-03-20-smp-agent-web.md b/rfcs/2026-03-20-smp-agent-web.md new file mode 100644 index 0000000000..8b015f0e8e --- /dev/null +++ b/rfcs/2026-03-20-smp-agent-web.md @@ -0,0 +1,299 @@ +# SMP Agent for Browser — Web Widget Infrastructure + +## 1. Problem & Goal + +The SimpleX web widget needs to create duplex connections, send and receive encrypted messages, and handle the full SMP agent lifecycle — all running in the browser. This requires a TypeScript implementation of the SMP protocol stack: encoding, transport, client, and agent layers, mirroring the Haskell implementation in simplexmq. + +This document covers the protocol infrastructure that lives in the simplexmq repository (`smp-web/`). The widget UI and chat-layer semantics (contact addresses, business addresses, group links) live in simplex-chat. + +## 2. Architecture + +Four layers, mirroring the Haskell codebase: + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Agent Layer │ +│ Duplex connections, X3DH key agreement, double ratchet, │ +│ message delivery, queue rotation, connection lifecycle │ +├─────────────────────────────────────────────────────────────┤ +│ Client Layer │ +│ Connection pool (per server), command/response correlation, │ +│ reconnection, backoff │ +├─────────────────────────────────────────────────────────────┤ +│ Transport Layer │ +│ WebSocket, SMP handshake, block framing (16384 bytes), │ +│ block encryption (X25519 DH + SbChainKeys) │ +├─────────────────────────────────────────────────────────────┤ +│ Protocol Layer │ +│ SMP commands (NEW, KEY, SUB, SEND, ACK, etc.), │ +│ binary encoding, transmission format │ +├─────────────────────────────────────────────────────────────┤ +│ Shared (from xftp-web) │ +│ encoding.ts, secretbox.ts, padding.ts, keys.ts, digest.ts │ +└─────────────────────────────────────────────────────────────┘ + │ + ▼ WebSocket (TLS via browser) + ┌───────────────┐ + │ SMP Server │ + │ (SNI → Warp │ + │ → WS upgrade)│ + └───────────────┘ +``` + +### Core Principle: Mirror Haskell Structure + +TypeScript code mirrors the Haskell module hierarchy as closely as possible. Each Haskell module has a corresponding TypeScript file, placed in the same relative path. Functions keep the same names. This enables: +- Easy cross-reference between codebases +- Sync as protocol evolves +- Code review by people who know the Haskell side +- Byte-for-byte testing of corresponding functions + +### File Structure + +``` +smp-web/ +├── src/ +│ ├── protocol.ts ← SMP commands, transmission format +│ ├── protocol/ +│ │ └── types.ts ← protocol types +│ ├── version.ts ← version range negotiation +│ ├── transport.ts ← handshake, block framing, THandle +│ ├── transport/ +│ │ └── websockets.ts ← WebSocket connection +│ ├── client.ts ← connection pool, correlation, reconnect +│ ├── crypto/ +│ │ ├── ratchet.ts ← double ratchet +│ │ └── shortLink.ts ← HKDF, link data decrypt +│ └── agent/ +│ ├── protocol.ts ← connection types, link data parsing +│ └── client.ts ← connection lifecycle, message delivery +├── package.json +└── tsconfig.json +``` + +Encoding and crypto primitives are imported directly from xftp-web (npm dependency). New files are only created where SMP-specific logic is needed. + +### Haskell Module → TypeScript File Mapping + +| Haskell Module | TypeScript File | Source | +|---|---|---| +| `Simplex.Messaging.Encoding` | xftp-web `protocol/encoding.ts` | import directly | +| `Simplex.Messaging.Crypto` | xftp-web `crypto/*` | import directly | +| `Simplex.Messaging.Protocol` | `protocol.ts` | new | +| `Simplex.Messaging.Protocol.Types` | `protocol/types.ts` | new | +| `Simplex.Messaging.Version` | `version.ts` | new | +| `Simplex.Messaging.Transport` | `transport.ts` | new | +| `Simplex.Messaging.Transport.WebSockets` | `transport/websockets.ts` | new | +| `Simplex.Messaging.Client` | `client.ts` | new | +| `Simplex.Messaging.Crypto.Ratchet` | `crypto/ratchet.ts` | new | +| `Simplex.Messaging.Crypto.ShortLink` | `crypto/shortLink.ts` | new | +| `Simplex.Messaging.Agent.Protocol` | `agent/protocol.ts` | new | +| `Simplex.Messaging.Agent.Client` | `agent/client.ts` | new | + +Function names in TypeScript match Haskell names (camelCase preserved). When a Haskell function is `smpClientHandshake`, TypeScript has `smpClientHandshake`. When Haskell has `contactShortLinkKdf`, TypeScript has `contactShortLinkKdf`. + +## 3. Relationship to xftp-web + +xftp-web (`simplexmq-2/xftp-web/`) is a production TypeScript XFTP client. smp-web reuses its foundations: + +**Reused directly (npm dependency)**: +- `protocol/encoding.ts` — Decoder class, Word16/Word32/Int64, ByteString, Large, Bool, Maybe, List encoding +- `crypto/secretbox.ts` — XSalsa20-Poly1305 (cbEncrypt/cbDecrypt, streaming) +- `crypto/padding.ts` — Block padding (2-byte length prefix + `#` fill) +- `crypto/keys.ts` — Ed25519, X25519 key generation, signing, DH, DER encoding +- `crypto/digest.ts` — SHA-256, SHA-512 +- `crypto/identity.ts` — X.509 certificate chain parsing, signature verification + +**New in smp-web**: +- SMP protocol commands and transmission format +- SMP handshake (different from XFTP handshake) +- WebSocket transport (XFTP uses HTTP/2 fetch) +- SMP client with queue-based correlation +- Agent layer (connections, ratchet, message processing) +- Short link operations (HKDF-SHA512, link data parsing) + +**Same build pattern**: +- TypeScript strict, ES2022 modules +- `tsc` → `dist/` +- Haskell tests via `callNode` (same function from XFTPWebTests) +- Each TypeScript function verified byte-for-byte against Haskell + +## 4. Server Changes + +### Done +- `attachStaticAndWS` — unified HTTP + WebSocket handler via `wai-websockets` +- SNI-based routing: browser (SNI) → Warp → WebSocket upgrade → SMP over WS; native (no SNI) → SMP over TLS +- `acceptWSConnection` — constructs `WS 'TServer` from TLS connection + Warp PendingConnection, preserves peer cert chain +- `AttachHTTP` takes `TLS 'TServer` (not raw Context), enabling proper cert chain forwarding +- Test: `testWebSocketAndTLS` verifies native TLS and WebSocket clients on same port + +### Remaining +- CORS headers for cross-origin widget embedding (pattern available in XFTP server) +- Server CLI configuration for enabling/disabling WebSocket support per port + +## 5. Build Approach + +Bottom-up, function-by-function. Each TypeScript function tested against its Haskell counterpart before building the next. + +**Test infrastructure**: `SMPWebTests.hs` reuses `callNode`, `jsOut`, `jsUint8` from `XFTPWebTests.hs` (generalized, not copied). + +**Pattern**: for each function: +1. Implement in TypeScript +2. Write Haskell test that calls it via `callNode` +3. Compare output byte-for-byte with Haskell reference +4. Also test cross-language: Haskell encodes → TypeScript decodes, and vice versa + +## 6. Implementation Phases + +### Phase 1: Protocol Encoding + Handshake + +Foundation layer. SMP-specific binary encoding and handshake. + +**Functions**: +- SMP transmission format: `[auth ByteString][corrId ByteString][entityId ByteString][command]` +- `encodeTransmission` / `parseTransmission` +- `parseSMPServerHandshake` — versionRange, sessionId, authPubKey (CertChainPubKey) +- `encodeSMPClientHandshake` — version, keyHash, authPubKey, proxyServer, clientService +- Server certificate chain verification (reuse xftp-web identity.ts) +- Version negotiation + +**Key encoding details**: +- `authPubKey` uses `encodeAuthEncryptCmds`: Nothing → empty (0 bytes), Just → raw smpEncode (NOT Maybe 0/1 prefix) +- `proxyServer`: Bool 'T'/'F' (v14+) +- `clientService`: Maybe '0'/'1' (v16+) + +### Phase 2: SMP Commands + +All commands needed for messaging. + +**Sender**: SKEY, SEND +**Receiver**: NEW, KEY, SUB, ACK, OFF, DEL +**Link**: LGET +**Common**: PING + +**For each command**: encode function + decode function for its response, tested against Haskell. + +### Phase 3: Transport + +WebSocket connection with SMP block framing. + +**Functions**: +- WebSocket connect (`wss://` URL) +- Block send/receive (16384-byte binary frames) +- SMP handshake over WebSocket +- Block encryption: X25519 DH → HKDF-SHA512 → SbChainKeys → per-block XSalsa20-Poly1305 + +**Block encryption flow**: +1. Client generates ephemeral X25519 keypair, sends public key in handshake +2. Server sends its signed DH key in handshake +3. Both sides compute DH shared secret +4. `sbcInit(sessionId, dhSecret)` → two 32-byte chain keys (HKDF-SHA512) +5. Each block: `sbcHkdf(chainKey)` → ephemeral key + nonce, advance chain +6. Encrypt/decrypt with XSalsa20-Poly1305, blockSize-16 padding target + +### Phase 4: Client + +Connection management layer. + +**Functions**: +- Connection pool: one WebSocket per SMP server +- Command/response correlation via corrId +- Send queue + receive queue (ABQueue pattern from simplexmq-js) +- Automatic reconnection with exponential backoff +- Timeout handling + +### Phase 5: Agent — Connection Establishment + +Duplex SMP connections with X3DH key agreement. + +**Functions**: +- Create receive queue (NEW) +- Join connection via invitation URI +- X3DH key agreement +- Send confirmation (SKEY + SEND) +- Complete handshake (HELLO exchange) +- Connection state machine + +### Phase 6: Agent — Double Ratchet + +Message encryption/decryption. + +**Functions**: +- Signal double ratchet implementation +- Header encryption +- Ratchet state management +- Key derivation (HKDF) +- Message sequence + hash chain verification + +### Phase 7: Agent — Message Delivery + +Send and receive messages through established connections. + +**Functions**: +- Send path: encrypt → encode agent envelope → SEND → handle OK/delivery receipt +- Receive path: SUB → receive MSG → decrypt → verify → ACK +- Delivery receipts +- Message acknowledgment + +### Phase 8: Short Links + +Entry point for the widget — parse short link, fetch profile. + +**Functions**: +- Parse short link URI (contact, group, business address types) +- HKDF key derivation (SHA-512): `contactShortLinkKdf` +- LGET command → LNK response +- Decrypt link data (XSalsa20-Poly1305) +- Parse FixedLinkData, ConnLinkData, UserLinkData +- Extract profile JSON + +## 7. Persistence + +Agent state (keys, ratchet, connections, messages) must persist across page reloads. + +**Open question**: storage backend. + +Options: +- **IndexedDB directly** — universal browser support, async API, no additional dependencies. Downside: key-value semantics, no SQL queries, manual indexing. +- **SQLite in browser** — sql.js (WASM-compiled SQLite) or wa-sqlite (with OPFS backend for persistence). Upside: matches Haskell agent's SQLite storage, schema can mirror `Simplex.Messaging.Agent.Store.SQLite`. Downside: additional dependency, WASM bundle size. +- **OPFS + SQLite** — Origin Private File System for durable storage, SQLite for structured access. Best durability, but limited browser support (no Safari private browsing). + +**Decision criteria**: how closely we want to mirror the Haskell agent's storage schema, bundle size budget, browser compatibility requirements. + +## 8. Testing Strategy + +### Unit Tests (per function) + +Haskell tests in `SMPWebTests.hs` using `callNode` pattern: +- TypeScript function called via Node.js subprocess +- Output compared byte-for-byte with Haskell reference +- Cross-language tests: encode in one language, decode in the other + +### Integration Tests + +Against live SMP server (spawned by test setup, same pattern as xftp-web globalSetup.ts): +- WebSocket connect + handshake +- Command round-trips (NEW, KEY, SUB, SEND, ACK) +- Message delivery through server +- Reconnection after disconnect + +### Browser Tests + +Vitest + Playwright (same as xftp-web): +- Full connection lifecycle in browser environment +- WebSocket transport in real browser +- Persistence round-trips + +## 9. Security Model + +Same principles as xftp-web: +- **TLS via browser** — browser handles certificate validation for WSS connections +- **SNI routing** — browser connections use SNI, routed to Warp + WebSocket handler +- **Server identity** — verified via certificate chain in SMP handshake (keyHash from short link or known servers) +- **Block encryption** — X25519 DH + SbChainKeys provides forward secrecy per block, on top of TLS +- **End-to-end encryption** — double ratchet between agent peers, server sees only encrypted blobs +- **No server-side secrets** — all keys derived and stored client-side +- **CORS** — required for cross-origin widget embedding, safe because SMP requires auth on every command +- **CSP** — strict content security policy for widget page + +**Threat model**: same as xftp-web. Primary risk is page substitution (malicious JS). Mitigated by HTTPS, CSP, SRI, and optionally IPFS hosting with published fingerprints. diff --git a/rfcs/2026-03-20-smp-agent-web/2026-03-20-smp-agent-web-spike.md b/rfcs/2026-03-20-smp-agent-web/2026-03-20-smp-agent-web-spike.md new file mode 100644 index 0000000000..dd01a7ad09 --- /dev/null +++ b/rfcs/2026-03-20-smp-agent-web/2026-03-20-smp-agent-web-spike.md @@ -0,0 +1,324 @@ +# SMP Agent Web: Spike Plan + +Revision 4, 2026-03-20 + +Parent RFC: [2026-03-20-smp-agent-web.md](../2026-03-20-smp-agent-web.md) + +## Revision History + +- **Rev 4**: Aligned with RFC. Restructured as bottom-up build plan with per-function Haskell tests. Router WebSocket support done. File structure mirrors Haskell modules. +- **Rev 3**: Fixed multiple encoding errors discovered during audit (see encoding details below). + +## Objective + +Fetch and display business/contact profile from a SimpleX short link URI, via WebSocket to SMP router. This is the first milestone of the SMP agent web implementation — it proves the protocol encoding, transport, crypto, and data parsing layers work end-to-end. + +The spike is not throwaway code. It is the beginning of the `smp-web/` TypeScript library, built bottom-up with each function tested against its Haskell counterpart. + +## What This Proves + +- WebSocket transport to SMP router works from browser +- SMP protocol encoding is correct (binary format, not ASCII) +- SMP handshake works (version negotiation, server certificate parsing) +- Crypto is compatible (HKDF-SHA512, XSalsa20-Poly1305) +- Short link data parsing matches Haskell (FixedLinkData, ConnLinkData, profile) + +## Success Criteria + +Haskell test creates a short link, TypeScript fetches and decodes it via WebSocket, profile data matches. + +## Protocol Flow + +``` +1. Parse short link URI + https://simplex.chat/c#?h=hosts&p=port&c=keyHash + → server, linkKey + +2. Derive keys (HKDF-SHA512) + linkKey → (linkId, sbKey) + +3. WebSocket connect + wss://server:443 (TLS handled by browser) + +4. SMP handshake + ← SMPServerHandshake {sessionId, smpVersionRange, authPubKey} + → SMPClientHandshake {smpVersion, keyHash, authPubKey=Nothing, proxyServer=False, clientService=Nothing} + +5. Send LGET + → [empty auth][corrId][linkId]["LGET"] + +6. Receive LNK + ← [auth][corrId][linkId]["LNK" space senderId encFixedData encUserData] + +7. Decrypt + XSalsa20-Poly1305 with sbKey + → FixedLinkData, ConnLinkData (with profile JSON) + +8. Display profile +``` + +Note: spike sends `authPubKey=Nothing` so block encryption is not used (blocks are padded only). Block encryption is added in steps 12-13. + + +## Build Approach + +Bottom-up, function-by-function. Each TypeScript function tested against its Haskell counterpart via `callNode` — the same pattern used in xftp-web (see `XFTPWebTests.hs`). + +**Project location**: `simplexmq-2/smp-web/` +**Tests**: `simplexmq-2/tests/SMPWebTests.hs` — reuses `callNode`/`jsOut`/`jsUint8` from XFTPWebTests (generalized, not copied) +**xftp-web**: npm dependency (encoding, crypto, padding imported directly) +**File structure**: mirrors Haskell module hierarchy (see RFC section 2) + +**Pattern for each function**: +1. Check if xftp-web already implements it (or something close). If so, import and reuse — export from xftp-web if not yet exported. Only write new code when no existing implementation covers the need. +2. Implement in TypeScript, in the file corresponding to its Haskell module +3. Write Haskell test that calls it via `callNode` +4. Compare output byte-for-byte with Haskell reference +5. Cross-language: Haskell encodes → TypeScript decodes, and vice versa + +### Parsing Approach + +All binary parsing uses xftp-web's `Decoder` class — the same class, not a copy. `Decoder` tracks position over a `Uint8Array`, throws on malformed input, returns subarray views (zero-copy). + +SMP command parsing follows the same pattern as xftp-web's `decodeResponse` in `commands.ts`: `readTag` reads bytes until space or end, switch dispatches on the tag string, fields are parsed sequentially with `Decoder` methods (`decodeBytes`, `decodeLarge`, `decodeBool`, etc.). + +**Prerequisite xftp-web change**: `readTag` and `readSpace` in xftp-web's `commands.ts` need to be exported so smp-web can import them. + +### WebSocket Transport Approach + +WebSocket transport follows the simplexmq-js `WSTransport` pattern: + +- `WebSocket` connects to `wss://` URL with `binaryType = 'arraybuffer'` +- `onmessage` enqueues received frames into an `ABQueue` (async bounded queue with backpressure) +- `onclose` closes the queue (sentinel-based) +- `readBlock()` dequeues one frame, validates it is exactly 16384 bytes +- `sendBlock(data)` sends one 16384-byte binary frame + +The `ABQueue` class from simplexmq-js provides backpressure via semaphores and clean async iteration. It can be included in smp-web or extracted as a shared utility. + +The SMP transport layer wraps WebSocket transport: +- Receives raw blocks → unpad → parse transmission +- Encodes transmission → pad → send as block +- After handshake, if block encryption is active: decrypt before unpad, encrypt after pad + + +## Encoding Reference + +Binary encoding rules (from `Simplex.Messaging.Encoding`): + +| Type | Format | +|------|--------| +| `Word16` | 2 bytes big-endian | +| `Word32` | 4 bytes big-endian | +| `ByteString` | 1-byte length + bytes (max 255) | +| `Large` | 2-byte length (BE) + bytes (max 65535) | +| `Bool` | 'T' (0x54) or 'F' (0x46) | +| `Maybe a` | '0' (0x30) for Nothing, '1' (0x31) + value for Just | +| `smpEncodeList` | 1-byte count + items | +| `UserLinkData` | ByteString if ≤254 bytes, else 0xFF + Large | + +**Critical**: `encodeAuthEncryptCmds Nothing` = empty (0 bytes), NOT 'F' or '0'. + +**Transmission format** (binary, NOT ASCII with spaces): +``` +[auth ByteString][corrId ByteString][entityId ByteString][command bytes] +``` + +For v7+ (`implySessId = True`): sessionId is NOT sent on wire, but is prepended to the `authorized` data for signature verification. For unauthenticated commands (LGET), this doesn't apply. + +**Block framing**: `pad(transmission, 16384)` = `[2-byte BE length][message][padding with '#' (0x23)]` + + +## Server Changes — DONE + +WebSocket support on the same port as native TLS is implemented and tested. + +- `attachStaticAndWS` — unified HTTP + WebSocket handler via `wai-websockets` +- SNI routing: browser (SNI) → Warp → WebSocket upgrade → SMP over WS +- `acceptWSConnection` — constructs `WS 'TServer` from `TLS 'TServer` + PendingConnection +- Test: `testWebSocketAndTLS` in `ServerTests.hs` + +**Remaining**: CORS headers for cross-origin widget embedding. + + +## Implementation Steps + +Each step produces working, tested code. Steps 1-11 work without block encryption. Steps 12-13 add it. + +### Step 1: Project Setup + xftp-web Changes + +**smp-web setup**: +- Create `smp-web/` with `package.json` (xftp-web + `@noble/hashes` as dependencies), `tsconfig.json` (ES2022, strict, same as xftp-web) +- Build: `tsc` → `dist/` + +**xftp-web change**: +- Export `readTag` and `readSpace` from `commands.ts` (currently unexported) so smp-web can import them + +**Test infrastructure**: +- Create `SMPWebTests.hs`, reusing `callNode`/`jsOut`/`jsUint8` from XFTPWebTests (generalize shared utilities into a common test module, not copy) +- First test: import `decodeBytes` from xftp-web, encode a ByteString, verify output matches Haskell `smpEncode` + +### Step 2: SMP Transmission Encode/Decode + +**File**: `protocol.ts` +**Haskell reference**: `Simplex.Messaging.Protocol` — `encodeTransmission_`, `transmissionP` + +**Implementation**: +- `encodeTransmission(corrId, entityId, command)`: `concatBytes(encodeBytes(emptyAuth), encodeBytes(corrId), encodeBytes(entityId), command)` — unsigned, empty auth byte (0x00) +- `decodeTransmission(data)`: sequential Decoder — `decodeBytes` for auth, corrId, entityId, then `takeAll` for command bytes +- Pad/unpad: reuse xftp-web `blockPad`/`blockUnpad` (same 2-byte length prefix + '#' padding, same 16384 block size) + +**Tests**: encode in TypeScript → decode in Haskell (`transmissionP`), encode in Haskell (`encodeTransmission_`) → decode in TypeScript. Byte-for-byte match. + +### Step 3: SMP Handshake Parse/Encode + +**File**: `transport.ts` +**Haskell reference**: `Simplex.Messaging.Transport` — `SMPServerHandshake`, `SMPClientHandshake` + +**Implementation**: +- `parseSMPServerHandshake(d: Decoder)`: `decodeWord16` × 2 for versionRange, `decodeBytes` for sessionId. For authPubKey: if `maxVersion >= 7` and bytes remaining, parse `CertChainPubKey` (reuse xftp-web `identity.ts` for X.509 cert chain parsing and signature extraction). If no bytes remain, authPubKey is absent (encodeAuthEncryptCmds encoded Nothing as empty). +- `encodeSMPClientHandshake(...)`: `concatBytes(encodeWord16(version), encodeBytes(keyHash), authPubKeyBytes, encodeBool(proxyServer), encodeMaybe(encodeService, clientService))`. Where authPubKey: empty bytes for Nothing, `encodeBytes(pubkey)` for Just. proxyServer only for v14+, clientService only for v16+. + +**Tests**: Haskell encodes `SMPServerHandshake` → TypeScript parses, all fields match. TypeScript encodes `SMPClientHandshake` → Haskell parses via `smpP`. + +### Step 4: LGET Command Encode + +**File**: `protocol.ts` +**Haskell reference**: `Simplex.Messaging.Protocol` — `LGET` command encoding + +**Implementation**: +- `encodeLGET()`: returns `ascii("LGET")` — 4 bytes, no parameters. The LinkId is carried as entityId in the transmission (step 2), not in the command body. +- Full LGET block: `blockPad(encodeTransmission(corrId, linkId, encodeLGET()), 16384)` + +**Tests**: encode full LGET block in TypeScript, Haskell unpad + `transmissionP` + `parseProtocol` decodes as `LGET` with correct corrId and linkId. + +### Step 5: LNK Response Parse + +**File**: `protocol.ts` +**Haskell reference**: `Simplex.Messaging.Protocol` — `LNK` response encoding (line 1834) + +**Implementation**: +- `decodeResponse(d: Decoder)`: `readTag(d)` → switch dispatch (same pattern as xftp-web `decodeResponse`) +- For `"LNK"`: `readSpace(d)`, `decodeBytes(d)` for senderId, `decodeLarge(d)` for encFixedData, `decodeLarge(d)` for encUserData +- Also handle `"ERR"` responses for error reporting + +**Tests**: Haskell encodes `LNK senderId (encFixed, encUser)` → TypeScript `decodeResponse` parses. All fields match byte-for-byte. + +### Step 6: Short Link URI Parse + +**File**: `agent/protocol.ts` +**Haskell reference**: `Simplex.Messaging.Agent.Protocol` — `ConnShortLink` StrEncoding instance (lines 1599-1612) + +**Implementation**: +- `parseShortLink(uri)`: regex to extract scheme (https/simplex), type char (c/g/a), linkKey (base64url, 43 chars → 32 bytes), query params (h=hosts, p=port, c=keyHash) +- `base64UrlDecode(s)`: pad to multiple of 4, replace `-`→`+`, `_`→`/`, decode +- Returns `{scheme, connType, server: {hosts, port, keyHash}, linkKey}` + +**Tests**: Haskell `strEncode` a `ConnShortLink` → TypeScript `parseShortLink` parses. All fields match. Test multiple formats: with/without query params, different type chars. + +### Step 7: HKDF Key Derivation + +**File**: `crypto/shortLink.ts` +**Haskell reference**: `Simplex.Messaging.Crypto.ShortLink` — `contactShortLinkKdf` (line 48) + +**Implementation**: +- `contactShortLinkKdf(linkKey)`: `hkdf(sha512, linkKey, new Uint8Array(0), "SimpleXContactLink", 56)` using `@noble/hashes/hkdf` + `@noble/hashes/sha512`. Split result: first 24 bytes = linkId, remaining 32 bytes = sbKey. + +**Note**: Haskell `C.hkdf` uses SHA-512, not SHA3-256. + +**Tests**: given known linkKey bytes, TypeScript and Haskell produce identical linkId and sbKey. + +### Step 8: Link Data Decrypt + +**File**: `crypto/shortLink.ts` +**Haskell reference**: `Simplex.Messaging.Crypto.ShortLink` — `decryptLinkData` (lines 100-120) + +**Implementation**: +- `decryptLinkData(sbKey, encFixedData, encUserData)`: + 1. For each EncDataBytes: `Decoder` → `decodeBytes(d)` for nonce (24 bytes), `decodeTail(d)` for ciphertext (includes Poly1305 tag) + 2. `cbDecrypt(sbKey, nonce, ciphertext)` via xftp-web `secretbox.ts` + 3. From decrypted plaintext: `decodeBytes(d)` for signature (1-byte len 0x40 + 64 bytes), `decodeTail(d)` for actual data + 4. Return both plaintext data blobs (signature verification skipped for spike) + +**Tests**: Haskell `encodeSignLinkData` + `sbEncrypt` with known key/nonce → TypeScript decrypts → plaintext matches. + +### Step 9: FixedLinkData / ConnLinkData Parse + +**File**: `agent/protocol.ts` +**Haskell reference**: `Simplex.Messaging.Agent.Protocol` — `FixedLinkData`, `ConnLinkData`, `UserContactData` Encoding instances + +**Implementation**: +- `decodeFixedLinkData(d)`: `decodeWord16` × 2 for agentVRange, `decodeBytes` for rootKey (32 bytes Ed25519), `decodeLarge` for linkConnReq, optional `decodeBytes` for linkEntityId (if bytes remaining) +- `decodeConnLinkData(d)`: `anyByte` for connectionMode ('C'=Contact), `decodeWord16` × 2 for agentVRange, then `decodeUserContactData` +- `decodeUserContactData(d)`: `decodeBool` for direct, `decodeList(decodeOwnerAuth, d)` for owners, `decodeList(decodeConnShortLink, d)` for relays, `decodeUserLinkData(d)` for userData +- `decodeUserLinkData(d)`: peek first byte — if 0xFF, skip it and `decodeLarge(d)`; otherwise `decodeBytes(d)` +- `parseProfile(userData)`: check first byte for 'X' (0x58, zstd compressed) — if so, decompress; otherwise `JSON.parse` directly + +**Tests**: Haskell encodes full `FixedLinkData` and `ContactLinkData` with known values → TypeScript decodes → all fields match. + +### Step 10: WebSocket Transport + +**File**: `transport/websockets.ts` +**Pattern reference**: simplexmq-js `WSTransport` + `ABQueue` + +**Implementation**: +- `ABQueue` class: semaphore-based async bounded queue (from simplexmq-js `queue.ts` — reimplement or include as utility). `enqueue`/`dequeue`/`close`, sentinel-based close, async iterator. +- `connectWS(url)`: `new WebSocket(url)`, `binaryType = 'arraybuffer'`, `onmessage` enqueues `Uint8Array` frames into ABQueue, `onclose` closes queue, `onerror` closes socket. Returns transport handle on `onopen`. +- `readBlock(transport)`: dequeue one frame, verify `byteLength === 16384`, return `Uint8Array` +- `sendBlock(transport, data)`: `ws.send(data)`, verify `data.length === 16384` +- `smpHandshake(transport, keyHash)`: `readBlock` → `blockUnpad` → `parseSMPServerHandshake` → negotiate version → `encodeSMPClientHandshake` → `blockPad` → `sendBlock`. Returns `{sessionId, version}`. + +**Integration test**: spawn test SMP server with web credentials (reuse `cfgWebOn` from SMPClient.hs), connect via WebSocket from Node.js, complete handshake, verify sessionId received. + +### Step 11: End-to-End Integration + +Wire steps 6-10 together: `parseShortLink` → `contactShortLinkKdf` → `connectWS` → `smpHandshake` → encode LGET block → `sendBlock` → `readBlock` → `blockUnpad` → `decodeTransmission` → `decodeResponse` → `decryptLinkData` → `decodeFixedLinkData` + `decodeConnLinkData` → `parseProfile`. + +**Test**: Haskell creates a contact address with short link (using agent), TypeScript fetches and decodes it via WebSocket. Profile displayName matches. This is the full spike proof: browser can fetch a SimpleX contact profile via SMP protocol. + +### Step 12: Block Encryption (DH + SbChainKeys) + +**File**: `transport.ts` +**Haskell reference**: `Simplex.Messaging.Crypto` — `sbcInit`, `sbcHkdf`; `Simplex.Messaging.Transport` — `tPutBlock`, `tGetBlock` + +**Implementation**: +- `generateX25519KeyPair()`, `dh(peerPub, ownPriv)` — reuse from xftp-web `keys.ts` +- `sbcInit(sessionId, dhSecret)`: `hkdf(sha512, dhSecret, sessionId, "SimpleXSbChainInit", 64)` → split at 32: `(sndChainKey, rcvChainKey)`. Note client swaps send/receive keys vs server (line 858 Transport.hs). +- `sbcHkdf(chainKey)`: `hkdf(sha512, chainKey, "", "SimpleXSbChain", 88)` → split: 32 bytes new chainKey, 32 bytes sbKey, 24 bytes nonce. Returns `{sbKey, nonce, nextChainKey}`. +- `encryptBlock(state, block)`: `sbcHkdf` → `cryptoBox(sbKey, nonce, pad(block, blockSize - 16))` → 16-byte tag + ciphertext +- `decryptBlock(state, block)`: `sbcHkdf` → split tag (first 16 bytes) + ciphertext → `cryptoBoxOpen` → `unpad` + +**Tests**: Haskell and TypeScript DH with same keys → identical chain keys. Haskell encrypts block → TypeScript decrypts (and vice versa). Chain key advances identically after each block. + +### Step 13: Full Handshake with Auth + +**File**: `transport.ts` +**Haskell reference**: `Simplex.Messaging.Transport` — `smpClientHandshake` (lines 792-842) + +**Implementation**: +- Update `smpHandshake` to generate ephemeral X25519 keypair and include public key in `encodeSMPClientHandshake` as authPubKey +- Parse server's `CertChainPubKey` from handshake: extract DH public key, verify X.509 certificate chain (reuse xftp-web `identity.ts` — `verifyIdentityProof`, `extractCertPublicKeyInfo`) +- Compute DH: `dh(serverDhPub, clientPrivKey)` → shared secret +- `sbcInit(sessionId, dhSecret)` → chain keys (with client-side swap) +- All subsequent `readBlock`/`sendBlock` go through `decryptBlock`/`encryptBlock` + +**Tests**: full handshake with real server, block encryption active, exchange encrypted commands. Haskell sends encrypted response → TypeScript decrypts correctly. + + +## Haskell Code References + +### Handshake +- `Simplex.Messaging.Transport` — `smpClientHandshake`, `smpServerHandshake`, `SMPServerHandshake`, `SMPClientHandshake` +- `encodeAuthEncryptCmds` — Nothing → empty, Just → raw smpEncode + +### Protocol +- `Simplex.Messaging.Protocol` — `LGET`, `LNK`, `encodeTransmission_`, `transmissionP` +- Block: `pad`/`unPad` in `Simplex.Messaging.Crypto` + +### Short Links +- `Simplex.Messaging.Crypto.ShortLink` — `contactShortLinkKdf`, `decryptLinkData` +- `Simplex.Messaging.Agent.Protocol` — `ConnShortLink`, `FixedLinkData`, `ConnLinkData`, `UserLinkData` + +### Block Encryption +- `Simplex.Messaging.Crypto` — `sbcInit`, `sbcHkdf`, `sbEncrypt`, `sbDecrypt`, `dh'` +- `Simplex.Messaging.Transport` — `blockEncryption`, `TSbChainKeys`, `tPutBlock`, `tGetBlock` diff --git a/simplexmq.cabal b/simplexmq.cabal index fe1aa2de0e..b3fa5175a8 100644 --- a/simplexmq.cabal +++ b/simplexmq.cabal @@ -510,6 +510,7 @@ test-suite simplexmq-test XFTPServerTests WebTests XFTPWebTests + SMPWebTests SMPWeb XFTPWeb Web.Embedded diff --git a/smp-web/.gitignore b/smp-web/.gitignore new file mode 100644 index 0000000000..320c107b3e --- /dev/null +++ b/smp-web/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +dist/ +package-lock.json diff --git a/smp-web/package.json b/smp-web/package.json new file mode 100644 index 0000000000..bad007e0cd --- /dev/null +++ b/smp-web/package.json @@ -0,0 +1,24 @@ +{ + "name": "@simplex-chat/smp-web", + "version": "0.1.0", + "description": "SMP protocol client for web/browser environments", + "license": "AGPL-3.0-only", + "repository": { + "type": "git", + "url": "git+https://github.com/simplex-chat/simplexmq.git", + "directory": "smp-web" + }, + "type": "module", + "files": ["src", "dist"], + "scripts": { + "build": "tsc" + }, + "dependencies": { + "@simplex-chat/xftp-web": "file:../xftp-web", + "@noble/hashes": "^1.5.0" + }, + "devDependencies": { + "typescript": "^5.4.0", + "ws": "^8.0.0" + } +} diff --git a/smp-web/src/index.ts b/smp-web/src/index.ts new file mode 100644 index 0000000000..fa23b4f606 --- /dev/null +++ b/smp-web/src/index.ts @@ -0,0 +1,12 @@ +// SMP protocol client for web/browser environments. +// Re-exports encoding primitives from xftp-web for convenience. +export { + Decoder, + encodeBytes, decodeBytes, + encodeLarge, decodeLarge, + encodeWord16, decodeWord16, + encodeBool, decodeBool, + encodeMaybe, decodeMaybe, + encodeList, decodeList, + concatBytes +} from "@simplex-chat/xftp-web/dist/protocol/encoding.js" diff --git a/smp-web/src/protocol.ts b/smp-web/src/protocol.ts new file mode 100644 index 0000000000..c32a332417 --- /dev/null +++ b/smp-web/src/protocol.ts @@ -0,0 +1,96 @@ +// SMP protocol commands and transmission format. +// Mirrors: Simplex.Messaging.Protocol + +import { + Decoder, concatBytes, + encodeBytes, decodeBytes, + decodeLarge +} from "@simplex-chat/xftp-web/dist/protocol/encoding.js" +import {readTag, readSpace} from "@simplex-chat/xftp-web/dist/protocol/commands.js" + +// -- Transmission encoding (Protocol.hs:2201-2203) +// encodeTransmission_ v (CorrId corrId, queueId, command) = +// smpEncode (corrId, queueId) <> encodeProtocol v command + +export function encodeTransmission(corrId: Uint8Array, entityId: Uint8Array, command: Uint8Array): Uint8Array { + return concatBytes( + encodeBytes(new Uint8Array(0)), // empty auth + encodeBytes(corrId), + encodeBytes(entityId), + command + ) +} + +// -- Transmission parsing (Protocol.hs:1629-1642) +// For implySessId = True (v7+): no sessId on wire + +export interface RawTransmission { + corrId: Uint8Array + entityId: Uint8Array + command: Uint8Array +} + +export function decodeTransmission(d: Decoder): RawTransmission { + const _auth = decodeBytes(d) // authenticator (empty for unsigned) + const corrId = decodeBytes(d) + const entityId = decodeBytes(d) + const command = d.takeAll() + return {corrId, entityId, command} +} + +// -- SMP command tags + +const SPACE = 0x20 + +function ascii(s: string): Uint8Array { + const buf = new Uint8Array(s.length) + for (let i = 0; i < s.length; i++) buf[i] = s.charCodeAt(i) + return buf +} + +// -- LGET command (Protocol.hs:1709) +// No parameters. EntityId carries LinkId in transmission. + +export function encodeLGET(): Uint8Array { + return ascii("LGET") +} + +// -- LNK response (Protocol.hs:1834) +// LNK sId d -> e (LNK_, ' ', sId, d) +// where d = (EncFixedDataBytes, EncUserDataBytes), both Large-encoded + +export interface LNKResponse { + senderId: Uint8Array + encFixedData: Uint8Array + encUserData: Uint8Array +} + +export function decodeLNK(d: Decoder): LNKResponse { + const senderId = decodeBytes(d) + const encFixedData = decodeLarge(d) + const encUserData = decodeLarge(d) + return {senderId, encFixedData, encUserData} +} + +// -- Response dispatch (same pattern as xftp-web decodeResponse) + +export type SMPResponse = + | {type: "LNK", response: LNKResponse} + | {type: "OK"} + | {type: "ERR", message: string} + +export function decodeResponse(d: Decoder): SMPResponse { + const tag = readTag(d) + switch (tag) { + case "LNK": { + readSpace(d) + return {type: "LNK", response: decodeLNK(d)} + } + case "OK": return {type: "OK"} + case "ERR": { + readSpace(d) + return {type: "ERR", message: readTag(d)} + } + default: throw new Error("unknown SMP response: " + tag) + } +} diff --git a/smp-web/tsconfig.json b/smp-web/tsconfig.json new file mode 100644 index 0000000000..2d66241bd4 --- /dev/null +++ b/smp-web/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ES2022", + "moduleResolution": "node", + "lib": ["ES2022"], + "outDir": "dist", + "rootDir": "src", + "declaration": true, + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "sourceMap": true + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "dist"] +} diff --git a/tests/SMPWebTests.hs b/tests/SMPWebTests.hs new file mode 100644 index 0000000000..cf21063439 --- /dev/null +++ b/tests/SMPWebTests.hs @@ -0,0 +1,92 @@ +{-# LANGUAGE OverloadedStrings #-} + +-- | Per-function tests for the smp-web TypeScript SMP client library. +-- Each test calls the Haskell function and the corresponding TypeScript function +-- via node, then asserts byte-identical output. +-- +-- Prerequisites: cd smp-web && npm install && npm run build +-- Run: cabal test --test-option=--match="/SMP Web Client/" +module SMPWebTests (smpWebTests) where + +import qualified Data.ByteString as B +import Simplex.Messaging.Encoding +import Test.Hspec hiding (it) +import Util +import XFTPWebTests (callNode_, jsOut, jsUint8) + +smpWebDir :: FilePath +smpWebDir = "smp-web" + +callNode :: String -> IO B.ByteString +callNode = callNode_ smpWebDir + +impProto :: String +impProto = "import { encodeTransmission, decodeTransmission, encodeLGET, decodeLNK, decodeResponse } from './dist/protocol.js';" + <> "import { Decoder } from '@simplex-chat/xftp-web/dist/protocol/encoding.js';" + +smpWebTests :: SpecWith () +smpWebTests = describe "SMP Web Client" $ do + describe "protocol" $ do + describe "transmission" $ do + it "encodeTransmission matches Haskell" $ do + let corrId = "1" + entityId = B.pack [1..24] + command = "LGET" + hsEncoded = smpEncode (corrId :: B.ByteString, entityId :: B.ByteString) <> command + tsEncoded <- callNode $ impProto + <> jsOut ("encodeTransmission(" + <> jsUint8 corrId <> "," + <> jsUint8 entityId <> "," + <> "new Uint8Array([0x4C,0x47,0x45,0x54])" -- "LGET" + <> ")") + -- TS encodes with empty auth prefix, HS encodeTransmission_ doesn't include auth + -- So TS output = [0x00] ++ hsEncoded + tsEncoded `shouldBe` (B.singleton 0 <> hsEncoded) + + it "decodeTransmission parses Haskell-encoded" $ do + let corrId = "abc" + entityId = B.pack [10..33] + command = "TEST" + encoded = smpEncode (B.empty :: B.ByteString) -- empty auth + <> smpEncode corrId + <> smpEncode entityId + <> command + -- TS decodes and returns corrId ++ entityId ++ command concatenated with length prefixes + tsResult <- callNode $ impProto + <> "const t = decodeTransmission(new Decoder(" <> jsUint8 encoded <> "));" + <> jsOut ("new Uint8Array([...t.corrId, ...t.entityId, ...t.command])") + tsResult `shouldBe` (corrId <> entityId <> command) + + describe "LGET" $ do + it "encodeLGET produces correct bytes" $ do + tsResult <- callNode $ impProto <> jsOut "encodeLGET()" + tsResult `shouldBe` "LGET" + + describe "LNK" $ do + it "decodeLNK parses correctly" $ do + let senderId = B.pack [1..24] + fixedData = B.pack [100..110] + userData = B.pack [200..220] + encoded = smpEncode senderId <> smpEncode (Large fixedData) <> smpEncode (Large userData) + tsResult <- callNode $ impProto + <> "const r = decodeLNK(new Decoder(" <> jsUint8 encoded <> "));" + <> jsOut ("new Uint8Array([...r.senderId, ...r.encFixedData, ...r.encUserData])") + tsResult `shouldBe` (senderId <> fixedData <> userData) + + describe "decodeResponse" $ do + it "decodes LNK response" $ do + let senderId = B.pack [1..24] + fixedData = B.pack [100..110] + userData = B.pack [200..220] + commandBytes = "LNK " <> smpEncode senderId <> smpEncode (Large fixedData) <> smpEncode (Large userData) + tsResult <- callNode $ impProto + <> "const r = decodeResponse(new Decoder(" <> jsUint8 commandBytes <> "));" + <> "if (r.type !== 'LNK') throw new Error('expected LNK, got ' + r.type);" + <> jsOut ("new Uint8Array([...r.response.senderId])") + tsResult `shouldBe` senderId + + it "decodes OK response" $ do + tsResult <- callNode $ impProto + <> "const r = decodeResponse(new Decoder(new Uint8Array([0x4F, 0x4B])));" -- "OK" + <> jsOut ("new Uint8Array([r.type === 'OK' ? 1 : 0])") + tsResult `shouldBe` B.singleton 1 diff --git a/tests/Test.hs b/tests/Test.hs index 63f97d8070..d945b549d0 100644 --- a/tests/Test.hs +++ b/tests/Test.hs @@ -37,6 +37,7 @@ import XFTPCLI import XFTPServerTests (xftpServerTests) import WebTests (webTests) import XFTPWebTests (xftpWebTests) +import SMPWebTests (smpWebTests) #if defined(dbPostgres) import Fixtures @@ -157,6 +158,7 @@ main = do #else describe "XFTP Web Client" $ xftpWebTests (pure ()) #endif + describe "SMP Web Client" smpWebTests describe "XRCP" remoteControlTests describe "Web" webTests describe "Server CLIs" cliTests diff --git a/tests/XFTPWebTests.hs b/tests/XFTPWebTests.hs index d948235244..c7b8723398 100644 --- a/tests/XFTPWebTests.hs +++ b/tests/XFTPWebTests.hs @@ -11,7 +11,7 @@ -- -- Prerequisites: cd xftp-web && npm install && npm run build -- Run: cabal test --test-option=--match="/XFTP Web Client/" -module XFTPWebTests (xftpWebTests) where +module XFTPWebTests (xftpWebTests, callNode_, jsOut, jsUint8, redirectConsole) where import Control.Concurrent (forkIO, newEmptyMVar, putMVar, takeMVar) import Control.Monad (replicateM, when) @@ -60,9 +60,9 @@ xftpWebDir = "xftp-web" redirectConsole :: String redirectConsole = "console.log = console.warn = (...a) => process.stderr.write(a.map(String).join(' ') + '\\n');" --- | Run an inline ES module script via node, return stdout as ByteString. -callNode :: String -> IO B.ByteString -callNode script = do +-- | Run an inline ES module script via node in a given directory, return stdout as ByteString. +callNode_ :: FilePath -> String -> IO B.ByteString +callNode_ dir script = do baseEnv <- getEnvironment let nodeEnv = ("NODE_TLS_REJECT_UNAUTHORIZED", "0") : baseEnv (_, Just hout, Just herr, ph) <- @@ -70,7 +70,7 @@ callNode script = do (proc "node" ["--input-type=module", "-e", redirectConsole <> script]) { std_out = CreatePipe, std_err = CreatePipe, - cwd = Just xftpWebDir, + cwd = Just dir, env = Just nodeEnv } errVar <- newEmptyMVar @@ -83,6 +83,9 @@ callNode script = do "node " <> show ec <> "\nstderr: " <> map (toEnum . fromIntegral) (B.unpack err) pure out +callNode :: String -> IO B.ByteString +callNode = callNode_ xftpWebDir + -- | Format a ByteString as a JS Uint8Array constructor. jsUint8 :: B.ByteString -> String jsUint8 bs = "new Uint8Array([" <> intercalate "," (map show (B.unpack bs)) <> "])" diff --git a/xftp-web/src/protocol/commands.ts b/xftp-web/src/protocol/commands.ts index 3ca43541fc..0bb4d6f597 100644 --- a/xftp-web/src/protocol/commands.ts +++ b/xftp-web/src/protocol/commands.ts @@ -81,7 +81,7 @@ export function encodePING(): Uint8Array { return ascii("PING") } // -- Response decoding -function readTag(d: Decoder): string { +export function readTag(d: Decoder): string { const start = d.offset() while (d.remaining() > 0) { if (d.buf[d.offset()] === 0x20 || d.buf[d.offset()] === 0x0a) break @@ -92,7 +92,7 @@ function readTag(d: Decoder): string { return s } -function readSpace(d: Decoder): void { +export function readSpace(d: Decoder): void { if (d.anyByte() !== 0x20) throw new Error("expected space") }