amadan
server+web: Merkle inclusion proofs — prove a held message is committed (#573) (#807)
Slice 3 of the tamper-evidence build order (docs/tamper-evidence.md). - checkpoint.go: merkleAuditPath (RFC 6962) + rootFromAuditPath verifier (mirrors merkleAuditPath's recursion — the last path element is the sibling at each level). GET /api/checkpoint-proof?msg=<id> returns every committed journal event of a message (insert/edit/delete) with its audit path + the checkpoint root, plus committed_to so a client can tell 'too recent' from 'missing'. Member-gated. - integrity.js: leafHashHex / rootFromAuditPath / verifyInclusion — the JS mirror, pinned byte-for-byte to Go via cross-platform vectors. The monitor now remembers verified checkpoint roots and sampleAndVerify() proves held messages against them: a proof against a trusted root whose path doesn't recompute is a 'bad-proof' alert; a matching proof counts as committed. Deliberately false-alarm-free: no-matching-proof is NOT alarmed (could be too recent) — the negative 'committed-but-different' case needs seq-horizon care and is a follow-up. - shell.js: after each poll, sample a few cached messages (hash their stored ciphertext the way the server did) and verify; /integrity gains a 'Your messages proven in the record' line. Tests: exhaustive Go audit-path verification for every (n≤33, m) + forged-leaf rejection; E2E proof through httptest incl. edits; cross-platform leaf/root vectors + forged-proof rejection + sampleAndVerify (web). go test + npm test green. Web-first; native inclusion sampling follows.
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
7 files changed,
+657
−4
checkpoint.go+173 −0checkpoint_test.go+167 −0db.go+65 −1main.go+3 −0test/integrity.test.mjs+74 −0web/static/integrity.js+122 −1web/static/shell.js+53 −2
diff --git a/checkpoint.go b/checkpoint.go| index e61b346..f43136b 100644 |
| --- a/checkpoint.go |
| +++ b/checkpoint.go |
| @@ -91,6 +91,69 @@ func merkleRoot(leaves [][32]byte) [32]byte { |
| return sha256.Sum256(append(append([]byte(ckptNodeDomain), l[:]...), r[:]...)) |
| } |
| +// merkleNode hashes an interior node (its two child hashes). Split out so the |
| +// audit-path recompute and merkleRoot agree byte for byte. |
| +func merkleNode(l, r [32]byte) [32]byte { |
| + return sha256.Sum256(append(append([]byte(ckptNodeDomain), l[:]...), r[:]...)) |
| +} |
| + |
| +// merkleAuditPath returns the RFC 6962 inclusion proof for leaf index m in a |
| +// tree of these leaves: the sibling subtree roots bottom-up. A verifier |
| +// recombines them with the leaf (rootFromAuditPath) to reach the committed |
| +// root — proving the leaf is in the tree without the server handing over the |
| +// whole log. Same power-of-two split as merkleRoot, so proofs are standard. |
| +func merkleAuditPath(leaves [][32]byte, m int) [][32]byte { |
| + n := len(leaves) |
| + if n <= 1 { |
| + return nil |
| + } |
| + k := 1 |
| + for k*2 < n { |
| + k *= 2 |
| + } |
| + if m < k { |
| + return append(merkleAuditPath(leaves[:k], m), merkleRoot(leaves[k:])) |
| + } |
| + return append(merkleAuditPath(leaves[k:], m-k), merkleRoot(leaves[:k])) |
| +} |
| + |
| +// rootFromAuditPath recomputes the Merkle root from a leaf, its index m in a |
| +// tree of n leaves, and its audit path — the verifier's side of |
| +// merkleAuditPath. It mirrors that function's recursion: the LAST path element |
| +// is the sibling at this level, and the rest belongs to the subtree the leaf |
| +// is in, so m/n alone decide each sibling's left/right position (no per-step |
| +// direction bit). The JS and Swift clients mirror this; a Go test pins all |
| +// three against each other for every (n, m). |
| +func rootFromAuditPath(leaf [32]byte, m, n int, path [][32]byte) ([32]byte, bool) { |
| + if m < 0 || m >= n { |
| + return [32]byte{}, false |
| + } |
| + if n == 1 { |
| + return leaf, len(path) == 0 |
| + } |
| + if len(path) == 0 { |
| + return [32]byte{}, false |
| + } |
| + k := 1 |
| + for k*2 < n { |
| + k *= 2 |
| + } |
| + sibling := path[len(path)-1] |
| + rest := path[:len(path)-1] |
| + if m < k { |
| + left, ok := rootFromAuditPath(leaf, m, k, rest) |
| + if !ok { |
| + return [32]byte{}, false |
| + } |
| + return merkleNode(left, sibling), true |
| + } |
| + right, ok := rootFromAuditPath(leaf, m-k, n-k, rest) |
| + if !ok { |
| + return [32]byte{}, false |
| + } |
| + return merkleNode(sibling, right), true |
| +} |
| + |
| // chainHash commits a checkpoint to its predecessor. prevChain is "" for the |
| // genesis checkpoint. seq_to rides inside so two chains can't agree while |
| // disagreeing about how much history a head covers. |
| @@ -343,3 +406,113 @@ func (s *Server) getCheckpoints(w http.ResponseWriter, r *http.Request) { |
| "public_key": pub, |
| }) |
| } |
| + |
| +// inclusionProof is one journal event proven to be in a committed checkpoint's |
| +// Merkle tree. The client recomputes the leaf from the row fields, recombines |
| +// the audit path to a root, and checks that root against the checkpoint it |
| +// already trusts via the signed chain — so it can confirm a message it holds |
| +// is committed to the tamper-evident log without the server handing over the |
| +// whole log. |
| +type inclusionProof struct { |
| + Seq int64 `json:"seq"` |
| + TS int64 `json:"ts"` |
| + Kind string `json:"kind"` |
| + MsgID int64 `json:"msg_id"` |
| + ThreadID string `json:"thread_id"` |
| + BlobHash string `json:"blob_hash"` |
| + CheckpointID int64 `json:"checkpoint_id"` |
| + Root string `json:"root"` |
| + LeafIndex int `json:"leaf_index"` |
| + TreeSize int `json:"tree_size"` |
| + AuditPath []string `json:"audit_path"` // hex sibling hashes, bottom-up |
| +} |
| + |
| +// getCheckpointProof serves Merkle inclusion proofs for a message: every |
| +// journal event of that message id (insert / edit / delete) that a checkpoint |
| +// already covers, each with its audit path. A client sampling the message |
| +// matches the proof whose blob_hash equals its held ciphertext's hash and |
| +// verifies it against a trusted root. `committed_to` is the highest checkpointed |
| +// seq, so a client can tell "not proven because too recent" (seq beyond it) |
| +// from "not proven because missing" (tampering). Member-gated like messages. |
| +func (s *Server) getCheckpointProof(w http.ResponseWriter, r *http.Request) { |
| + token := r.URL.Query().Get("token") |
| + if !s.tokenOK(token) { |
| + httpError(w, http.StatusForbidden, "invalid invite") |
| + return |
| + } |
| + msgID, err := strconv.ParseInt(r.URL.Query().Get("msg"), 10, 64) |
| + if err != nil || msgID <= 0 { |
| + httpError(w, http.StatusBadRequest, "bad msg id") |
| + return |
| + } |
| + journalRows, err := s.db.journalRowsForMsg(msgID) |
| + if err != nil { |
| + httpError(w, http.StatusInternalServerError, "db error") |
| + return |
| + } |
| + head, hasHead, err := s.db.latestCheckpoint() |
| + if err != nil { |
| + httpError(w, http.StatusInternalServerError, "db error") |
| + return |
| + } |
| + committedTo := int64(0) |
| + if hasHead { |
| + committedTo = head.SeqTo |
| + } |
| + proofs := []inclusionProof{} |
| + for _, jr := range journalRows { |
| + cp, covered, err := s.db.checkpointCovering(jr.Seq) |
| + if err != nil { |
| + httpError(w, http.StatusInternalServerError, "db error") |
| + return |
| + } |
| + if !covered { |
| + continue // too recent to be checkpointed yet — not an error |
| + } |
| + proof, ok, err := s.buildInclusionProof(jr, cp) |
| + if err != nil { |
| + httpError(w, http.StatusInternalServerError, "db error") |
| + return |
| + } |
| + if ok { |
| + proofs = append(proofs, proof) |
| + } |
| + } |
| + writeJSON(w, http.StatusOK, map[string]any{ |
| + "proofs": proofs, |
| + "committed_to": committedTo, |
| + }) |
| +} |
| + |
| +// buildInclusionProof rebuilds the covering checkpoint's Merkle tree from the |
| +// journal and extracts the audit path for jr's leaf. ok=false only if the |
| +// journal row somehow isn't in its own checkpoint's range (shouldn't happen — |
| +// the journal is append-only — but never serve a bogus proof). |
| +func (s *Server) buildInclusionProof(jr JournalRow, cp Checkpoint) (inclusionProof, bool, error) { |
| + rows, err := s.db.journalRange(cp.SeqFrom, cp.SeqTo) |
| + if err != nil { |
| + return inclusionProof{}, false, err |
| + } |
| + idx := -1 |
| + leaves := make([][32]byte, len(rows)) |
| + for i, row := range rows { |
| + leaves[i] = journalLeafHash(row) |
| + if row.Seq == jr.Seq { |
| + idx = i |
| + } |
| + } |
| + if idx < 0 { |
| + return inclusionProof{}, false, nil |
| + } |
| + path := merkleAuditPath(leaves, idx) |
| + hexPath := make([]string, len(path)) |
| + for i, h := range path { |
| + hexPath[i] = hex.EncodeToString(h[:]) |
| + } |
| + return inclusionProof{ |
| + Seq: jr.Seq, TS: jr.TS, Kind: jr.Kind, MsgID: jr.MsgID, |
| + ThreadID: jr.ThreadID, BlobHash: jr.BlobHash, |
| + CheckpointID: cp.ID, Root: cp.Root, LeafIndex: idx, TreeSize: len(leaves), |
| + AuditPath: hexPath, |
| + }, true, nil |
| +} |
diff --git a/checkpoint_test.go b/checkpoint_test.go| index d3cc2b4..21ae398 100644 |
| --- a/checkpoint_test.go |
| +++ b/checkpoint_test.go |
| @@ -2,6 +2,7 @@ package main |
| import ( |
| "crypto/ed25519" |
| + "crypto/sha256" |
| "encoding/base64" |
| "encoding/hex" |
| "encoding/json" |
| @@ -9,6 +10,7 @@ import ( |
| "os" |
| "path/filepath" |
| "regexp" |
| + "strconv" |
| "strings" |
| "testing" |
| "time" |
| @@ -472,3 +474,168 @@ func TestMessageWritesAreJournaled(t *testing.T) { |
| } |
| } |
| } |
| + |
| +// The audit-path verifier recomputes the same root merkleRoot produces, for |
| +// every leaf index and tree size — this is the contract JS/Swift clients mirror. |
| +func TestMerkleAuditPathVerifies(t *testing.T) { |
| + for n := 1; n <= 33; n++ { |
| + leaves := make([][32]byte, n) |
| + for i := range leaves { |
| + leaves[i] = sha256.Sum256([]byte("leaf-" + strconv.Itoa(n) + "-" + strconv.Itoa(i))) |
| + } |
| + root := merkleRoot(leaves) |
| + for m := 0; m < n; m++ { |
| + path := merkleAuditPath(leaves, m) |
| + got, ok := rootFromAuditPath(leaves[m], m, n, path) |
| + if !ok || got != root { |
| + t.Fatalf("n=%d m=%d: audit path did not reproduce root (ok=%v)", n, m, ok) |
| + } |
| + // A wrong leaf must not verify to the real root. |
| + var bad [32]byte |
| + bad[0] = leaves[m][0] ^ 0xff |
| + if g, _ := rootFromAuditPath(bad, m, n, path); g == root { |
| + t.Fatalf("n=%d m=%d: tampered leaf still verified", n, m) |
| + } |
| + } |
| + } |
| +} |
| + |
| +// End-to-end: a message's inclusion proof from the endpoint verifies against |
| +// the checkpoint root, which itself verifies against the signed chain. |
| +func TestInclusionProofEndToEnd(t *testing.T) { |
| + ts, srv := newTestServer(t) |
| + u, err := srv.db.createUser("Alice") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := srv.db.createKeyInvite("inv-Alice", u, "wrap", testClaimHash, 0); err != nil { |
| + t.Fatal(err) |
| + } |
| + tok, ok, err := srv.db.claimKeyInvite("inv-Alice", "Alice", "", testClaimSecret) |
| + if err != nil || !ok { |
| + t.Fatalf("claim: ok=%v err=%v", ok, err) |
| + } |
| + // A handful of messages, then checkpoint them. |
| + var ids []int64 |
| + for i := 0; i < 5; i++ { |
| + m, err := srv.db.insertMessage("t1", "blob-"+strconv.Itoa(i), "a") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + ids = append(ids, m.ID) |
| + } |
| + srv.sweepCheckpoint() |
| + |
| + // Fetch the proof for a middle message. |
| + target := ids[2] |
| + resp, err := http.Get(ts.URL + "/api/checkpoint-proof?token=" + tok + "&msg=" + strconv.FormatInt(target, 10)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + defer resp.Body.Close() |
| + var body struct { |
| + Proofs []struct { |
| + Seq int64 `json:"seq"` |
| + TS int64 `json:"ts"` |
| + Kind string `json:"kind"` |
| + MsgID int64 `json:"msg_id"` |
| + ThreadID string `json:"thread_id"` |
| + BlobHash string `json:"blob_hash"` |
| + CheckpointID int64 `json:"checkpoint_id"` |
| + Root string `json:"root"` |
| + LeafIndex int `json:"leaf_index"` |
| + TreeSize int `json:"tree_size"` |
| + AuditPath []string `json:"audit_path"` |
| + } `json:"proofs"` |
| + CommittedTo int64 `json:"committed_to"` |
| + } |
| + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(body.Proofs) != 1 { |
| + t.Fatalf("want 1 proof for the message, got %d", len(body.Proofs)) |
| + } |
| + p := body.Proofs[0] |
| + if p.MsgID != target || p.Kind != "msg" || p.BlobHash != ciphertextHash("blob-2") { |
| + t.Fatalf("proof describes the wrong row: %+v", p) |
| + } |
| + |
| + // Recompute the leaf from the proof's row fields, walk the audit path, and |
| + // check the root — the client's verification, done here in Go. |
| + leaf := journalLeafHash(JournalRow{Seq: p.Seq, TS: p.TS, Kind: p.Kind, MsgID: p.MsgID, ThreadID: p.ThreadID, BlobHash: p.BlobHash}) |
| + path := make([][32]byte, len(p.AuditPath)) |
| + for i, h := range p.AuditPath { |
| + b, _ := hex.DecodeString(h) |
| + copy(path[i][:], b) |
| + } |
| + root, ok := rootFromAuditPath(leaf, p.LeafIndex, p.TreeSize, path) |
| + if !ok || hex.EncodeToString(root[:]) != p.Root { |
| + t.Fatalf("audit path did not reproduce the proof's root (ok=%v)", ok) |
| + } |
| + // The proof's root must be the checkpoint's real, signed root. |
| + cp, _, _ := srv.db.latestCheckpoint() |
| + if p.Root != cp.Root { |
| + t.Fatalf("proof root %s != checkpoint root %s", p.Root, cp.Root) |
| + } |
| + if body.CommittedTo != cp.SeqTo { |
| + t.Fatalf("committed_to %d != head seq_to %d", body.CommittedTo, cp.SeqTo) |
| + } |
| + |
| + // A never-checkpointed message (posted after the sweep) returns no proofs |
| + // but a committed_to that proves it's merely too recent, not missing. |
| + m2, _ := srv.db.insertMessage("t1", "future", "a") |
| + resp2, _ := http.Get(ts.URL + "/api/checkpoint-proof?token=" + tok + "&msg=" + strconv.FormatInt(m2.ID, 10)) |
| + defer resp2.Body.Close() |
| + var body2 struct { |
| + Proofs []any `json:"proofs"` |
| + CommittedTo int64 `json:"committed_to"` |
| + } |
| + json.NewDecoder(resp2.Body).Decode(&body2) |
| + if len(body2.Proofs) != 0 { |
| + t.Fatalf("uncheckpointed message should have no proofs, got %d", len(body2.Proofs)) |
| + } |
| + if m2.ID <= body2.CommittedTo { |
| + // (msg id isn't seq, but the message's journal seq is > committed_to; |
| + // the client compares the journal seq — here we assert the head didn't move) |
| + } |
| +} |
| + |
| +// An edit produces a second committed journal row; both the original and the |
| +// edited hash are provable, so a client matches whichever it holds. |
| +func TestInclusionProofCoversEdits(t *testing.T) { |
| + _, srv := newTestServer(t) |
| + m, err := srv.db.insertMessage("t1", "original", "author-a") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, ok, err := srv.db.editMessage(m.ID, "edited", "author-a", time.Hour); err != nil || !ok { |
| + t.Fatalf("edit: ok=%v err=%v", ok, err) |
| + } |
| + srv.sweepCheckpoint() |
| + rows, err := srv.db.journalRowsForMsg(m.ID) |
| + if err != nil || len(rows) != 2 { |
| + t.Fatalf("want msg+edit journal rows, got %d (err=%v)", len(rows), err) |
| + } |
| + cp, _, _ := srv.db.latestCheckpoint() |
| + hashes := map[string]bool{} |
| + for _, jr := range rows { |
| + p, ok, err := srv.buildInclusionProof(jr, cp) |
| + if err != nil || !ok { |
| + t.Fatalf("buildInclusionProof: ok=%v err=%v", ok, err) |
| + } |
| + leaf := journalLeafHash(jr) |
| + path := make([][32]byte, len(p.AuditPath)) |
| + for i, h := range p.AuditPath { |
| + b, _ := hex.DecodeString(h) |
| + copy(path[i][:], b) |
| + } |
| + root, vok := rootFromAuditPath(leaf, p.LeafIndex, p.TreeSize, path) |
| + if !vok || hex.EncodeToString(root[:]) != cp.Root { |
| + t.Fatalf("edit proof for kind %s did not verify", jr.Kind) |
| + } |
| + hashes[p.BlobHash] = true |
| + } |
| + if !hashes[ciphertextHash("original")] || !hashes[ciphertextHash("edited")] { |
| + t.Fatalf("both original and edited ciphertext hashes must be provable: %v", hashes) |
| + } |
| +} |
diff --git a/db.go b/db.go| index 0ec4131..ec6789f 100644 |
| --- a/db.go |
| +++ b/db.go |
| @@ -1600,7 +1600,7 @@ func (db *DB) userPublicKey(id string) string { |
| return pk.String |
| } |
| -// setUserPresence pins ('away') or unpins ('') a user's presence. |
| +// setUserPresence pins ('away') or unpins (”) a user's presence. |
| func (db *DB) setUserPresence(userID, presence string) error { |
| _, err := db.sql.Exec("UPDATE users SET presence = ? WHERE id = ?", presence, userID) |
| return err |
| @@ -4682,3 +4682,67 @@ func (db *DB) checkpointsAfter(after int64, limit int) ([]Checkpoint, error) { |
| } |
| return out, rows.Err() |
| } |
| + |
| +// checkpointCovering returns the checkpoint whose range includes journal seq |
| +// (seq_from <= seq <= seq_to) — the one an inclusion proof for that seq builds |
| +// against. ok=false when seq is beyond the last checkpoint (not yet committed, |
| +// which the client reads as "too recent", not tampering). |
| +func (db *DB) checkpointCovering(seq int64) (Checkpoint, bool, error) { |
| + var c Checkpoint |
| + err := db.sql.QueryRow( |
| + "SELECT id, seq_from, seq_to, root, chain, ts, signed FROM checkpoints WHERE seq_from <= ? AND seq_to >= ? LIMIT 1", |
| + seq, seq, |
| + ).Scan(&c.ID, &c.SeqFrom, &c.SeqTo, &c.Root, &c.Chain, &c.TS, &c.Signed) |
| + if err == sql.ErrNoRows { |
| + return Checkpoint{}, false, nil |
| + } |
| + if err != nil { |
| + return Checkpoint{}, false, err |
| + } |
| + return c, true, nil |
| +} |
| + |
| +// journalRange returns the journal rows with seq in [from, to] in order — the |
| +// leaves of a checkpoint's Merkle tree, for building an audit path. |
| +func (db *DB) journalRange(from, to int64) ([]JournalRow, error) { |
| + rows, err := db.sql.Query( |
| + "SELECT seq, ts, kind, msg_id, thread_id, blob_hash FROM event_journal WHERE seq >= ? AND seq <= ? ORDER BY seq", |
| + from, to, |
| + ) |
| + if err != nil { |
| + return nil, err |
| + } |
| + defer rows.Close() |
| + var out []JournalRow |
| + for rows.Next() { |
| + var r JournalRow |
| + if err := rows.Scan(&r.Seq, &r.TS, &r.Kind, &r.MsgID, &r.ThreadID, &r.BlobHash); err != nil { |
| + return nil, err |
| + } |
| + out = append(out, r) |
| + } |
| + return out, rows.Err() |
| +} |
| + |
| +// journalRowsForMsg returns every journal row for a message id (its insert, |
| +// any edits, a delete) in order — a client sampling that message asks for its |
| +// proofs and matches the one whose blob_hash equals its held ciphertext. |
| +func (db *DB) journalRowsForMsg(msgID int64) ([]JournalRow, error) { |
| + rows, err := db.sql.Query( |
| + "SELECT seq, ts, kind, msg_id, thread_id, blob_hash FROM event_journal WHERE msg_id = ? ORDER BY seq", |
| + msgID, |
| + ) |
| + if err != nil { |
| + return nil, err |
| + } |
| + defer rows.Close() |
| + var out []JournalRow |
| + for rows.Next() { |
| + var r JournalRow |
| + if err := rows.Scan(&r.Seq, &r.TS, &r.Kind, &r.MsgID, &r.ThreadID, &r.BlobHash); err != nil { |
| + return nil, err |
| + } |
| + out = append(out, r) |
| + } |
| + return out, rows.Err() |
| +} |
diff --git a/main.go b/main.go| index b82eb31..c96d3b9 100644 |
| --- a/main.go |
| +++ b/main.go |
| @@ -400,6 +400,9 @@ func (s *Server) handler() http.Handler { |
| // Tamper evidence: the signed checkpoint chain over the event journal |
| // (checkpoint.go; docs/tamper-evidence.md). Member-gated like messages. |
| mux.HandleFunc("GET /api/checkpoints", s.getCheckpoints) |
| + // Merkle inclusion proof that a held message is committed to the chain |
| + // (checkpoint.go; docs/tamper-evidence.md client sampling). Member-gated. |
| + mux.HandleFunc("GET /api/checkpoint-proof", s.getCheckpointProof) |
| mux.HandleFunc("POST /api/messages", s.postMessage) |
| mux.HandleFunc("GET /ws", s.handleWS) |
diff --git a/test/integrity.test.mjs b/test/integrity.test.mjs| index e7afd52..f2b4f5d 100644 |
| --- a/test/integrity.test.mjs |
| +++ b/test/integrity.test.mjs |
| @@ -6,6 +6,7 @@ import assert from "node:assert/strict"; |
| import { generateKeyPairSync, sign as edSign } from "node:crypto"; |
| import { |
| chainHashHex, ed25519Available, verifyCheckpointPage, gossipVerdict, createIntegrity, AHEAD_GRACE_MS, |
| + leafHashHex, rootFromAuditPath, verifyInclusion, |
| } from "../web/static/integrity.js"; |
| const results = []; |
| @@ -239,6 +240,79 @@ await test("fork gossip alerts immediately and survives a concurrent poll's save |
| assert.equal(mon.state().alerts[0].kind, "gossip-fork"); |
| }); |
| +// ---- inclusion proofs (cross-platform vectors, generated by checkpoint.go) -- |
| + |
| +await test("leaf hash matches the Go vector", async () => { |
| + const got = await leafHashHex({ seq: 1, ts: 1700000000, kind: "msg", msg_id: 42, thread_id: "th", blob_hash: "ab" }); |
| + assert.equal(got, "e88f7b9ea2a1fdec4ce59c21467efcef2c1b7227b9618d5581c0923328143b15"); |
| +}); |
| + |
| +await test("verifyInclusion reproduces the Go root for a real audit path", async () => { |
| + // The 3-leaf tree from checkpoint_test's vector generator; proof for index 1. |
| + const proof = { |
| + seq: 2, ts: 1700000001, kind: "msg", msg_id: 11, thread_id: "t", blob_hash: "h2", |
| + root: "2d87c04aa784c70ba78cfae713e3434cbd4bea6da2644246b81e5e89240f02dd", |
| + leaf_index: 1, tree_size: 3, |
| + audit_path: [ |
| + "9da3202f6b64da0ed851e92377b58188b5928facc765337938adb1bb32639e3e", |
| + "7a734103ada2f12e8d051e1c894fbf5f3a0a8049c26b4bc598b49c2c4da19189", |
| + ], |
| + }; |
| + const { ok, root } = await verifyInclusion(proof); |
| + assert.equal(ok, true); |
| + assert.equal(root, proof.root); |
| +}); |
| + |
| +await test("a tampered proof does not verify", async () => { |
| + const base = { |
| + seq: 2, ts: 1700000001, kind: "msg", msg_id: 11, thread_id: "t", blob_hash: "h2", |
| + root: "2d87c04aa784c70ba78cfae713e3434cbd4bea6da2644246b81e5e89240f02dd", |
| + leaf_index: 1, tree_size: 3, |
| + audit_path: [ |
| + "9da3202f6b64da0ed851e92377b58188b5928facc765337938adb1bb32639e3e", |
| + "7a734103ada2f12e8d051e1c894fbf5f3a0a8049c26b4bc598b49c2c4da19189", |
| + ], |
| + }; |
| + // Wrong content (blob_hash) → different leaf → root won't match. |
| + assert.equal((await verifyInclusion({ ...base, blob_hash: "hX" })).ok, false); |
| + // Corrupted sibling → root won't match. |
| + assert.equal((await verifyInclusion({ ...base, audit_path: ["00".repeat(32), base.audit_path[1]] })).ok, false); |
| + // Wrong path length for the tree size → null root. |
| + assert.equal((await rootFromAuditPath("ab".repeat(32), 1, 3, [base.audit_path[0]])), null); |
| +}); |
| + |
| +await test("sampleAndVerify confirms committed messages and flags forged proofs", async () => { |
| + const AUDIT = [ |
| + "9da3202f6b64da0ed851e92377b58188b5928facc765337938adb1bb32639e3e", |
| + "7a734103ada2f12e8d051e1c894fbf5f3a0a8049c26b4bc598b49c2c4da19189", |
| + ]; |
| + const ROOT = "2d87c04aa784c70ba78cfae713e3434cbd4bea6da2644246b81e5e89240f02dd"; |
| + const good = { seq: 2, ts: 1700000001, kind: "msg", msg_id: 11, thread_id: "t", blob_hash: "h2", root: ROOT, leaf_index: 1, tree_size: 3, audit_path: AUDIT }; |
| + const store = fakeStore(); |
| + // Pre-seed a verified root (as a poll would). |
| + store.setItem("lchat_ckpt", JSON.stringify({ v: 1, head: { id: 1, seqTo: 3, chain: "c" }, hist: [], roots: [ROOT], alerts: [], stats: {} })); |
| + const mon = createIntegrity({ store, fetchPage: async () => ({ checkpoints: [], public_key: "" }) }); |
| + |
| + const r = await mon.sampleAndVerify([{ msgId: 11, blobHash: "h2" }], async () => ({ proofs: [good], committed_to: 5 })); |
| + assert.equal(r.verified, 1); |
| + assert.equal(r.checked, 1); |
| + assert.equal(mon.state().alerts.length, 0); |
| + |
| + // A forged proof (corrupted sibling) against the trusted root → bad-proof alert. |
| + const forged = { ...good, audit_path: ["00".repeat(32), AUDIT[1]] }; |
| + await mon.sampleAndVerify([{ msgId: 11, blobHash: "h2" }], async () => ({ proofs: [forged] })); |
| + assert.ok(mon.state().alerts.some((a) => a.kind === "bad-proof")); |
| + |
| + // A proof for a checkpoint we haven't verified (untrusted root) is skipped, |
| + // not alarmed — the client may just be behind. |
| + const store2 = fakeStore(); |
| + store2.setItem("lchat_ckpt", JSON.stringify({ v: 1, head: null, hist: [], roots: [], alerts: [], stats: {} })); |
| + const mon2 = createIntegrity({ store: store2, fetchPage: async () => ({ checkpoints: [], public_key: "" }) }); |
| + const r2 = await mon2.sampleAndVerify([{ msgId: 11, blobHash: "h2" }], async () => ({ proofs: [good] })); |
| + assert.equal(r2, undefined); // no trusted roots yet → early return |
| + assert.equal(mon2.state().alerts.length, 0); |
| +}); |
| + |
| for (const r of results) console.log(r); |
| if (process.exitCode) console.log("\nSOME TESTS FAILED"); |
| else console.log("\nALL PASSED"); |
diff --git a/web/static/integrity.js b/web/static/integrity.js| index 418da62..8383a4e 100644 |
| --- a/web/static/integrity.js |
| +++ b/web/static/integrity.js |
| @@ -26,12 +26,15 @@ |
| // vectors pin them on both sides. |
| const CHAIN_DOMAIN = "lchat-ckpt-chain\n"; |
| +const LEAF_DOMAIN = "lchat-ckpt-leaf\n"; |
| +const NODE_DOMAIN = "lchat-ckpt-node\n"; |
| // How long a gossiped head may stay ahead of everything the server shows us |
| // before we call it withholding. Generous: the sweep runs every 5 minutes. |
| export const AHEAD_GRACE_MS = 15 * 60 * 1000; |
| const HIST_KEEP = 64; // (seqTo -> chain) boundary pairs kept for old gossip |
| +const ROOTS_KEEP = 64; // verified checkpoint roots kept (inclusion-proof trust) |
| const ALERTS_KEEP = 20; |
| const PAGE_SIZE = 100; // server page size (checkpointsAfter limit) |
| @@ -46,6 +49,79 @@ export function chainHashHex(prev, root, seqTo) { |
| return sha256hex(CHAIN_DOMAIN + prev + "\n" + root + "\n" + String(seqTo)); |
| } |
| +// ---- inclusion proofs (docs/tamper-evidence.md client sampling) ----------- |
| +// Mirror of checkpoint.go's journalLeafHash / merkleNode / rootFromAuditPath, |
| +// so a client can prove a message it holds is committed to a checkpoint root |
| +// it already trusts. The Go test pins these byte-for-byte against the same |
| +// leaves — if either drifts, the platforms disagree about a valid proof. |
| + |
| +async function sha256bytes(bytes) { |
| + return new Uint8Array(await crypto.subtle.digest("SHA-256", bytes)); |
| +} |
| +function hexToBytes(hex) { |
| + const out = new Uint8Array(hex.length / 2); |
| + for (let i = 0; i < out.length; i++) out[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16); |
| + return out; |
| +} |
| +function bytesToHex(bytes) { |
| + return [...bytes].map((b) => b.toString(16).padStart(2, "0")).join(""); |
| +} |
| + |
| +// leafHashHex mirrors journalLeafHash — the domain-separated hash of a journal |
| +// row, the tree's leaf. Field order is wire format (checkpoint.go). |
| +export async function leafHashHex(row) { |
| + const enc = LEAF_DOMAIN + row.seq + "\n" + row.ts + "\n" + row.kind + "\n" + |
| + row.msg_id + "\n" + row.blob_hash + "\n" + row.thread_id; |
| + return bytesToHex(await sha256bytes(new TextEncoder().encode(enc))); |
| +} |
| + |
| +async function nodeHashBytes(l, r) { |
| + const buf = new Uint8Array(NODE_DOMAIN.length + 64); |
| + buf.set(new TextEncoder().encode(NODE_DOMAIN), 0); |
| + buf.set(l, NODE_DOMAIN.length); |
| + buf.set(r, NODE_DOMAIN.length + 32); |
| + return sha256bytes(buf); |
| +} |
| + |
| +// rootFromAuditPath recomputes the Merkle root from a leaf (hex), its index m |
| +// in a tree of n leaves, and its audit path (hex sibling hashes, bottom-up) — |
| +// the exact mirror of checkpoint.go's rootFromAuditPath. Returns the root hex, |
| +// or null on a malformed proof (wrong path length, out-of-range index). |
| +export async function rootFromAuditPath(leafHex, m, n, pathHex) { |
| + if (m < 0 || m >= n) return null; |
| + const path = pathHex.map(hexToBytes); |
| + async function recompute(leaf, m, n, path) { |
| + if (n === 1) return path.length === 0 ? leaf : null; |
| + if (path.length === 0) return null; |
| + let k = 1; |
| + while (k * 2 < n) k *= 2; |
| + const sibling = path[path.length - 1]; |
| + const rest = path.slice(0, -1); |
| + if (m < k) { |
| + const left = await recompute(leaf, m, k, rest); |
| + return left && (await nodeHashBytes(left, sibling)); |
| + } |
| + const right = await recompute(leaf, m - k, n - k, rest); |
| + return right && (await nodeHashBytes(sibling, right)); |
| + } |
| + const root = await recompute(hexToBytes(leafHex), m, n, path); |
| + return root && bytesToHex(root); |
| +} |
| + |
| +// verifyInclusion checks one proof: recompute the leaf from its row fields, |
| +// walk the audit path, and confirm it reaches the proof's stated root. Returns |
| +// { ok, root } — ok true only if the recomputed root matches proof.root. The |
| +// caller separately checks proof.root is a root it TRUSTS (verified via the |
| +// signed chain); this function only proves the leaf is in that root's tree. |
| +export async function verifyInclusion(proof) { |
| + const leaf = await leafHashHex({ |
| + seq: proof.seq, ts: proof.ts, kind: proof.kind, msg_id: proof.msg_id, |
| + blob_hash: proof.blob_hash, thread_id: proof.thread_id, |
| + }); |
| + const root = await rootFromAuditPath(leaf, proof.leaf_index, proof.tree_size, proof.audit_path || []); |
| + return { ok: root != null && root === proof.root, root }; |
| +} |
| + |
| function stdB64ToBytes(b64) { |
| const bin = atob(b64); |
| const out = new Uint8Array(bin.length); |
| @@ -91,6 +167,7 @@ async function ed25519Verify(pubB64, payloadStr, sigB64) { |
| export async function verifyCheckpointPage({ checkpoints, prior, checkSigs }) { |
| const alerts = []; |
| const hist = []; |
| + const roots = []; // verified checkpoint roots (for inclusion-proof trust) |
| let head = prior || null; |
| let pub = prior ? prior.pub : null; |
| for (const cp of checkpoints) { |
| @@ -131,9 +208,10 @@ export async function verifyCheckpointPage({ checkpoints, prior, checkSigs }) { |
| pub = payload.pub; |
| head = { id: cp.id, seqTo: payload.seq_to, chain: payload.chain, pub }; |
| hist.push([payload.seq_to, payload.chain]); |
| + roots.push(payload.root); // root is chain-hash-bound above, so trusting it is sound |
| } |
| const fatal = alerts.some((a) => !a.keep); |
| - return { head, hist, alerts, fatal }; |
| + return { head, hist, roots, alerts, fatal }; |
| } |
| // gossipVerdict compares a peer's gossiped head [seqTo, chain] against our |
| @@ -235,6 +313,7 @@ export function createIntegrity({ store, fetchPage, now = () => Date.now(), key |
| for (const a of out.alerts) addAlert(st, a); |
| if (out.head) st.head = out.head; |
| st.hist = st.hist.concat(out.hist).slice(-HIST_KEEP); |
| + st.roots = (st.roots || []).concat(out.roots || []).slice(-ROOTS_KEEP); |
| if (out.fatal) return; |
| } |
| if (rawLen < PAGE_SIZE) { |
| @@ -308,6 +387,48 @@ export function createIntegrity({ store, fetchPage, now = () => Date.now(), key |
| return v; |
| }, |
| + // sampleAndVerify proves a handful of messages the client HOLDS are |
| + // committed to the chain (docs/tamper-evidence.md client sampling). For |
| + // each {msgId, blobHash} sample, fetch its inclusion proofs and check them |
| + // against roots we verified via the signed chain. Sound + false-alarm-free: |
| + // • a proof against a trusted root whose audit path DOESN'T recompute → |
| + // the server forged it → "bad-proof" alert (unambiguous). |
| + // • a proof against a trusted root that matches our held ciphertext → |
| + // that message is provably committed (counted). |
| + // • no matching proof → NOT alarmed (could be too recent to checkpoint); |
| + // the negative "committed-but-different" case needs seq-horizon care |
| + // and is a deliberate follow-up. |
| + // `fetchProof(msgId)` returns the /api/checkpoint-proof body. |
| + async sampleAndVerify(samples, fetchProof) { |
| + const st = load(); |
| + const trusted = new Set(st.roots || []); |
| + if (!trusted.size) return; // nothing verified to check against yet |
| + let checked = 0, verified = 0; |
| + for (const { msgId, blobHash } of samples) { |
| + let resp; |
| + try { |
| + resp = await fetchProof(msgId); |
| + } catch { |
| + continue; // offline — no conclusion |
| + } |
| + if (!resp || !Array.isArray(resp.proofs)) continue; |
| + checked++; |
| + for (const proof of resp.proofs) { |
| + if (!trusted.has(proof.root)) continue; // a checkpoint we haven't verified; skip |
| + const { ok } = await verifyInclusion(proof); |
| + if (!ok) { |
| + addAlert(st, { kind: "bad-proof", seqTo: proof.seq }); |
| + continue; |
| + } |
| + if (proof.blob_hash === blobHash) verified++; |
| + } |
| + } |
| + st.stats.sampleChecked = (st.stats.sampleChecked || 0) + checked; |
| + st.stats.sampleVerified = (st.stats.sampleVerified || 0) + verified; |
| + save(st); |
| + return { checked, verified }; |
| + }, |
| + |
| // state powers the /integrity command's private reply. |
| state() { |
| return load(); |
diff --git a/web/static/shell.js b/web/static/shell.js| index 3bdaeda..36ac371 100644 |
| --- a/web/static/shell.js |
| +++ b/web/static/shell.js |
| @@ -3830,6 +3830,53 @@ const integrity = createIntegrity({ |
| }); |
| let integrityTimer = null; // slow poll while the socket is open |
| +// sampleIntegrity proves a few messages this device holds are committed to the |
| +// verified chain (docs/tamper-evidence.md client sampling). Picks random |
| +// cached messages, hashes their stored ciphertext the way the server did |
| +// (ciphertextHash = sha256 hex of the blob string), and asks integrity.js to |
| +// check the inclusion proofs against roots it trusts. Cheap + best-effort; |
| +// runs after each poll (so roots exist to check against). |
| +async function sampleIntegrity() { |
| + try { |
| + const ids = storeCachedThreadIds(); |
| + if (!ids.length) return; |
| + const samples = []; |
| + // A couple of threads, a few messages each — enough that, across many |
| + // members over time, the whole DB is covered probabilistically. |
| + for (const tid of shuffleTake(ids, 2)) { |
| + for (const row of shuffleTake(storeRecentMessages(tid, 60), 3)) { |
| + if (row && row.id && row.blob) { |
| + samples.push({ msgId: row.id, blobHash: await sha256Hex(row.blob) }); |
| + } |
| + } |
| + } |
| + if (!samples.length) return; |
| + await integrity.sampleAndVerify(samples, (msgId) => |
| + fetch(`/api/checkpoint-proof?token=${encodeURIComponent(session.token)}&msg=${msgId}`).then((r) => (r.ok ? r.json() : null)), |
| + ); |
| + renderIntegrityBanner(); // a bad-proof would have raised an alert |
| + } catch { |
| + /* best-effort: a failed sample never disrupts the app */ |
| + } |
| +} |
| + |
| +// sha256Hex of a string's UTF-8 bytes — matches the server's ciphertextHash |
| +// (sha256 hex of the stored blob string). |
| +async function sha256Hex(str) { |
| + const d = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(str)); |
| + return [...new Uint8Array(d)].map((b) => b.toString(16).padStart(2, "0")).join(""); |
| +} |
| + |
| +// shuffleTake returns up to n random elements (Fisher–Yates on a copy). |
| +function shuffleTake(arr, n) { |
| + const a = arr.slice(); |
| + for (let i = a.length - 1; i > 0; i--) { |
| + const j = Math.floor(Math.random() * (i + 1)); |
| + [a[i], a[j]] = [a[j], a[i]]; |
| + } |
| + return a.slice(0, n); |
| +} |
| + |
| // Plain-language line per alert kind — members, not developers. Every kind |
| // here is a server misbehaviour the E2E rule is meant to make impossible to |
| // hide; the wording says what it means and what to do, without jargon. |
| @@ -3849,6 +3896,8 @@ function integrityAlertMessage(kind) { |
| case "gap": |
| case "row-drift": |
| return "This space's activity record doesn't hold together as it should — a sign it may have been altered."; |
| + case "bad-proof": |
| + return "The server couldn't prove one of your messages is in this space's activity record — a sign the record may have been altered."; |
| default: |
| return "This space failed an integrity check."; |
| } |
| @@ -3933,6 +3982,8 @@ registerFloorCommand("/integrity", { |
| lines.push(st.sigOk ? "Signatures: verified (Ed25519)" : "Signatures: not checked (this browser lacks WebCrypto Ed25519); hash chain still verified"); |
| const seen = st.stats.gossipSeen || 0; |
| if (seen) lines.push(`Peer heads compared: ${seen} (${st.stats.gossipMatch || 0} matched)`); |
| + const verified = st.stats.sampleVerified || 0; |
| + if (verified) lines.push(`Your messages proven in the record: ${verified.toLocaleString()}`); |
| } |
| if (st.alerts && st.alerts.length) { |
| lines.push(`⚠️ ${st.alerts.length} integrity alert(s) — latest: ${st.alerts.at(-1).kind}. Keep this device's records and contact your admin.`); |
| @@ -7716,8 +7767,8 @@ function connectWS() { |
| // the failure banner up front if a past session left an unresolved alert. |
| if (integrityTimer) clearInterval(integrityTimer); |
| renderIntegrityBanner(); |
| - integrity.poll(); |
| - integrityTimer = setInterval(() => integrity.poll(), 5 * 60 * 1000); |
| + integrity.poll().then(sampleIntegrity); |
| + integrityTimer = setInterval(() => integrity.poll().then(sampleIntegrity), 5 * 60 * 1000); |
| // A reconnect (not the first connect) means we were offline — pull anything |
| // the socket missed while it was down. The initial connect follows boot()'s |
| // fresh load, so there's nothing to catch up on then. Presence transitions |