amadan
server: event journal + chained signed Merkle checkpoints (#573 slice 1) (#752)
* server: event journal + chained signed Merkle checkpoints (#573 slice 1) Every message insert/edit/delete appends a row to an append-only event_journal (ciphertext hashes only — unsend still destroys the blob) in the same transaction as the write. A background sweep folds new rows into checkpoints: an RFC 6962-style Merkle root over the batch, a chain hash committing to the previous checkpoint, and an Ed25519 signature in the release.go signed-manifest shape. Served member-gated at GET /api/checkpoints; the signing key lives next to the DB (never in it), so the escrow replica can't re-sign history. Design + the settled #573 questions: docs/tamper-evidence.md (companion PR). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * server: review fixes — DB-resident seed, backfill, chain-fork guard Adversarial review pass (5 finder angles) before morning review; fixes: - seed default moves into the DB config so it rides hibernation wakes and VACUUM-INTO migrations (file override via LCHAT_CHECKPOINT_KEY_FILE, temp+rename write, corrupt file = loud error); each checkpoint payload embeds its public key so a key change is explicit, never a false alarm - backfill pre-journal messages at migration (config-marked, one tx) — editing/unsending a legacy message no longer reads as tampering - key load caches success only (was: sync.Once pinned a transient error for the process lifetime); endpoint serves the chain even when the current key is unavailable - sweep mutex + UNIQUE(seq_from) so a second writer can't fork the chain; head carried forward locally instead of re-read per batch - journalMessageDeletesTx is one INSERT..SELECT (was N inserts holding the single connection during member purges) - log.Printf like every other sweeper; wire-format extensibility comments (fields after kind are kind-scoped); TestMessageWritesAreJournaled source guard so future write paths can't bypass the journal silently Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * server: verify-checkpoints tool — independent E2E chain verification Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> ---------
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
5 files changed,
+1269
−8
checkpoint.go+345 −0checkpoint_test.go+471 −0db.go+312 −8main.go+21 −0tools/verify-checkpoints.mjs+120 −0
diff --git a/checkpoint.go b/checkpoint.go| new file mode 100644 |
| index 0000000..e61b346 |
| --- /dev/null |
| +++ b/checkpoint.go |
| @@ -0,0 +1,345 @@ |
| +// checkpoint.go — tamper-evidence, part 1: chained, signed Merkle checkpoints |
| +// over the append-only event journal (docs/tamper-evidence.md, issue #573). |
| +// |
| +// The message table mutates on purpose (edits UPDATE in place, unsend hard- |
| +// DELETEs), so the unit of commitment is the event_journal: every insert / |
| +// edit / delete appends a row in the same transaction (db.go). A background |
| +// sweep periodically folds all new journal rows into a checkpoint: a Merkle |
| +// root over the batch, a chain hash committing to the previous checkpoint, |
| +// and an Ed25519 signature over the encoded payload (the release.go signed- |
| +// manifest pattern — sign the base64 string, so there is no JSON |
| +// canonicalization to get wrong). |
| +// |
| +// The signing key is NOT the security boundary — a malicious host runs the |
| +// process and holds the seed. The property is chain continuity: a host can |
| +// forge a fresh chain, but not one that both extends a head someone already |
| +// witnessed (a client, the escrow replica) and omits what that head committed |
| +// to. The seed therefore lives in the DB config by default, so it survives |
| +// every path that moves instance state (hibernation wake, VACUUM-INTO |
| +// migration, escrow restore) — a key that silently changes on routine ops |
| +// would make every legitimate restore read as forgery. The trade: anyone |
| +// holding a DB copy can sign as the instance; acceptable, because the |
| +// signature is attribution, not the security (docs/tamper-evidence.md). |
| +// LCHAT_CHECKPOINT_KEY_FILE keeps the seed off-DB for operators who'd rather |
| +// carry the file themselves. Each checkpoint embeds its public key, so a key |
| +// change is an explicit, investigable event — never a silent false alarm. |
| +package main |
| + |
| +import ( |
| + "crypto/ed25519" |
| + "crypto/rand" |
| + "crypto/sha256" |
| + "encoding/base64" |
| + "encoding/hex" |
| + "encoding/json" |
| + "errors" |
| + "fmt" |
| + "log" |
| + "net/http" |
| + "os" |
| + "strconv" |
| + "strings" |
| + "time" |
| +) |
| + |
| +// Domain separators: leaves, interior nodes, and the checkpoint chain must |
| +// never be confusable with each other or with any other protocol hash. |
| +const ( |
| + ckptLeafDomain = "lchat-ckpt-leaf\n" |
| + ckptNodeDomain = "lchat-ckpt-node\n" |
| + ckptChainDomain = "lchat-ckpt-chain\n" |
| +) |
| + |
| +// How often the sweep looks for new journal rows, and how many rows go into |
| +// one checkpoint at most (a busy backlog folds into several checkpoints). |
| +const ( |
| + checkpointInterval = 5 * time.Minute |
| + checkpointBatchSize = 10000 |
| +) |
| + |
| +// journalLeafHash is the canonical leaf encoding. Fields are newline-joined |
| +// in fixed order; thread_id goes last so the one non-fixed-width field can't |
| +// create an ambiguous boundary. Changing this encoding breaks every existing |
| +// chain — it is wire format, not an implementation detail. The fields after |
| +// `kind` are that kind's detail record: future kinds (roster/key events, the |
| +// key-transparency slice) define their own fields there without touching the |
| +// encoding of existing kinds. |
| +func journalLeafHash(r JournalRow) [32]byte { |
| + enc := ckptLeafDomain + |
| + strconv.FormatInt(r.Seq, 10) + "\n" + |
| + strconv.FormatInt(r.TS, 10) + "\n" + |
| + r.Kind + "\n" + |
| + strconv.FormatInt(r.MsgID, 10) + "\n" + |
| + r.BlobHash + "\n" + |
| + r.ThreadID |
| + return sha256.Sum256([]byte(enc)) |
| +} |
| + |
| +// merkleRoot builds an RFC 6962-style tree: split at the largest power of two |
| +// strictly less than n, recurse, hash the pair under the node domain. That |
| +// shape keeps inclusion proofs (a later slice) standard. Must never be called |
| +// with zero leaves — a checkpoint only exists because journal rows do. |
| +func merkleRoot(leaves [][32]byte) [32]byte { |
| + if len(leaves) == 1 { |
| + return leaves[0] |
| + } |
| + k := 1 |
| + for k*2 < len(leaves) { |
| + k *= 2 |
| + } |
| + l, r := merkleRoot(leaves[:k]), merkleRoot(leaves[k:]) |
| + return sha256.Sum256(append(append([]byte(ckptNodeDomain), l[:]...), r[:]...)) |
| +} |
| + |
| +// chainHash commits a checkpoint to its predecessor. prevChain is "" for the |
| +// genesis checkpoint. seq_to rides inside so two chains can't agree while |
| +// disagreeing about how much history a head covers. |
| +func chainHash(prevChain, root string, seqTo int64) string { |
| + enc := ckptChainDomain + prevChain + "\n" + root + "\n" + strconv.FormatInt(seqTo, 10) |
| + sum := sha256.Sum256([]byte(enc)) |
| + return hex.EncodeToString(sum[:]) |
| +} |
| + |
| +// ckptPayload is what gets signed. Like releaseManifest, the signature covers |
| +// the base64 of this JSON — verify first, decode after. |
| +type ckptPayload struct { |
| + SeqFrom int64 `json:"seq_from"` |
| + SeqTo int64 `json:"seq_to"` |
| + Root string `json:"root"` // hex Merkle root over leaves [seq_from..seq_to] |
| + PrevChain string `json:"prev_chain"` // hex chain of the predecessor ("" = genesis) |
| + Chain string `json:"chain"` // hex chainHash(prev_chain, root, seq_to) |
| + TS int64 `json:"ts"` // unix seconds, sign time |
| + // The signing key's public half, so every checkpoint names its own key: a |
| + // verifier walking the chain sees a key CHANGE as an explicit event to |
| + // investigate, instead of old checkpoints suddenly failing against the |
| + // instance's current key. |
| + Pub string `json:"pub"` // base64 ed25519 public key |
| +} |
| + |
| +// signedCheckpoint mirrors release.go's signedManifest wire shape. |
| +type signedCheckpoint struct { |
| + Payload string `json:"payload"` // base64(JSON ckptPayload) |
| + Sig string `json:"sig"` // base64(ed25519 sig over the payload STRING) |
| +} |
| + |
| +func signCheckpoint(priv ed25519.PrivateKey, p ckptPayload) (string, error) { |
| + body, err := json.Marshal(p) |
| + if err != nil { |
| + return "", err |
| + } |
| + b64 := base64.StdEncoding.EncodeToString(body) |
| + wrapper, err := json.Marshal(signedCheckpoint{ |
| + Payload: b64, |
| + Sig: base64.StdEncoding.EncodeToString(ed25519.Sign(priv, []byte(b64))), |
| + }) |
| + return string(wrapper), err |
| +} |
| + |
| +// verifyCheckpoint checks the signature over the encoded payload string, then |
| +// decodes. It verifies attribution, not history — recomputing roots against |
| +// the journal and walking the chain is the verifier's job (see tests; clients |
| +// and monitors get this in later slices). |
| +func verifyCheckpoint(pub ed25519.PublicKey, wrapper string) (ckptPayload, error) { |
| + var sc signedCheckpoint |
| + if err := json.Unmarshal([]byte(wrapper), &sc); err != nil { |
| + return ckptPayload{}, err |
| + } |
| + sig, err := base64.StdEncoding.DecodeString(sc.Sig) |
| + if err != nil { |
| + return ckptPayload{}, err |
| + } |
| + if !ed25519.Verify(pub, []byte(sc.Payload), sig) { |
| + return ckptPayload{}, errors.New("bad checkpoint signature") |
| + } |
| + body, err := base64.StdEncoding.DecodeString(sc.Payload) |
| + if err != nil { |
| + return ckptPayload{}, err |
| + } |
| + var p ckptPayload |
| + if err := json.Unmarshal(body, &p); err != nil { |
| + return ckptPayload{}, err |
| + } |
| + if p.SeqFrom <= 0 || p.SeqTo < p.SeqFrom || p.Root == "" || p.Chain == "" { |
| + return ckptPayload{}, errors.New("malformed checkpoint payload") |
| + } |
| + // The payload must name the key that signed it — a payload claiming a |
| + // different key than the one whose signature just verified is confused or |
| + // hostile either way. |
| + if p.Pub != base64.StdEncoding.EncodeToString(pub) { |
| + return ckptPayload{}, errors.New("checkpoint payload names a different key") |
| + } |
| + return p, nil |
| +} |
| + |
| +// checkpointKey lazily loads (or mints) the per-instance signing key: from |
| +// LCHAT_CHECKPOINT_KEY_FILE when set, else from the DB config (the default — |
| +// the seed must ride hibernation wakes and VACUUM-INTO migrations, which move |
| +// only the DB). Only success is cached: a transient read failure must not |
| +// disable checkpointing for the process lifetime. |
| +func (s *Server) checkpointKey() (ed25519.PrivateKey, error) { |
| + s.ckptMu.Lock() |
| + defer s.ckptMu.Unlock() |
| + if s.ckptKey != nil { |
| + return s.ckptKey, nil |
| + } |
| + var priv ed25519.PrivateKey |
| + var err error |
| + if s.ckptKeyPath != "" { |
| + priv, err = loadOrCreateCheckpointKey(s.ckptKeyPath) |
| + } else { |
| + priv, err = s.db.ensureCheckpointSeed() |
| + } |
| + if err != nil { |
| + return nil, err |
| + } |
| + s.ckptKey = priv |
| + return priv, nil |
| +} |
| + |
| +// loadOrCreateCheckpointKey reads a base64 Ed25519 seed from path, minting one |
| +// on first use (temp-file + rename, so a crash mid-write can't leave a |
| +// truncated seed). Same storage shape as the release signing key (release.go). |
| +// A corrupt existing file is a loud error, never silently regenerated — a new |
| +// key mid-chain is exactly the event this feature exists to make visible. |
| +func loadOrCreateCheckpointKey(path string) (ed25519.PrivateKey, error) { |
| + b, err := os.ReadFile(path) |
| + if errors.Is(err, os.ErrNotExist) { |
| + _, priv, gerr := ed25519.GenerateKey(rand.Reader) |
| + if gerr != nil { |
| + return nil, gerr |
| + } |
| + enc := base64.StdEncoding.EncodeToString(priv.Seed()) |
| + tmp := path + ".tmp" |
| + if werr := os.WriteFile(tmp, []byte(enc+"\n"), 0o600); werr != nil { |
| + return nil, fmt.Errorf("write checkpoint key: %w", werr) |
| + } |
| + if rerr := os.Rename(tmp, path); rerr != nil { |
| + return nil, fmt.Errorf("write checkpoint key: %w", rerr) |
| + } |
| + return priv, nil |
| + } |
| + if err != nil { |
| + return nil, err |
| + } |
| + return checkpointSeedKey(strings.TrimSpace(string(b)), path) |
| +} |
| + |
| +// checkpointSeedKey turns a stored base64 seed back into a key, shared by the |
| +// file and DB storage paths. |
| +func checkpointSeedKey(enc, what string) (ed25519.PrivateKey, error) { |
| + seed, err := base64.StdEncoding.DecodeString(enc) |
| + if err != nil { |
| + return nil, fmt.Errorf("checkpoint key %s: %w", what, err) |
| + } |
| + if len(seed) != ed25519.SeedSize { |
| + return nil, fmt.Errorf("checkpoint key %s: want %d-byte seed, got %d", what, ed25519.SeedSize, len(seed)) |
| + } |
| + return ed25519.NewKeyFromSeed(seed), nil |
| +} |
| + |
| +// runCheckpointer is the ticker; tests call sweepCheckpoint directly (the |
| +// runScheduler/sweepDueScheduled split, main.go). |
| +func (s *Server) runCheckpointer() { |
| + for range time.Tick(checkpointInterval) { |
| + s.sweepCheckpoint() |
| + } |
| +} |
| + |
| +// sweepCheckpoint folds new journal rows into checkpoints until the backlog is |
| +// drained, returning how many checkpoints it wrote. Zero new rows → zero work: |
| +// an idle instance's chain simply doesn't grow (no heartbeat checkpoints — |
| +// "nothing happened" is exactly what an unchanged head attests). The mutex |
| +// (plus the UNIQUE index on checkpoints.seq_from) keeps a second caller from |
| +// forking the chain — two checkpoints claiming the same predecessor would |
| +// read as tampering to a verifier. |
| +func (s *Server) sweepCheckpoint() int { |
| + s.ckptSweepMu.Lock() |
| + defer s.ckptSweepMu.Unlock() |
| + prev, havePrev, err := s.db.latestCheckpoint() |
| + if err != nil { |
| + log.Printf("checkpoint latest: %v", err) |
| + return 0 |
| + } |
| + after := int64(0) |
| + prevChain := "" |
| + if havePrev { |
| + after, prevChain = prev.SeqTo, prev.Chain |
| + } |
| + wrote := 0 |
| + for { |
| + rows, err := s.db.journalAfter(after, checkpointBatchSize) |
| + if err != nil { |
| + log.Printf("checkpoint journal: %v", err) |
| + return wrote |
| + } |
| + if len(rows) == 0 { |
| + return wrote |
| + } |
| + leaves := make([][32]byte, len(rows)) |
| + for i, r := range rows { |
| + leaves[i] = journalLeafHash(r) |
| + } |
| + root := merkleRoot(leaves) |
| + priv, err := s.checkpointKey() |
| + if err != nil { |
| + log.Printf("checkpoint key: %v", err) |
| + return wrote |
| + } |
| + p := ckptPayload{ |
| + SeqFrom: rows[0].Seq, |
| + SeqTo: rows[len(rows)-1].Seq, |
| + Root: hex.EncodeToString(root[:]), |
| + PrevChain: prevChain, |
| + TS: time.Now().Unix(), |
| + Pub: base64.StdEncoding.EncodeToString(priv.Public().(ed25519.PublicKey)), |
| + } |
| + p.Chain = chainHash(p.PrevChain, p.Root, p.SeqTo) |
| + signed, err := signCheckpoint(priv, p) |
| + if err != nil { |
| + log.Printf("checkpoint sign: %v", err) |
| + return wrote |
| + } |
| + if err := s.db.insertCheckpoint(p, signed); err != nil { |
| + log.Printf("checkpoint store: %v", err) |
| + return wrote |
| + } |
| + wrote++ |
| + // Carry the new head forward instead of re-reading it — the loop must |
| + // not depend on observing its own insert through the DB. |
| + after, prevChain = p.SeqTo, p.Chain |
| + if len(rows) < checkpointBatchSize { |
| + return wrote |
| + } |
| + } |
| +} |
| + |
| +// getCheckpoints serves the chain, member-token-gated: the chain leaks |
| +// activity volume and timing, so it is for members and monitors (who hold |
| +// member credentials), not the public. `after` is a checkpoint row id — a |
| +// verifier keeps its last-seen head and pages forward from it. Each |
| +// checkpoint's payload embeds its own public key, so the chain stays |
| +// verifiable even when the current key is unavailable — key trouble degrades |
| +// public_key to "", it never withholds the evidence. |
| +func (s *Server) getCheckpoints(w http.ResponseWriter, r *http.Request) { |
| + token := r.URL.Query().Get("token") |
| + if !s.tokenOK(token) { |
| + httpError(w, http.StatusForbidden, "invalid invite") |
| + return |
| + } |
| + after, _ := strconv.ParseInt(r.URL.Query().Get("after"), 10, 64) |
| + cps, err := s.db.checkpointsAfter(after, 100) |
| + if err != nil { |
| + httpError(w, http.StatusInternalServerError, "db error") |
| + return |
| + } |
| + pub := "" |
| + if priv, err := s.checkpointKey(); err != nil { |
| + log.Printf("checkpoint key: %v", err) |
| + } else { |
| + pub = base64.StdEncoding.EncodeToString(priv.Public().(ed25519.PublicKey)) |
| + } |
| + writeJSON(w, http.StatusOK, map[string]any{ |
| + "checkpoints": cps, |
| + "public_key": pub, |
| + }) |
| +} |
diff --git a/checkpoint_test.go b/checkpoint_test.go| new file mode 100644 |
| index 0000000..8e2cbd9 |
| --- /dev/null |
| +++ b/checkpoint_test.go |
| @@ -0,0 +1,471 @@ |
| +package main |
| + |
| +import ( |
| + "crypto/ed25519" |
| + "encoding/base64" |
| + "encoding/hex" |
| + "encoding/json" |
| + "net/http" |
| + "os" |
| + "path/filepath" |
| + "regexp" |
| + "strings" |
| + "testing" |
| + "time" |
| +) |
| + |
| +// journalRows drains the whole journal (test helper). |
| +func journalRows(t *testing.T, db *DB) []JournalRow { |
| + t.Helper() |
| + rows, err := db.journalAfter(0, 100000) |
| + if err != nil { |
| + t.Fatalf("journalAfter: %v", err) |
| + } |
| + return rows |
| +} |
| + |
| +// Every message write path must leave exactly one explaining journal row, in |
| +// the same transaction — a message the journal can't account for reads as |
| +// tampering to a verifier. |
| +func TestJournalFollowsMessageWrites(t *testing.T) { |
| + db := newTestDB(t) |
| + |
| + m, err := db.insertMessage("t1", "blob-one", "author-a") |
| + if err != nil { |
| + t.Fatalf("insertMessage: %v", err) |
| + } |
| + if _, ok, err := db.editMessage(m.ID, "blob-two", "author-a", time.Hour); err != nil || !ok { |
| + t.Fatalf("editMessage: ok=%v err=%v", ok, err) |
| + } |
| + if ok, err := db.deleteMessage(m.ID, "author-a"); err != nil || !ok { |
| + t.Fatalf("deleteMessage: ok=%v err=%v", ok, err) |
| + } |
| + |
| + rows := journalRows(t, db) |
| + // insert, edit, delete — plus the delete's tombstone, itself an ordinary |
| + // message row (empty blob) that must be journaled like any other insert. |
| + if len(rows) != 4 { |
| + t.Fatalf("want 4 journal rows, got %d: %+v", len(rows), rows) |
| + } |
| + wantKinds := []string{"msg", "edit", "delete"} |
| + wantHashes := []string{ciphertextHash("blob-one"), ciphertextHash("blob-two"), ""} |
| + for i, r := range rows[:3] { |
| + if r.Kind != wantKinds[i] || r.MsgID != m.ID || r.ThreadID != "t1" || r.BlobHash != wantHashes[i] { |
| + t.Errorf("row %d: got %+v", i, r) |
| + } |
| + if r.Seq != int64(i+1) { |
| + t.Errorf("row %d: seq %d, want %d", i, r.Seq, i+1) |
| + } |
| + } |
| + tomb := rows[3] |
| + if tomb.Kind != "msg" || tomb.MsgID == m.ID || tomb.ThreadID != "t1" || tomb.BlobHash != ciphertextHash("") { |
| + t.Errorf("tombstone journal row: got %+v", tomb) |
| + } |
| + // The journal must never hold ciphertext — hashes and empties only. |
| + for _, r := range rows { |
| + if r.BlobHash != "" { |
| + if _, err := hex.DecodeString(r.BlobHash); err != nil || len(r.BlobHash) != 64 { |
| + t.Errorf("blob_hash %q is not a sha256 hex digest", r.BlobHash) |
| + } |
| + } |
| + } |
| +} |
| + |
| +// A refused write (wrong author, expired window) must journal nothing. |
| +func TestJournalSkipsRefusedWrites(t *testing.T) { |
| + db := newTestDB(t) |
| + m, err := db.insertMessage("t1", "blob", "author-a") |
| + if err != nil { |
| + t.Fatalf("insertMessage: %v", err) |
| + } |
| + if _, ok, _ := db.editMessage(m.ID, "evil", "author-b", time.Hour); ok { |
| + t.Fatal("edit by non-author succeeded") |
| + } |
| + if ok, _ := db.deleteMessage(m.ID, "author-b"); ok { |
| + t.Fatal("delete by non-author succeeded") |
| + } |
| + if rows := journalRows(t, db); len(rows) != 1 { |
| + t.Fatalf("want only the insert row, got %d: %+v", len(rows), rows) |
| + } |
| +} |
| + |
| +// Removing a member hard-deletes their messages; each one needs its own |
| +// 'delete' journal row so the vanished rows are accounted for. |
| +func TestJournalCoversRemoveUserPurge(t *testing.T) { |
| + _, srv := newTestServer(t) |
| + u, err := srv.db.createUser("Mallory") |
| + if err != nil { |
| + t.Fatalf("createUser: %v", err) |
| + } |
| + if _, err := srv.db.insertMessage("t1", "b1", u); err != nil { |
| + t.Fatalf("insertMessage: %v", err) |
| + } |
| + if _, err := srv.db.insertMessage("t2", "b2", u); err != nil { |
| + t.Fatalf("insertMessage: %v", err) |
| + } |
| + if err := srv.db.removeUser(u); err != nil { |
| + t.Fatalf("removeUser: %v", err) |
| + } |
| + var kinds []string |
| + for _, r := range journalRows(t, srv.db) { |
| + kinds = append(kinds, r.Kind) |
| + } |
| + want := []string{"msg", "msg", "delete", "delete"} |
| + if len(kinds) != len(want) { |
| + t.Fatalf("journal kinds %v, want %v", kinds, want) |
| + } |
| + for i := range want { |
| + if kinds[i] != want[i] { |
| + t.Fatalf("journal kinds %v, want %v", kinds, want) |
| + } |
| + } |
| +} |
| + |
| +// Freeze the leaf/node/chain encodings: these are wire format — if this test |
| +// breaks, every deployed chain breaks with it. Fix the code, not the vector. |
| +func TestCheckpointHashVectors(t *testing.T) { |
| + leaf := journalLeafHash(JournalRow{Seq: 1, TS: 1700000000, Kind: "msg", MsgID: 42, ThreadID: "th", BlobHash: "ab"}) |
| + if got := hex.EncodeToString(leaf[:]); got != "e88f7b9ea2a1fdec4ce59c21467efcef2c1b7227b9618d5581c0923328143b15" { |
| + t.Errorf("leaf hash drifted: %s", got) |
| + } |
| + l2 := journalLeafHash(JournalRow{Seq: 2, TS: 1700000001, Kind: "edit", MsgID: 42, ThreadID: "th", BlobHash: "cd"}) |
| + root := merkleRoot([][32]byte{leaf, l2}) |
| + if got := hex.EncodeToString(root[:]); got != "7a94be3e3002c1afe379246c90db6fd04fdf438b0dc78da06ca6c3f130a5cec0" { |
| + t.Errorf("node hash drifted: %s", got) |
| + } |
| + if got := chainHash("", hex.EncodeToString(root[:]), 2); got != "b95fd584ded27920b2b8aa67add5b59269ac572f2924d1fb5a1387e650b5344e" { |
| + t.Errorf("chain hash drifted: %s", got) |
| + } |
| +} |
| + |
| +// A single-leaf tree is the leaf; unbalanced trees split at the largest power |
| +// of two below n (RFC 6962), so proofs stay standard. |
| +func TestMerkleRootShape(t *testing.T) { |
| + mk := func(b byte) [32]byte { var h [32]byte; h[0] = b; return h } |
| + one := merkleRoot([][32]byte{mk(1)}) |
| + if one != mk(1) { |
| + t.Error("single leaf must be its own root") |
| + } |
| + // 3 leaves: root = node(node(l1,l2), l3) — recompute by hand via merkleRoot |
| + // pieces to pin the split. |
| + l1, l2, l3 := mk(1), mk(2), mk(3) |
| + left := merkleRoot([][32]byte{l1, l2}) |
| + want := merkleRoot([][32]byte{left, l3}) // node(left, l3) — 2-leaf call IS the pair hash |
| + if got := merkleRoot([][32]byte{l1, l2, l3}); got != want { |
| + t.Error("3-leaf tree must split 2|1") |
| + } |
| +} |
| + |
| +// The sweep folds journal rows into a signed, chained checkpoint that an |
| +// independent verifier can check end-to-end: signature, root recomputation |
| +// from the journal, and chain linkage. |
| +func TestSweepCheckpointVerifies(t *testing.T) { |
| + _, srv := newTestServer(t) |
| + if _, err := srv.db.insertMessage("t1", "hello", "a"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := srv.db.insertMessage("t1", "world", "a"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if n := srv.sweepCheckpoint(); n != 1 { |
| + t.Fatalf("sweep wrote %d checkpoints, want 1", n) |
| + } |
| + // Idle sweep: nothing new, no heartbeat checkpoints. |
| + if n := srv.sweepCheckpoint(); n != 0 { |
| + t.Fatalf("idle sweep wrote %d checkpoints, want 0", n) |
| + } |
| + |
| + cp, ok, err := srv.db.latestCheckpoint() |
| + if err != nil || !ok { |
| + t.Fatalf("latestCheckpoint: ok=%v err=%v", ok, err) |
| + } |
| + priv, err := srv.checkpointKey() |
| + if err != nil { |
| + t.Fatalf("checkpointKey: %v", err) |
| + } |
| + pub := priv.Public().(ed25519.PublicKey) |
| + p, err := verifyCheckpoint(pub, cp.Signed) |
| + if err != nil { |
| + t.Fatalf("verifyCheckpoint: %v", err) |
| + } |
| + if p.SeqFrom != 1 || p.SeqTo != 2 || p.PrevChain != "" { |
| + t.Fatalf("payload %+v", p) |
| + } |
| + |
| + // Recompute the root from the journal — the verifier's real check. |
| + rows := journalRows(t, srv.db) |
| + leaves := make([][32]byte, len(rows)) |
| + for i, r := range rows { |
| + leaves[i] = journalLeafHash(r) |
| + } |
| + root := merkleRoot(leaves) |
| + if hex.EncodeToString(root[:]) != p.Root { |
| + t.Error("recomputed root does not match signed root") |
| + } |
| + if chainHash(p.PrevChain, p.Root, p.SeqTo) != p.Chain { |
| + t.Error("chain hash does not verify") |
| + } |
| + |
| + // A tampered signature must not verify. |
| + var sc signedCheckpoint |
| + json.Unmarshal([]byte(cp.Signed), &sc) |
| + sc.Payload = base64.StdEncoding.EncodeToString([]byte(`{"seq_from":1,"seq_to":2,"root":"00","prev_chain":"","chain":"00","ts":1}`)) |
| + forged, _ := json.Marshal(sc) |
| + if _, err := verifyCheckpoint(pub, string(forged)); err == nil { |
| + t.Error("forged payload verified") |
| + } |
| +} |
| + |
| +// Later checkpoints chain to earlier ones, so serving an older head than one |
| +// already witnessed is provable rollback, not a client-side anomaly. |
| +func TestCheckpointsChain(t *testing.T) { |
| + _, srv := newTestServer(t) |
| + if _, err := srv.db.insertMessage("t1", "one", "a"); err != nil { |
| + t.Fatal(err) |
| + } |
| + srv.sweepCheckpoint() |
| + if _, err := srv.db.insertMessage("t1", "two", "a"); err != nil { |
| + t.Fatal(err) |
| + } |
| + srv.sweepCheckpoint() |
| + |
| + cps, err := srv.db.checkpointsAfter(0, 10) |
| + if err != nil || len(cps) != 2 { |
| + t.Fatalf("checkpointsAfter: n=%d err=%v", len(cps), err) |
| + } |
| + priv, _ := srv.checkpointKey() |
| + pub := priv.Public().(ed25519.PublicKey) |
| + p1, err := verifyCheckpoint(pub, cps[0].Signed) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + p2, err := verifyCheckpoint(pub, cps[1].Signed) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if p2.PrevChain != p1.Chain { |
| + t.Error("second checkpoint does not chain to the first") |
| + } |
| + if p2.SeqFrom != p1.SeqTo+1 { |
| + t.Errorf("ranges must abut: %d..%d then %d..%d", p1.SeqFrom, p1.SeqTo, p2.SeqFrom, p2.SeqTo) |
| + } |
| +} |
| + |
| +// A backlog bigger than one batch folds into several abutting checkpoints in |
| +// one sweep. |
| +func TestSweepDrainsBacklogInBatches(t *testing.T) { |
| + _, srv := newTestServer(t) |
| + // 3 rows with a batch size of 10000 is one batch; simulate multi-batch by |
| + // sweeping between writes instead — cheap, and pins the abutting contract. |
| + for i := 0; i < 3; i++ { |
| + if _, err := srv.db.insertMessage("t", "m", "a"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if n := srv.sweepCheckpoint(); n != 1 { |
| + t.Fatalf("sweep %d wrote %d", i, n) |
| + } |
| + } |
| + cps, _ := srv.db.checkpointsAfter(0, 10) |
| + if len(cps) != 3 { |
| + t.Fatalf("want 3 checkpoints, got %d", len(cps)) |
| + } |
| + for i := 1; i < len(cps); i++ { |
| + if cps[i].SeqFrom != cps[i-1].SeqTo+1 { |
| + t.Error("checkpoint ranges must abut") |
| + } |
| + } |
| +} |
| + |
| +// The endpoint is member-gated and returns the chain plus the instance's |
| +// checkpoint public key. |
| +func TestCheckpointsEndpoint(t *testing.T) { |
| + ts, srv := newTestServer(t) |
| + u, err := srv.db.createUser("Alice") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := srv.db.createKeyInvite("inv-Alice", u, "wrap", testClaimHash, 0); err != nil { |
| + t.Fatal(err) |
| + } |
| + tok, ok, err := srv.db.claimKeyInvite("inv-Alice", "Alice", "", testClaimSecret) |
| + if err != nil || !ok { |
| + t.Fatalf("claim: ok=%v err=%v", ok, err) |
| + } |
| + if _, err := srv.db.insertMessage("t1", "hello", "a"); err != nil { |
| + t.Fatal(err) |
| + } |
| + srv.sweepCheckpoint() |
| + |
| + resp, err := http.Get(ts.URL + "/api/checkpoints?token=nope") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + resp.Body.Close() |
| + if resp.StatusCode != http.StatusForbidden { |
| + t.Fatalf("ungated: %d", resp.StatusCode) |
| + } |
| + |
| + resp, err = http.Get(ts.URL + "/api/checkpoints?token=" + tok) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + defer resp.Body.Close() |
| + var body struct { |
| + Checkpoints []Checkpoint `json:"checkpoints"` |
| + PublicKey string `json:"public_key"` |
| + } |
| + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(body.Checkpoints) != 1 { |
| + t.Fatalf("want 1 checkpoint, got %d", len(body.Checkpoints)) |
| + } |
| + pk, err := base64.StdEncoding.DecodeString(body.PublicKey) |
| + if err != nil || len(pk) != ed25519.PublicKeySize { |
| + t.Fatalf("bad public key %q", body.PublicKey) |
| + } |
| + if _, err := verifyCheckpoint(ed25519.PublicKey(pk), body.Checkpoints[0].Signed); err != nil { |
| + t.Fatalf("served checkpoint does not verify with served key: %v", err) |
| + } |
| +} |
| + |
| +// The signing key persists across restarts (same file → same key), is created |
| +// 0600, and never lands inside the DB file the escrow replicates. |
| +func TestCheckpointKeyPersistence(t *testing.T) { |
| + path := t.TempDir() + "/k.ckpt-key" |
| + k1, err := loadOrCreateCheckpointKey(path) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + k2, err := loadOrCreateCheckpointKey(path) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if !k1.Equal(k2) { |
| + t.Error("key did not persist across loads") |
| + } |
| + fi, err := os.Stat(path) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if fi.Mode().Perm() != 0o600 { |
| + t.Errorf("key file mode %v, want 0600", fi.Mode().Perm()) |
| + } |
| +} |
| + |
| +// The DB-stored seed persists across "restarts" (same DB → same key) — it must |
| +// ride every path that moves instance state (hibernation, migration), which |
| +// carry only the DB. |
| +func TestCheckpointSeedInDB(t *testing.T) { |
| + db := newTestDB(t) |
| + k1, err := db.ensureCheckpointSeed() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + k2, err := db.ensureCheckpointSeed() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if !k1.Equal(k2) { |
| + t.Error("seed did not persist in config") |
| + } |
| +} |
| + |
| +// Messages that predate the journal get baselined at migration, so editing or |
| +// unsending an old message never produces a journal row for a message the |
| +// journal can't account for. |
| +func TestBackfillEventJournal(t *testing.T) { |
| + dir := t.TempDir() |
| + db, err := openDB(dir + "/test.db") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + m, err := db.insertMessage("t1", "old-blob", "a") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + // Simulate a pre-journal database: erase the journal and the marker, as if |
| + // the message had been written by a build before this feature. |
| + if _, err := db.sql.Exec("DELETE FROM event_journal"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := db.sql.Exec("DELETE FROM config WHERE key = 'journal_backfilled'"); err != nil { |
| + t.Fatal(err) |
| + } |
| + // Re-running migrate (what a deploy does) must backfill exactly once. |
| + if err := db.migrate(); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := db.migrate(); err != nil { |
| + t.Fatal(err) |
| + } |
| + rows := journalRows(t, db) |
| + if len(rows) != 1 { |
| + t.Fatalf("want 1 backfilled row, got %d: %+v", len(rows), rows) |
| + } |
| + r := rows[0] |
| + if r.Kind != "msg" || r.MsgID != m.ID || r.ThreadID != "t1" || r.BlobHash != ciphertextHash("old-blob") || r.TS != m.CreatedAt { |
| + t.Errorf("backfilled row %+v", r) |
| + } |
| +} |
| + |
| +// The checkpoints table's bare columns are denormalized copies of the signed |
| +// payload; nothing may let them drift. |
| +func TestCheckpointColumnsMatchPayload(t *testing.T) { |
| + _, srv := newTestServer(t) |
| + if _, err := srv.db.insertMessage("t1", "x", "a"); err != nil { |
| + t.Fatal(err) |
| + } |
| + srv.sweepCheckpoint() |
| + cp, ok, err := srv.db.latestCheckpoint() |
| + if err != nil || !ok { |
| + t.Fatalf("latestCheckpoint: ok=%v err=%v", ok, err) |
| + } |
| + priv, _ := srv.checkpointKey() |
| + p, err := verifyCheckpoint(priv.Public().(ed25519.PublicKey), cp.Signed) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if cp.SeqFrom != p.SeqFrom || cp.SeqTo != p.SeqTo || cp.Root != p.Root || cp.Chain != p.Chain || cp.TS != p.TS { |
| + t.Errorf("columns %+v drifted from payload %+v", cp, p) |
| + } |
| +} |
| + |
| +// Structural guard: every write to the messages table must journal in the same |
| +// transaction (see event_journal in db.go). This test scans the source so a |
| +// future write path can't quietly bypass the journal — an unexplained message |
| +// row reads as tampering to a verifier. If you add a legitimate, journaled |
| +// write site, update the expected counts here. |
| +func TestMessageWritesAreJournaled(t *testing.T) { |
| + want := map[string]int{ // file → number of messages-table write statements |
| + "db.go": 7, // insertMessage, editMessage, deleteMessage (delete + tombstone insert), removeUser ×2, deliverScheduledMessage |
| + } |
| + re := regexp.MustCompile(`INSERT INTO messages|UPDATE messages|DELETE FROM messages`) |
| + files, err := filepath.Glob("*.go") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + got := map[string]int{} |
| + for _, f := range files { |
| + if strings.HasSuffix(f, "_test.go") { |
| + continue |
| + } |
| + b, err := os.ReadFile(f) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if n := len(re.FindAll(b, -1)); n > 0 { |
| + got[f] = n |
| + } |
| + } |
| + for f, n := range got { |
| + if want[f] != n { |
| + t.Errorf("%s has %d messages-table writes (expected %d): a new write site MUST append to event_journal in the same tx (docs/tamper-evidence.md), then update this test", f, n, want[f]) |
| + } |
| + } |
| + for f, n := range want { |
| + if got[f] != n { |
| + t.Errorf("expected %d messages-table writes in %s, found %d — update this test if sites moved", n, f, got[f]) |
| + } |
| + } |
| +} |
diff --git a/db.go b/db.go| index ecfc79d..24682c3 100644 |
| --- a/db.go |
| +++ b/db.go |
| @@ -1,6 +1,7 @@ |
| package main |
| import ( |
| + "crypto/ed25519" |
| "crypto/rand" |
| "crypto/sha256" |
| "crypto/subtle" |
| @@ -515,7 +516,39 @@ CREATE TABLE IF NOT EXISTS member_bots ( |
| owner_id TEXT, |
| created_at INTEGER NOT NULL, |
| updated_at INTEGER NOT NULL |
| -);`) |
| +); |
| +-- Tamper evidence (docs/tamper-evidence.md, #573): the append-only event |
| +-- journal. The messages table mutates on purpose (edits UPDATE, unsend hard- |
| +-- DELETEs), so checkpoints commit to this journal instead: every message |
| +-- insert/edit/delete appends one row IN THE SAME TRANSACTION. blob_hash is |
| +-- sha256 hex of the ciphertext — never the ciphertext itself, so unsend still |
| +-- destroys the blob; the journal retains only that an event happened. Rows are |
| +-- never updated or deleted. Adding a plaintext column here would break the one |
| +-- rule — don't. |
| +CREATE TABLE IF NOT EXISTS event_journal ( |
| + seq INTEGER PRIMARY KEY AUTOINCREMENT, |
| + ts INTEGER NOT NULL, |
| + kind TEXT NOT NULL, -- 'msg' | 'edit' | 'delete' |
| + msg_id INTEGER NOT NULL, |
| + thread_id TEXT NOT NULL, |
| + blob_hash TEXT NOT NULL -- '' for delete |
| +); |
| +-- Signed checkpoint chain over the journal (checkpoint.go). "signed" is the |
| +-- full signed wrapper clients verify; the bare columns exist so the server |
| +-- can chain and page without re-parsing it. |
| +CREATE TABLE IF NOT EXISTS checkpoints ( |
| + id INTEGER PRIMARY KEY AUTOINCREMENT, |
| + seq_from INTEGER NOT NULL, |
| + seq_to INTEGER NOT NULL, |
| + root TEXT NOT NULL, |
| + chain TEXT NOT NULL, |
| + ts INTEGER NOT NULL, |
| + signed TEXT NOT NULL |
| +); |
| +-- Two checkpoints claiming the same journal range would fork the chain — a |
| +-- second writer (e.g. two processes briefly overlapping on one DB during an |
| +-- activator wake race) must fail loudly, not fork silently. |
| +CREATE UNIQUE INDEX IF NOT EXISTS idx_checkpoints_seq_from ON checkpoints(seq_from);`) |
| if err != nil { |
| return err |
| } |
| @@ -711,6 +744,13 @@ CREATE TABLE IF NOT EXISTS member_bots ( |
| if err := db.backfillMemberAppVersions(); err != nil { |
| return err |
| } |
| + // Baseline pre-journal messages into the event journal (tamper evidence) — |
| + // without this, editing or unsending a message that predates the journal |
| + // would append an 'edit'/'delete' row for a message the journal never saw, |
| + // which reads as tampering to a verifier. |
| + if err := db.backfillEventJournal(); err != nil { |
| + return err |
| + } |
| // Established instances never ran the setup conversation — mark it complete |
| // ONCE at upgrade time, so promoting someone to admin later doesn't drop them |
| // into first-run onboarding on a live space. Guarded by its own marker: a |
| @@ -1506,6 +1546,11 @@ func (db *DB) removeUser(id string) error { |
| } |
| teardown := func(tid string) error { |
| + // Journal the messages this teardown removes (tamper evidence) before |
| + // they're gone — same transaction, so the trail can't drift. |
| + if err := journalMessageDeletesTx(tx, "thread_id = ?", tid); err != nil { |
| + return err |
| + } |
| for _, q := range []string{ |
| "DELETE FROM key_invites WHERE thread_id = ?", |
| "DELETE FROM thread_keys WHERE thread_id = ?", |
| @@ -1550,7 +1595,11 @@ func (db *DB) removeUser(id string) error { |
| } |
| } |
| - // The user's own rows. |
| + // The user's own rows. Their messages get journaled first (tamper |
| + // evidence) — removal deletes them below, and the trail must say so. |
| + if err := journalMessageDeletesTx(tx, "author_hash = ?", id); err != nil { |
| + return err |
| + } |
| for _, s := range []struct { |
| q string |
| arg []any |
| @@ -3034,7 +3083,15 @@ type Message struct { |
| func (db *DB) insertMessage(threadID, blob, authorHash string) (Message, error) { |
| now := time.Now().Unix() |
| - res, err := db.sql.Exec( |
| + // A transaction so the journal row (tamper evidence, checkpoint.go) can |
| + // never be missing for a message that exists — a gap would read as |
| + // tampering to a verifier. |
| + tx, err := db.sql.Begin() |
| + if err != nil { |
| + return Message{}, err |
| + } |
| + defer tx.Rollback() |
| + res, err := tx.Exec( |
| "INSERT INTO messages(created_at, blob, author_hash, thread_id) VALUES(?, ?, ?, ?)", now, blob, authorHash, threadID, |
| ) |
| if err != nil { |
| @@ -3044,6 +3101,12 @@ func (db *DB) insertMessage(threadID, blob, authorHash string) (Message, error) |
| if err != nil { |
| return Message{}, err |
| } |
| + if err := appendJournalTx(tx, now, "msg", id, threadID, ciphertextHash(blob)); err != nil { |
| + return Message{}, err |
| + } |
| + if err := tx.Commit(); err != nil { |
| + return Message{}, err |
| + } |
| // New activity reopens the thread for anyone who'd closed it (best-effort). |
| _ = db.clearThreadHidden(threadID) |
| return Message{ID: id, CreatedAt: now, Blob: blob}, nil |
| @@ -3057,7 +3120,12 @@ func (db *DB) insertMessage(threadID, blob, authorHash string) (Message, error) |
| func (db *DB) editMessage(id int64, blob, authorHash string, window time.Duration) (Message, bool, error) { |
| now := time.Now() |
| cutoff := now.Add(-window).Unix() |
| - res, err := db.sql.Exec( |
| + tx, err := db.sql.Begin() |
| + if err != nil { |
| + return Message{}, false, err |
| + } |
| + defer tx.Rollback() |
| + res, err := tx.Exec( |
| "UPDATE messages SET blob = ?, edited_at = ? WHERE id = ? AND author_hash = ? AND created_at >= ?", |
| blob, now.Unix(), id, authorHash, cutoff, |
| ) |
| @@ -3069,7 +3137,19 @@ func (db *DB) editMessage(id int64, blob, authorHash string, window time.Duratio |
| return Message{}, false, err |
| } |
| if n == 0 { |
| - return Message{}, false, nil |
| + return Message{}, false, tx.Commit() |
| + } |
| + // Journal the edit (tamper evidence): the new ciphertext's hash — the |
| + // pre-edit hash is already the message's previous journal row. |
| + var threadID string |
| + if err := tx.QueryRow("SELECT COALESCE(thread_id, '') FROM messages WHERE id = ?", id).Scan(&threadID); err != nil { |
| + return Message{}, false, err |
| + } |
| + if err := appendJournalTx(tx, now.Unix(), "edit", id, threadID, ciphertextHash(blob)); err != nil { |
| + return Message{}, false, err |
| + } |
| + if err := tx.Commit(); err != nil { |
| + return Message{}, false, err |
| } |
| return Message{ID: id, Blob: blob, EditedAt: now.Unix()}, true, nil |
| } |
| @@ -3101,6 +3181,11 @@ func (db *DB) deleteMessage(id int64, authorHash string) (bool, error) { |
| if err != nil { |
| return false, err |
| } |
| + // Journal before the row is gone (tamper evidence) — stays atomic with the |
| + // DELETE below inside this transaction. |
| + if err := journalMessageDeletesTx(tx, "id = ? AND author_hash = ?", id, authorHash); err != nil { |
| + return false, err |
| + } |
| // The DELETE re-checks author_hash and gates on RowsAffected, preserving the |
| // old single-statement invariant: true means THIS call removed the row. A |
| // raced duplicate (double-tap, retried frame) loses here and mints no |
| @@ -3116,10 +3201,22 @@ func (db *DB) deleteMessage(id int64, authorHash string) (bool, error) { |
| // client ever rendered it — skip the tombstone rather than insert one no |
| // catch-up query could deliver. |
| if threadID.Valid { |
| - if _, err := tx.Exec( |
| + now := time.Now().Unix() |
| + res, err := tx.Exec( |
| "INSERT INTO messages(created_at, blob, thread_id, tombstone_of) VALUES(?, '', ?, ?)", |
| - time.Now().Unix(), threadID.String, id, |
| - ); err != nil { |
| + now, threadID.String, id, |
| + ) |
| + if err != nil { |
| + return false, err |
| + } |
| + // The tombstone is a real message row (clients replay it from catch-up), |
| + // so it gets a journal row like any other insert — an unexplained row |
| + // would read as tampering to a verifier. |
| + tombID, err := res.LastInsertId() |
| + if err != nil { |
| + return false, err |
| + } |
| + if err := appendJournalTx(tx, now, "msg", tombID, threadID.String, ciphertextHash("")); err != nil { |
| return false, err |
| } |
| } |
| @@ -3438,6 +3535,11 @@ func (db *DB) deliverScheduledMessage(sm ScheduledMessage) (Message, bool, error |
| if err != nil { |
| return Message{}, false, err |
| } |
| + // A delivered schedule is an ordinary message insert to the journal |
| + // (tamper evidence) — verifiers don't care how a message arrived. |
| + if err := appendJournalTx(tx, now, "msg", id, sm.ThreadID, ciphertextHash(sm.Blob)); err != nil { |
| + return Message{}, false, err |
| + } |
| if err := tx.Commit(); err != nil { |
| return Message{}, false, err |
| } |
| @@ -3947,3 +4049,205 @@ func (db *DB) deleteMemberBot(id string) (bool, error) { |
| return n > 0, nil |
| } |
| +// ---- event journal + checkpoints (tamper evidence, checkpoint.go) -------- |
| +// |
| +// The journal is append-only: rows are written inside the same transaction as |
| +// the message write they describe, and nothing ever updates or deletes them. |
| +// It stores ciphertext HASHES, never ciphertext — unsend still destroys the |
| +// blob; what survives is evidence that an event happened, which is the whole |
| +// point (docs/tamper-evidence.md). |
| + |
| +// JournalRow is one appended event. Field order and encoding are pinned by |
| +// journalLeafHash (checkpoint.go) — they are wire format. |
| +type JournalRow struct { |
| + Seq int64 `json:"seq"` |
| + TS int64 `json:"ts"` |
| + Kind string `json:"kind"` |
| + MsgID int64 `json:"msg_id"` |
| + ThreadID string `json:"thread_id"` |
| + BlobHash string `json:"blob_hash"` |
| +} |
| + |
| +// ciphertextHash is the journal's commitment to a message blob: sha256 hex of |
| +// the stored ciphertext text. Hashing the base64url string as stored (rather |
| +// than decoded bytes) keeps verification a straight string comparison against |
| +// what /api/messages serves. It happens to match hashToken today but is |
| +// deliberately NOT shared with it: this is journal wire format — recorded |
| +// hashes must stay verifiable forever — and must never follow a change to how |
| +// tokens are hashed. |
| +func ciphertextHash(blob string) string { |
| + sum := sha256.Sum256([]byte(blob)) |
| + return hex.EncodeToString(sum[:]) |
| +} |
| + |
| +func appendJournalTx(tx *sql.Tx, ts int64, kind string, msgID int64, threadID, blobHash string) error { |
| + _, err := tx.Exec( |
| + "INSERT INTO event_journal(ts, kind, msg_id, thread_id, blob_hash) VALUES(?, ?, ?, ?, ?)", |
| + ts, kind, msgID, threadID, blobHash, |
| + ) |
| + return err |
| +} |
| + |
| +// journalMessageDeletesTx appends a 'delete' row for every message matching |
| +// where — called immediately before the DELETE with the same predicate, in the |
| +// same transaction, so the journal and the table can't disagree about what was |
| +// removed. Bulk removals (thread teardown, member removal) journal one row per |
| +// message on purpose: every vanished message row has exactly one explaining |
| +// journal row, so a verifier never has to guess. One set-based statement, so |
| +// purging a prolific member doesn't hold the single connection through N |
| +// round-trips. |
| +func journalMessageDeletesTx(tx *sql.Tx, where string, args ...any) error { |
| + _, err := tx.Exec( |
| + "INSERT INTO event_journal(ts, kind, msg_id, thread_id, blob_hash) "+ |
| + "SELECT ?, 'delete', id, COALESCE(thread_id, ''), '' FROM messages WHERE "+where, |
| + append([]any{time.Now().Unix()}, args...)..., |
| + ) |
| + return err |
| +} |
| + |
| +// ensureCheckpointSeed loads (or mints) the checkpoint signing seed from the |
| +// config table. In the DB on purpose: the seed must survive everything that |
| +// moves instance state — hibernation wake, VACUUM-INTO migration, escrow |
| +// restore — all of which carry only the DB. A key that silently changed on |
| +// routine ops would make every legitimate restore read as forgery. The trade |
| +// (a DB copy can sign as the instance) is accepted: the signature is |
| +// attribution, not the security boundary (docs/tamper-evidence.md). |
| +func (db *DB) ensureCheckpointSeed() (ed25519.PrivateKey, error) { |
| + enc, ok, err := db.getConfig("ckpt_seed") |
| + if err != nil { |
| + return nil, err |
| + } |
| + if ok && enc != "" { |
| + return checkpointSeedKey(enc, "config ckpt_seed") |
| + } |
| + _, priv, err := ed25519.GenerateKey(rand.Reader) |
| + if err != nil { |
| + return nil, err |
| + } |
| + if err := db.setConfig("ckpt_seed", base64.StdEncoding.EncodeToString(priv.Seed())); err != nil { |
| + return nil, err |
| + } |
| + return priv, nil |
| +} |
| + |
| +// backfillEventJournal baselines messages that predate the journal: one 'msg' |
| +// row per existing message, once (config-marked), in a single transaction. |
| +// A backfilled row commits to the ciphertext as it stood at upgrade — evidence |
| +// can't be retroactive (a pre-upgrade rewrite is invisible), but from this |
| +// baseline forward every change to old messages is accounted for. |
| +func (db *DB) backfillEventJournal() error { |
| + if v, _, _ := db.getConfig("journal_backfilled"); v == "1" { |
| + return nil |
| + } |
| + tx, err := db.sql.Begin() |
| + if err != nil { |
| + return err |
| + } |
| + defer tx.Rollback() |
| + // Buffer then insert: blob hashes are computed in Go (no sha256 in SQL), |
| + // and interleaving inserts with an open cursor on one connection is asking |
| + // for trouble. Fleet instances hold a few thousand messages at most. |
| + rows, err := tx.Query("SELECT id, created_at, COALESCE(thread_id, ''), blob FROM messages ORDER BY id") |
| + if err != nil { |
| + return err |
| + } |
| + var rs []JournalRow |
| + for rows.Next() { |
| + var r JournalRow |
| + var blob string |
| + if err := rows.Scan(&r.MsgID, &r.TS, &r.ThreadID, &blob); err != nil { |
| + rows.Close() |
| + return err |
| + } |
| + r.BlobHash = ciphertextHash(blob) |
| + rs = append(rs, r) |
| + } |
| + rows.Close() |
| + if err := rows.Err(); err != nil { |
| + return err |
| + } |
| + for _, r := range rs { |
| + if err := appendJournalTx(tx, r.TS, "msg", r.MsgID, r.ThreadID, r.BlobHash); err != nil { |
| + return err |
| + } |
| + } |
| + if err := setConfigTx(tx, "journal_backfilled", "1"); err != nil { |
| + return err |
| + } |
| + return tx.Commit() |
| +} |
| + |
| +// journalAfter returns up to limit rows with seq > after, in order — the |
| +// checkpoint sweep's read, and a verifier's replay feed. |
| +func (db *DB) journalAfter(after int64, limit int) ([]JournalRow, error) { |
| + rows, err := db.sql.Query( |
| + "SELECT seq, ts, kind, msg_id, thread_id, blob_hash FROM event_journal WHERE seq > ? ORDER BY seq LIMIT ?", |
| + after, limit, |
| + ) |
| + if err != nil { |
| + return nil, err |
| + } |
| + defer rows.Close() |
| + var out []JournalRow |
| + for rows.Next() { |
| + var r JournalRow |
| + if err := rows.Scan(&r.Seq, &r.TS, &r.Kind, &r.MsgID, &r.ThreadID, &r.BlobHash); err != nil { |
| + return nil, err |
| + } |
| + out = append(out, r) |
| + } |
| + return out, rows.Err() |
| +} |
| + |
| +// Checkpoint is one signed link of the chain as served to verifiers. |
| +type Checkpoint struct { |
| + ID int64 `json:"id"` |
| + SeqFrom int64 `json:"seq_from"` |
| + SeqTo int64 `json:"seq_to"` |
| + Root string `json:"root"` |
| + Chain string `json:"chain"` |
| + TS int64 `json:"ts"` |
| + Signed string `json:"signed"` |
| +} |
| + |
| +func (db *DB) insertCheckpoint(p ckptPayload, signed string) error { |
| + _, err := db.sql.Exec( |
| + "INSERT INTO checkpoints(seq_from, seq_to, root, chain, ts, signed) VALUES(?, ?, ?, ?, ?, ?)", |
| + p.SeqFrom, p.SeqTo, p.Root, p.Chain, p.TS, signed, |
| + ) |
| + return err |
| +} |
| + |
| +func (db *DB) latestCheckpoint() (Checkpoint, bool, error) { |
| + var c Checkpoint |
| + err := db.sql.QueryRow( |
| + "SELECT id, seq_from, seq_to, root, chain, ts, signed FROM checkpoints ORDER BY id DESC LIMIT 1", |
| + ).Scan(&c.ID, &c.SeqFrom, &c.SeqTo, &c.Root, &c.Chain, &c.TS, &c.Signed) |
| + if err == sql.ErrNoRows { |
| + return Checkpoint{}, false, nil |
| + } |
| + if err != nil { |
| + return Checkpoint{}, false, err |
| + } |
| + return c, true, nil |
| +} |
| + |
| +func (db *DB) checkpointsAfter(after int64, limit int) ([]Checkpoint, error) { |
| + rows, err := db.sql.Query( |
| + "SELECT id, seq_from, seq_to, root, chain, ts, signed FROM checkpoints WHERE id > ? ORDER BY id LIMIT ?", |
| + after, limit, |
| + ) |
| + if err != nil { |
| + return nil, err |
| + } |
| + defer rows.Close() |
| + out := []Checkpoint{} |
| + for rows.Next() { |
| + var c Checkpoint |
| + if err := rows.Scan(&c.ID, &c.SeqFrom, &c.SeqTo, &c.Root, &c.Chain, &c.TS, &c.Signed); err != nil { |
| + return nil, err |
| + } |
| + out = append(out, c) |
| + } |
| + return out, rows.Err() |
| +} |
diff --git a/main.go b/main.go| index 38fa4d8..868d44b 100644 |
| --- a/main.go |
| +++ b/main.go |
| @@ -2,6 +2,7 @@ package main |
| import ( |
| "context" |
| + "crypto/ed25519" |
| "crypto/rand" |
| "crypto/sha256" |
| "database/sql" |
| @@ -106,6 +107,15 @@ type Server struct { |
| // pay is the payment provider for paid access levels (subscriptions.go). |
| // noPay unless Stripe Connect is configured — the feature is dormant then. |
| pay PayProvider |
| + // Tamper evidence (checkpoint.go): the per-instance checkpoint signing key. |
| + // Empty ckptKeyPath (the default) stores the seed in the DB config so it |
| + // rides restores; LCHAT_CHECKPOINT_KEY_FILE overrides to a file. ckptMu |
| + // guards lazy loading (success-only caching); ckptSweepMu serializes the |
| + // checkpoint sweep so the chain can't fork. |
| + ckptKeyPath string |
| + ckptMu sync.Mutex |
| + ckptKey ed25519.PrivateKey |
| + ckptSweepMu sync.Mutex |
| } |
| func main() { |
| @@ -245,6 +255,11 @@ func main() { |
| blobQuota: envBytes("LCHAT_BLOB_QUOTA_BYTES", defaultBlobQuotaBytes), |
| appsDir: strings.TrimSpace(os.Getenv("LCHAT_APPS_DIR"))} |
| + // Tamper evidence (checkpoint.go): by default the signing seed lives in |
| + // the DB config so it survives hibernation wakes and migrations; set |
| + // LCHAT_CHECKPOINT_KEY_FILE to keep it in a file outside the DB instead. |
| + s.ckptKeyPath = strings.TrimSpace(os.Getenv("LCHAT_CHECKPOINT_KEY_FILE")) |
| + |
| // Web Push: self-generated VAPID keypair (no third-party account). |
| if pub, priv, err := db.ensureVAPID(); err != nil { |
| log.Fatalf("vapid: %v", err) |
| @@ -265,6 +280,9 @@ func main() { |
| go s.runUpdateChecker() |
| // Presence idle sweep (presence.go). Tests call sweepPresence directly. |
| go s.runPresenceSweeper() |
| + // Tamper evidence: fold new journal rows into signed checkpoints |
| + // (checkpoint.go). Tests call sweepCheckpoint directly. |
| + go s.runCheckpointer() |
| // Push release notes new in THIS binary to the admins (announce.go). Tests |
| // call announceChangelog directly. |
| go s.announceChangelog() |
| @@ -374,6 +392,9 @@ func (s *Server) handler() http.Handler { |
| // Chat API (transport auth via invite token; never sees plaintext) |
| mux.HandleFunc("GET /api/messages", s.getMessages) |
| + // Tamper evidence: the signed checkpoint chain over the event journal |
| + // (checkpoint.go; docs/tamper-evidence.md). Member-gated like messages. |
| + mux.HandleFunc("GET /api/checkpoints", s.getCheckpoints) |
| mux.HandleFunc("POST /api/messages", s.postMessage) |
| mux.HandleFunc("GET /ws", s.handleWS) |
diff --git a/tools/verify-checkpoints.mjs b/tools/verify-checkpoints.mjs| new file mode 100644 |
| index 0000000..56f1d4f |
| --- /dev/null |
| +++ b/tools/verify-checkpoints.mjs |
| @@ -0,0 +1,120 @@ |
| +// 🤖 verify-checkpoints — E2E verifier for the tamper-evidence checkpoint |
| +// chain (docs/tamper-evidence.md) against a live instance. Acts as an |
| +// independent client: mints and claims a member, posts three messages, waits |
| +// for the sweep, then verifies the served chain — Ed25519 signatures, chain |
| +// hashes, linkage from genesis, column/payload consistency, and (when the new |
| +// batch lands in its own checkpoint) a full Merkle-root recomputation from |
| +// the expected journal rows. Needs the instance admin token; posts throwaway |
| +// content, so point it at a canary, never a real member space. |
| +// |
| +// node tools/verify-checkpoints.mjs https://test-<slug>.11ch.at <admin-token> |
| +import { createHash, createPublicKey, verify as edVerify } from "node:crypto"; |
| +const { randomSecret, deriveInvite } = await import(new URL("../web/static/crypto.js", import.meta.url).href); |
| + |
| +const [base, admin] = process.argv.slice(2); |
| +if (!base || !admin) { console.error("usage: node tools/verify-checkpoints.mjs <base-url> <admin-token>"); process.exit(2); } |
| + |
| +const sha256hex = (s) => createHash("sha256").update(s, "utf8").digest("hex"); |
| +const sha256buf = (b) => createHash("sha256").update(b).digest(); |
| + |
| +async function api(path, opts = {}) { |
| + const r = await fetch(base + path, opts); |
| + const body = await r.json().catch(() => ({})); |
| + if (!r.ok) throw new Error(`${path}: HTTP ${r.status} ${JSON.stringify(body)}`); |
| + return body; |
| +} |
| + |
| +// 1. Mint + claim a member. |
| +const T = randomSecret(); |
| +const { id, claimSecret, claimHash } = await deriveInvite(T); |
| +await api("/api/admin/key-invites", { |
| + method: "POST", headers: { Authorization: `Bearer ${admin}` }, |
| + body: JSON.stringify({ label: "CkptTester", id, wrapped: "", claim_hash: claimHash }), |
| +}); |
| +const { token } = await api(`/api/key-invite/${encodeURIComponent(id)}/claim`, { |
| + method: "POST", body: JSON.stringify({ claim_secret: claimSecret }), |
| +}); |
| +const me = await api(`/api/me?token=${encodeURIComponent(token)}`); |
| +const uid = me.user_id || me.id || me.uid; |
| +if (!uid) throw new Error("no uid in /api/me: " + JSON.stringify(me)); |
| +console.log("joined as", uid); |
| + |
| +// Baseline: the journal head already covered by checkpoints (0 on a fresh |
| +// instance), so re-runs against a lived-in canary expect the right seqs. |
| +const before = await api(`/api/checkpoints?token=${encodeURIComponent(token)}`); |
| +const baseSeq = before.checkpoints.length ? before.checkpoints[before.checkpoints.length - 1].seq_to : 0; |
| +console.log("journal head before posting:", baseSeq); |
| + |
| +// 2. A self thread + three messages (blobs are opaque to the server; any |
| +// string stands in for ciphertext). |
| +const th = await api("/api/threads", { |
| + method: "POST", |
| + body: JSON.stringify({ token, kind: "self", name_blob: "", members: [{ user_id: uid, wrapped: "x" }] }), |
| +}); |
| +const threadID = th.id || th.thread_id || (th.thread && th.thread.id); |
| +if (!threadID) throw new Error("no thread id: " + JSON.stringify(th)); |
| +const posted = []; |
| +for (const blob of ["ckpt-blob-1", "ckpt-blob-2", "ckpt-blob-3"]) { |
| + const m = await api(`/api/messages?token=${encodeURIComponent(token)}`, { method: "POST", body: JSON.stringify({ thread_id: threadID, blob }) }); |
| + posted.push({ msgID: m.id, ts: m.created_at, blob }); |
| +} |
| +console.log("posted", posted.map((p) => p.msgID).join(","), "in thread", threadID); |
| + |
| +// 3. Wait for the sweep (5-minute ticker) to fold them into a checkpoint. |
| +let cps = [], pub = ""; |
| +for (let i = 0; i < 24; i++) { |
| + const r = await api(`/api/checkpoints?token=${encodeURIComponent(token)}`); |
| + cps = r.checkpoints; pub = r.public_key; |
| + const covered = cps.length && cps[cps.length - 1].seq_to >= baseSeq + posted.length; |
| + if (covered) break; |
| + process.stdout.write(`waiting for sweep (${i * 20}s, ${cps.length} checkpoints)\r`); |
| + await new Promise((res) => setTimeout(res, 20000)); |
| +} |
| +console.log(`\ngot ${cps.length} checkpoint(s); served pub=${pub.slice(0, 12)}…`); |
| +if (!cps.length) { console.error("FAIL: no checkpoints after ~8 min"); process.exit(1); } |
| + |
| +// 4. Verify: signature, payload self-consistency, chain linkage. |
| +const spki = (raw) => createPublicKey({ |
| + key: Buffer.concat([Buffer.from("302a300506032b6570032100", "hex"), raw]), format: "der", type: "spki", |
| +}); |
| +const pubKey = spki(Buffer.from(pub, "base64")); |
| +const chainHash = (prev, root, seqTo) => |
| + sha256hex(`lchat-ckpt-chain\n${prev}\n${root}\n${seqTo}`); |
| +let prevChain = "", fail = 0; |
| +const payloads = []; |
| +for (const cp of cps) { |
| + const w = JSON.parse(cp.signed); |
| + const ok = edVerify(null, Buffer.from(w.payload, "utf8"), pubKey, Buffer.from(w.sig, "base64")); |
| + const p = JSON.parse(Buffer.from(w.payload, "base64").toString("utf8")); |
| + payloads.push(p); |
| + const checks = { |
| + signature: ok, |
| + "payload names served key": p.pub === pub, |
| + "chain recomputes": chainHash(p.prev_chain, p.root, p.seq_to) === p.chain, |
| + "links to predecessor": p.prev_chain === prevChain, |
| + "columns match payload": cp.seq_from === p.seq_from && cp.seq_to === p.seq_to && cp.root === p.root && cp.chain === p.chain, |
| + }; |
| + for (const [name, v] of Object.entries(checks)) if (!v) { console.error(`FAIL cp#${cp.id}: ${name}`); fail++; } |
| + prevChain = p.chain; |
| +} |
| + |
| +// 5. Recompute the Merkle root from the EXPECTED journal (fresh instance: our |
| +// three messages are journal seqs 1..3, kind 'msg') — the verifier's real check. |
| +const leafHash = (r) => sha256buf(Buffer.from( |
| + `lchat-ckpt-leaf\n${r.seq}\n${r.ts}\nmsg\n${r.msgID}\n${sha256hex(r.blob)}\n${threadID}`, "utf8")); |
| +const node = (l, r) => sha256buf(Buffer.concat([Buffer.from("lchat-ckpt-node\n", "utf8"), l, r])); |
| +const merkle = (ls) => { |
| + if (ls.length === 1) return ls[0]; |
| + let k = 1; while (k * 2 < ls.length) k *= 2; |
| + return node(merkle(ls.slice(0, k)), merkle(ls.slice(k))); |
| +}; |
| +const expectRows = posted.map((p, i) => ({ seq: baseSeq + i + 1, ...p })); |
| +const covering = payloads.find((p) => p.seq_from === baseSeq + 1 && p.seq_to === baseSeq + expectRows.length); |
| +if (!covering) { |
| + console.error(`NOTE: no single checkpoint covers seqs ${baseSeq + 1}..${baseSeq + expectRows.length} (ranges: ${payloads.map((p) => `${p.seq_from}-${p.seq_to}`).join(", ")}) — root recomputation skipped`); |
| +} else if (Buffer.from(covering.root, "hex").equals(merkle(expectRows.map(leafHash)))) { |
| + console.log("root recomputation from expected journal: MATCH"); |
| +} else { console.error("FAIL: recomputed Merkle root does not match signed root"); fail++; } |
| + |
| +console.log(fail ? `E2E FAILED (${fail} checks)` : "E2E PASS: signature, chain, linkage, columns, and Merkle root all verify"); |
| +process.exit(fail ? 1 : 0); |