amadan

eleven / messenger Public

server: seal the link-preview cache at rest + TTL eviction (#727)

The cache stored plaintext URLs and OpenGraph cards derived from message
content, never evicted — and hibernation now replicates it to S3. Rows are
now found by a salted sha256 of the URL and hold one AES-GCM blob under a
key HKDF-derived from the URL itself (convergent): the server derives it
transiently at request time — when it already sees the URL, the documented
unfurl exception — and stores only ciphertext. A put-time sweep evicts rows
older than 30 days. The old plaintext table is dropped on migrate (it's a
cache; rows re-fetch on demand). Clients and the JSON shape are untouched.

The per-instance salt defeats cross-instance precomputation only; at rest
the protection is the URL's own entropy — bulk reading of the cache is
gone, guessable URLs stay guessable (docs/no-plaintext.md).

Part of #724, step 2 of docs/no-plaintext.md.
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Paul Campbell eed9117d7ab32de696a3723033715500b5e54285 parent 9fec5b2
3 files changed, +260 −27
  • db.go +72 −24
  • unfurl.go +91 −3
  • unfurl_test.go +97 −0
diff --git a/db.go b/db.go
index 85293d5..c561870 100644
--- a/db.go
+++ b/db.go
@@ -298,13 +298,16 @@ CREATE TABLE IF NOT EXISTS device_tokens (
label TEXT NOT NULL DEFAULT '',
created_at INTEGER NOT NULL
);
-CREATE TABLE IF NOT EXISTS link_previews (
- url TEXT PRIMARY KEY,
- title TEXT NOT NULL DEFAULT '',
- description TEXT NOT NULL DEFAULT '',
- image TEXT NOT NULL DEFAULT '',
- ok INTEGER NOT NULL DEFAULT 0,
- fetched_at INTEGER NOT NULL
+-- Link-preview cache, sealed at rest (docs/no-plaintext.md): the row key is a
+-- salted hash of the URL and the card is AES-GCM ciphertext under a key derived
+-- from the URL itself (convergent — only someone who knows the URL can find or
+-- open a row; the server derives it transiently at request time, when it sees
+-- the URL anyway). fetched_at stays plaintext for TTL eviction — a timestamp,
+-- like messages.created_at.
+CREATE TABLE IF NOT EXISTS link_previews_sealed (
+ url_hash TEXT PRIMARY KEY,
+ blob TEXT NOT NULL,
+ fetched_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS threads (
id TEXT PRIMARY KEY,
@@ -548,6 +551,11 @@ CREATE TABLE IF NOT EXISTS member_bots (
// the table so a stolen DB retains no plaintext credentials (same
// rationale as the invites drop above).
"DROP TABLE IF EXISTS reactions",
+ // The old link-preview cache stored plaintext URLs + OpenGraph cards
+ // derived from message content, unevicted. Superseded by
+ // link_previews_sealed (ciphertext at rest); it's a cache, so old rows
+ // are simply dropped and re-fetched on demand.
+ "DROP TABLE IF EXISTS link_previews",
// Which user a push subscription belongs to, so removing a user can drop their
// subscriptions (sign-out clears them by token; removal has only the user id).
"ALTER TABLE push_subscriptions ADD COLUMN user_id TEXT",
@@ -753,36 +761,76 @@ func (db *DB) backfillTokenHash(table, keyCol string) error {
return nil
}
-// ---- link previews (cache) ----------------------------------------------
+// ---- link previews (cache, sealed at rest) -------------------------------
+//
+// Rows are found by a salted hash of the URL and opened with a key derived
+// from the URL (see sealPreview/openPreview in unfurl.go). The per-instance
+// random salt only defeats precomputation across instances — it is not a
+// secret; the protection at rest is the URL's own entropy, which is exactly
+// the honest claim: bulk reading of the cache is gone, guessable URLs are
+// guessable (docs/no-plaintext.md § the plan).
+
+// previewSalt returns the instance's preview-cache salt, minting it on first
+// use (INSERT OR IGNORE + read-back keeps concurrent first calls agreeing).
+func (db *DB) previewSalt() (string, error) {
+ salt, err := randToken()
+ if err != nil {
+ return "", err
+ }
+ if _, err := db.sql.Exec(
+ "INSERT OR IGNORE INTO config(key, value) VALUES('preview_salt', ?)", salt); err != nil {
+ return "", err
+ }
+ v, _, err := db.getConfig("preview_salt")
+ return v, err
+}
func (db *DB) getLinkPreview(u string) (LinkPreview, bool, error) {
- var pv LinkPreview
- var ok int
- err := db.sql.QueryRow(
- "SELECT url, title, description, image, ok, fetched_at FROM link_previews WHERE url = ?", u,
- ).Scan(&pv.URL, &pv.Title, &pv.Description, &pv.Image, &ok, &pv.FetchedAt)
+ salt, err := db.previewSalt()
+ if err != nil {
+ return LinkPreview{}, false, err
+ }
+ var blob string
+ var fetchedAt int64
+ err = db.sql.QueryRow(
+ "SELECT blob, fetched_at FROM link_previews_sealed WHERE url_hash = ?", previewURLHash(salt, u),
+ ).Scan(&blob, &fetchedAt)
if err == sql.ErrNoRows {
return LinkPreview{}, false, nil
}
if err != nil {
return LinkPreview{}, false, err
}
- pv.OK = ok != 0
+ pv, ok := openPreview(salt, u, blob)
+ if !ok {
+ // Undecryptable row (corruption): treat as a miss; the next put
+ // overwrites it under the same hash.
+ return LinkPreview{}, false, nil
+ }
+ pv.URL = u
+ pv.FetchedAt = fetchedAt
return pv, true, nil
}
func (db *DB) putLinkPreview(pv LinkPreview) error {
- ok := 0
- if pv.OK {
- ok = 1
+ salt, err := db.previewSalt()
+ if err != nil {
+ return err
}
- _, err := db.sql.Exec(
- `INSERT INTO link_previews(url, title, description, image, ok, fetched_at)
- VALUES(?, ?, ?, ?, ?, ?)
- ON CONFLICT(url) DO UPDATE SET
- title=excluded.title, description=excluded.description,
- image=excluded.image, ok=excluded.ok, fetched_at=excluded.fetched_at`,
- pv.URL, pv.Title, pv.Description, pv.Image, ok, pv.FetchedAt,
+ blob, err := sealPreview(salt, pv)
+ if err != nil {
+ return err
+ }
+ // Put-time TTL sweep: the cache self-prunes with activity, so previews of
+ // long-forgotten URLs don't sit at rest (or in hibernation replicas) forever.
+ db.sql.Exec("DELETE FROM link_previews_sealed WHERE fetched_at < ?",
+ time.Now().Unix()-previewMaxAge)
+ _, err = db.sql.Exec(
+ `INSERT INTO link_previews_sealed(url_hash, blob, fetched_at)
+ VALUES(?, ?, ?)
+ ON CONFLICT(url_hash) DO UPDATE SET
+ blob=excluded.blob, fetched_at=excluded.fetched_at`,
+ previewURLHash(salt, pv.URL), blob, pv.FetchedAt,
)
return err
}
diff --git a/unfurl.go b/unfurl.go
index 592800e..c535d61 100644
--- a/unfurl.go
+++ b/unfurl.go
@@ -1,6 +1,14 @@
package main
import (
+ "crypto/aes"
+ "crypto/cipher"
+ "crypto/hkdf"
+ "crypto/rand"
+ "crypto/sha256"
+ "encoding/base64"
+ "encoding/hex"
+ "encoding/json"
"errors"
"fmt"
"html"
@@ -20,9 +28,10 @@ import (
// disabled later if that trade-off stops being acceptable.
const (
- previewFreshTTL = 7 * 24 * 60 * 60 // serve-stale-after, then refresh in the background
- previewFailTTL = 60 * 60 // retry failed/blank lookups sooner
- previewMaxBytes = 512 * 1024 // only read the start of the page
+ previewFreshTTL = 7 * 24 * 60 * 60 // serve-stale-after, then refresh in the background
+ previewFailTTL = 60 * 60 // retry failed/blank lookups sooner
+ previewMaxAge = 30 * 24 * 60 * 60 // evict cache rows older than this (put-time sweep)
+ previewMaxBytes = 512 * 1024 // only read the start of the page
previewTimeout = 6 * time.Second
)
@@ -36,6 +45,85 @@ type LinkPreview struct {
FetchedAt int64 `json:"-"`
}
+// ---- sealing the cache at rest (docs/no-plaintext.md) ---------------------
+//
+// Convergent encryption: both the row key and the AES key are derived from the
+// URL itself (plus a per-instance salt that only defeats cross-instance
+// precomputation). The server sees URLs transiently at request time — the
+// documented unfurl exception — derives, opens or seals, and stores only
+// ciphertext, so the cache at rest reveals nothing a reader doesn't already
+// know the URL for.
+
+// previewURLHash is the row key: hex sha256(salt || url).
+func previewURLHash(salt, u string) string {
+ h := sha256.Sum256([]byte(salt + u))
+ return hex.EncodeToString(h[:])
+}
+
+// previewAEAD derives the URL's AES-GCM cipher.
+func previewAEAD(salt, u string) (cipher.AEAD, error) {
+ key, err := hkdf.Key(sha256.New, []byte(u), []byte(salt), "lchat link preview v1", 32)
+ if err != nil {
+ return nil, err
+ }
+ block, err := aes.NewCipher(key)
+ if err != nil {
+ return nil, err
+ }
+ return cipher.NewGCM(block)
+}
+
+// sealedPreview is the card as stored inside the ciphertext — everything but
+// the URL (which is the key) and fetched_at (plaintext, for eviction).
+type sealedPreview struct {
+ Title string `json:"title"`
+ Description string `json:"description"`
+ Image string `json:"image"`
+ OK bool `json:"ok"`
+}
+
+// sealPreview encrypts a card to the base64url iv||ciphertext format
+// (crypto.js's convention, kept for consistency).
+func sealPreview(salt string, pv LinkPreview) (string, error) {
+ aead, err := previewAEAD(salt, pv.URL)
+ if err != nil {
+ return "", err
+ }
+ iv := make([]byte, aead.NonceSize())
+ if _, err := rand.Read(iv); err != nil {
+ return "", err
+ }
+ plain, err := json.Marshal(sealedPreview{
+ Title: pv.Title, Description: pv.Description, Image: pv.Image, OK: pv.OK,
+ })
+ if err != nil {
+ return "", err
+ }
+ return base64.RawURLEncoding.EncodeToString(aead.Seal(iv, iv, plain, nil)), nil
+}
+
+// openPreview decrypts a stored card; false on any mismatch or corruption
+// (callers treat that as a cache miss).
+func openPreview(salt, u, blob string) (LinkPreview, bool) {
+ aead, err := previewAEAD(salt, u)
+ if err != nil {
+ return LinkPreview{}, false
+ }
+ raw, err := base64.RawURLEncoding.DecodeString(blob)
+ if err != nil || len(raw) < aead.NonceSize() {
+ return LinkPreview{}, false
+ }
+ plain, err := aead.Open(nil, raw[:aead.NonceSize()], raw[aead.NonceSize():], nil)
+ if err != nil {
+ return LinkPreview{}, false
+ }
+ var sp sealedPreview
+ if json.Unmarshal(plain, &sp) != nil {
+ return LinkPreview{}, false
+ }
+ return LinkPreview{Title: sp.Title, Description: sp.Description, Image: sp.Image, OK: sp.OK}, true
+}
+
// allowPrivatePreviewHosts is flipped on only in tests (so an httptest server on
// loopback can be fetched); in production the SSRF guard always applies.
var allowPrivatePreviewHosts = false
diff --git a/unfurl_test.go b/unfurl_test.go
index eaa43d4..95d83b8 100644
--- a/unfurl_test.go
+++ b/unfurl_test.go
@@ -5,7 +5,9 @@ import (
"net"
"net/http"
"net/http/httptest"
+ "strings"
"testing"
+ "time"
)
func TestIsPublicIP(t *testing.T) {
@@ -95,3 +97,98 @@ func TestUnfurlEndpoint(t *testing.T) {
t.Fatal("preview was not cached")
}
}
+
+// The cache must hold no plaintext at rest: rows are found by a salted hash of
+// the URL and opened only with a key derived from the URL itself.
+func TestLinkPreviewSealedAtRest(t *testing.T) {
+ db := newTestDB(t)
+ pv := LinkPreview{
+ URL: "https://example.com/a-private-doc", Title: "Quarterly Numbers",
+ Description: "Do not share", Image: "https://example.com/chart.png",
+ OK: true, FetchedAt: time.Now().Unix(),
+ }
+ if err := db.putLinkPreview(pv); err != nil {
+ t.Fatalf("putLinkPreview: %v", err)
+ }
+
+ // Round-trip.
+ got, ok, err := db.getLinkPreview(pv.URL)
+ if err != nil || !ok {
+ t.Fatalf("getLinkPreview: ok=%v err=%v", ok, err)
+ }
+ if got.Title != pv.Title || got.Description != pv.Description || got.Image != pv.Image || !got.OK {
+ t.Fatalf("round-trip = %+v", got)
+ }
+
+ // At rest: neither the URL nor any card field appears in the stored row.
+ var urlHash, blob string
+ if err := db.sql.QueryRow("SELECT url_hash, blob FROM link_previews_sealed").Scan(&urlHash, &blob); err != nil {
+ t.Fatalf("raw row: %v", err)
+ }
+ stored := urlHash + blob
+ for _, secret := range []string{pv.URL, "example.com", "Quarterly", "Do not share", "chart.png"} {
+ if strings.Contains(stored, secret) {
+ t.Fatalf("plaintext %q found in the stored row", secret)
+ }
+ }
+
+ // A different URL neither finds nor opens the row.
+ if _, ok, _ := db.getLinkPreview("https://example.com/other"); ok {
+ t.Fatal("wrong URL matched a cached row")
+ }
+ salt, _ := db.previewSalt()
+ if _, ok := openPreview(salt, "https://example.com/other", blob); ok {
+ t.Fatal("wrong URL decrypted the blob")
+ }
+}
+
+// Old rows age out on the put-time sweep, and the legacy plaintext table is
+// dropped by migration.
+func TestLinkPreviewEvictionAndLegacyDrop(t *testing.T) {
+ dir := t.TempDir()
+ path := dir + "/old.db"
+ old, err := openDB(path)
+ if err != nil {
+ t.Fatalf("openDB: %v", err)
+ }
+ // Seed a legacy plaintext table as an old binary would have left it.
+ if _, err := old.sql.Exec(`CREATE TABLE IF NOT EXISTS link_previews (
+ url TEXT PRIMARY KEY, title TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '',
+ image TEXT NOT NULL DEFAULT '', ok INTEGER NOT NULL DEFAULT 0, fetched_at INTEGER NOT NULL);
+ INSERT INTO link_previews VALUES ('https://example.com/x', 'Plain Title', 'plain', '', 1, 0)`); err != nil {
+ t.Fatalf("seed legacy: %v", err)
+ }
+ old.sql.Close()
+
+ db, err := openDB(path)
+ if err != nil {
+ t.Fatalf("reopen: %v", err)
+ }
+ defer db.sql.Close()
+ var n int
+ if err := db.sql.QueryRow(
+ "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='link_previews'").Scan(&n); err != nil {
+ t.Fatalf("sqlite_master: %v", err)
+ }
+ if n != 0 {
+ t.Fatal("legacy plaintext link_previews table survived migration")
+ }
+
+ // Eviction: an ancient row disappears when the next put sweeps.
+ stale := LinkPreview{URL: "https://example.com/stale", Title: "old", OK: true,
+ FetchedAt: time.Now().Unix() - previewMaxAge - 60}
+ if err := db.putLinkPreview(stale); err != nil {
+ t.Fatalf("put stale: %v", err)
+ }
+ fresh := LinkPreview{URL: "https://example.com/fresh", Title: "new", OK: true,
+ FetchedAt: time.Now().Unix()}
+ if err := db.putLinkPreview(fresh); err != nil {
+ t.Fatalf("put fresh: %v", err)
+ }
+ if _, ok, _ := db.getLinkPreview(stale.URL); ok {
+ t.Fatal("stale row survived the put-time sweep")
+ }
+ if _, ok, _ := db.getLinkPreview(fresh.URL); !ok {
+ t.Fatal("fresh row missing")
+ }
+}