| 1 | package main |
| 2 | |
| 3 | // 🤖 Blob mirroring + audit (federated blobs Phase A, #992 — |
| 4 | // docs/federated-blob-storage.md): a second S3-compatible store holds a copy |
| 5 | // of every store-backed blob, and both stores are spot-checked against the |
| 6 | // sha256 the upload recorded. Write-once ciphertext makes all of this |
| 7 | // embarrassingly simple: the mirror sweep is idempotent and order-free (the |
| 8 | // blobs table IS the journal — no state machine, no lease), and a re-run |
| 9 | // after any crash converges. |
| 10 | // |
| 11 | // Enabled per instance by the LCHAT_BLOB_S3_MIRROR_* env quartet (+ key |
| 12 | // pair) beside the primary's; requires the primary S3 store (a mirror of the |
| 13 | // bare disk layout is the fleet backup's job, not this one's). Reads chain |
| 14 | // primary → mirror → disk, so a store outage degrades to slower reads, never |
| 15 | // errors. Deletes reap both stores. Changing the mirror TARGET later means |
| 16 | // resetting the watermark: UPDATE blobs SET mirrored_at=0 — documented beside |
| 17 | // the env reference; Phase B's store registry replaces this with per-store |
| 18 | // placement rows. |
| 19 | |
| 20 | import ( |
| 21 | "crypto/sha256" |
| 22 | "encoding/hex" |
| 23 | "errors" |
| 24 | "io" |
| 25 | "io/fs" |
| 26 | "log" |
| 27 | "time" |
| 28 | ) |
| 29 | |
| 30 | // mirroredBlobStore composes the primary (which itself falls back to disk for |
| 31 | // pre-flip blobs) with the mirror. The store interface stays the seam: the |
| 32 | // sweep below copies through create/commit, so nothing here knows S3. |
| 33 | type mirroredBlobStore struct { |
| 34 | primary blobStore |
| 35 | mirror blobStore |
| 36 | } |
| 37 | |
| 38 | func (m *mirroredBlobStore) create() (blobWriter, error) { return m.primary.create() } |
| 39 | |
| 40 | func (m *mirroredBlobStore) open(id string) (io.ReadSeekCloser, int64, time.Time, error) { |
| 41 | r, size, mod, err := m.primary.open(id) |
| 42 | if err == nil { |
| 43 | return r, size, mod, nil |
| 44 | } |
| 45 | if errors.Is(err, fs.ErrNotExist) { |
| 46 | // Primary outage looks like errors, not not-founds; a genuine miss |
| 47 | // (a lost primary object) served from the mirror is the redundancy |
| 48 | // paying out — loudly, because it means the primary needs healing. |
| 49 | if r2, s2, m2, err2 := m.mirror.open(id); err2 == nil { |
| 50 | log.Printf("blobs: %s served from the MIRROR (primary miss — investigate)", id) |
| 51 | return r2, s2, m2, nil |
| 52 | } |
| 53 | } |
| 54 | return nil, 0, time.Time{}, err |
| 55 | } |
| 56 | |
| 57 | func (m *mirroredBlobStore) remove(id string) error { |
| 58 | err := m.primary.remove(id) |
| 59 | if err2 := m.mirror.remove(id); err == nil { |
| 60 | err = err2 |
| 61 | } |
| 62 | return err |
| 63 | } |
| 64 | |
| 65 | // ---- the mirror sweep ------------------------------------------------------- |
| 66 | |
| 67 | // unmirroredBlobs returns store-backed rows the mirror lacks, oldest first. |
| 68 | // Legacy in-DB rows (bytes in the data column) are excluded by the |
| 69 | // length(data)=0 predicate — they ride the DB's own replication. |
| 70 | func (db *DB) unmirroredBlobs(limit int) ([]struct{ ID, SHA string }, error) { |
| 71 | rows, err := db.sql.Query( |
| 72 | `SELECT id, sha256 FROM blobs |
| 73 | WHERE mirrored_at = 0 AND length(data) = 0 |
| 74 | ORDER BY created_at LIMIT ?`, limit) |
| 75 | if err != nil { |
| 76 | return nil, err |
| 77 | } |
| 78 | defer rows.Close() |
| 79 | var out []struct{ ID, SHA string } |
| 80 | for rows.Next() { |
| 81 | var r struct{ ID, SHA string } |
| 82 | if err := rows.Scan(&r.ID, &r.SHA); err != nil { |
| 83 | return nil, err |
| 84 | } |
| 85 | out = append(out, r) |
| 86 | } |
| 87 | return out, rows.Err() |
| 88 | } |
| 89 | |
| 90 | func (db *DB) markMirrored(id, sha256Hex string) error { |
| 91 | _, err := db.sql.Exec( |
| 92 | "UPDATE blobs SET mirrored_at = ?, sha256 = ? WHERE id = ?", |
| 93 | time.Now().Unix(), sha256Hex, id) |
| 94 | return err |
| 95 | } |
| 96 | |
| 97 | // mirrorOnce copies one batch of unmirrored blobs primary→mirror, hashing in |
| 98 | // transit: rows without a sha256 (pre-hash uploads) get it backfilled — the |
| 99 | // sweep reads the full bytes anyway — and rows WITH one are verified, so a |
| 100 | // primary that corrupted a blob is caught before the corruption spreads to |
| 101 | // the copy. Returns how many blobs were mirrored this pass. |
| 102 | func (db *DB) mirrorOnce(batch int) (int, error) { |
| 103 | m, ok := db.blobs.(*mirroredBlobStore) |
| 104 | if !ok { |
| 105 | return 0, nil |
| 106 | } |
| 107 | pending, err := db.unmirroredBlobs(batch) |
| 108 | if err != nil { |
| 109 | return 0, err |
| 110 | } |
| 111 | done := 0 |
| 112 | for _, row := range pending { |
| 113 | r, _, _, err := m.primary.open(row.ID) |
| 114 | if err != nil { |
| 115 | if errors.Is(err, fs.ErrNotExist) { |
| 116 | // Deleted since the query (GC won the race) — the row is gone |
| 117 | // or going; either way the next pass won't see it. A row that |
| 118 | // PERSISTS with no bytes is a durability incident the audit |
| 119 | // pass reports. |
| 120 | continue |
| 121 | } |
| 122 | return done, err |
| 123 | } |
| 124 | w, err := m.mirror.create() |
| 125 | if err != nil { |
| 126 | r.Close() |
| 127 | return done, err |
| 128 | } |
| 129 | hasher := sha256.New() |
| 130 | _, copyErr := io.Copy(io.MultiWriter(w, hasher), r) |
| 131 | r.Close() |
| 132 | if copyErr != nil { |
| 133 | w.abort() |
| 134 | return done, copyErr |
| 135 | } |
| 136 | sum := hex.EncodeToString(hasher.Sum(nil)) |
| 137 | if row.SHA != "" && sum != row.SHA { |
| 138 | // The PRIMARY's bytes don't match what was uploaded. Never mirror |
| 139 | // corruption — leave the row unmirrored and shout; the mirror may |
| 140 | // be about to become the only good copy there is. |
| 141 | w.abort() |
| 142 | log.Printf("blobs: INTEGRITY: primary bytes for %s hash %.12s, recorded %.12s — not mirrored", row.ID, sum, row.SHA) |
| 143 | continue |
| 144 | } |
| 145 | if err := w.commit(row.ID); err != nil { |
| 146 | return done, err |
| 147 | } |
| 148 | if err := db.markMirrored(row.ID, sum); err != nil { |
| 149 | return done, err |
| 150 | } |
| 151 | done++ |
| 152 | } |
| 153 | return done, nil |
| 154 | } |
| 155 | |
| 156 | // ---- the audit sweep -------------------------------------------------------- |
| 157 | |
| 158 | // blobAuditResult is one pass's verdict, per store — returned (not just |
| 159 | // logged) so tests can pin the behavior and Phase B can publish it. |
| 160 | type blobAuditResult struct { |
| 161 | Checked, BadPrimary, BadMirror, MissingPrimary, MissingMirror int |
| 162 | } |
| 163 | |
| 164 | // sampleHashedBlobs picks up to k random mirrored, hashed rows to audit. |
| 165 | func (db *DB) sampleHashedBlobs(k int) (map[string]string, error) { |
| 166 | rows, err := db.sql.Query( |
| 167 | `SELECT id, sha256 FROM blobs |
| 168 | WHERE sha256 != '' AND mirrored_at > 0 AND length(data) = 0 |
| 169 | ORDER BY RANDOM() LIMIT ?`, k) |
| 170 | if err != nil { |
| 171 | return nil, err |
| 172 | } |
| 173 | defer rows.Close() |
| 174 | out := map[string]string{} |
| 175 | for rows.Next() { |
| 176 | var id, sha string |
| 177 | if err := rows.Scan(&id, &sha); err != nil { |
| 178 | return nil, err |
| 179 | } |
| 180 | out[id] = sha |
| 181 | } |
| 182 | return out, rows.Err() |
| 183 | } |
| 184 | |
| 185 | func hashFromStore(store blobStore, id string) (string, error) { |
| 186 | r, _, _, err := store.open(id) |
| 187 | if err != nil { |
| 188 | return "", err |
| 189 | } |
| 190 | defer r.Close() |
| 191 | hasher := sha256.New() |
| 192 | if _, err := io.Copy(hasher, r); err != nil { |
| 193 | return "", err |
| 194 | } |
| 195 | return hex.EncodeToString(hasher.Sum(nil)), nil |
| 196 | } |
| 197 | |
| 198 | // auditOnce full-reads k sampled blobs from EACH store and verifies them |
| 199 | // against the recorded hash — probabilistic coverage over time at a few |
| 200 | // requests per pass (docs/federated-blob-storage.md § audit sweep). Mismatches |
| 201 | // and misses are loud: an integrity failure on either store is the signal the |
| 202 | // whole trust-but-verify model hangs on. |
| 203 | func (db *DB) auditBlobsOnce(k int) (blobAuditResult, error) { |
| 204 | var res blobAuditResult |
| 205 | m, ok := db.blobs.(*mirroredBlobStore) |
| 206 | if !ok { |
| 207 | return res, nil |
| 208 | } |
| 209 | sample, err := db.sampleHashedBlobs(k) |
| 210 | if err != nil { |
| 211 | return res, err |
| 212 | } |
| 213 | check := func(store blobStore, id, want string, bad, missing *int, name string) { |
| 214 | got, err := hashFromStore(store, id) |
| 215 | switch { |
| 216 | case errors.Is(err, fs.ErrNotExist): |
| 217 | *missing++ |
| 218 | log.Printf("blobs: AUDIT: %s MISSING from %s", id, name) |
| 219 | case err != nil: |
| 220 | // Transient store trouble isn't an integrity verdict — skip, the |
| 221 | // next pass re-samples. |
| 222 | case got != want: |
| 223 | *bad++ |
| 224 | log.Printf("blobs: AUDIT: %s CORRUPT on %s (hash %.12s, recorded %.12s)", id, name, got, want) |
| 225 | } |
| 226 | } |
| 227 | for id, want := range sample { |
| 228 | res.Checked++ |
| 229 | check(m.primary, id, want, &res.BadPrimary, &res.MissingPrimary, "primary") |
| 230 | check(m.mirror, id, want, &res.BadMirror, &res.MissingMirror, "mirror") |
| 231 | } |
| 232 | return res, nil |
| 233 | } |
| 234 | |
| 235 | // ---- the loops --------------------------------------------------------------- |
| 236 | |
| 237 | // blobMirrorKick nudges the sweep after an upload so a fresh blob mirrors |
| 238 | // within seconds while the space is awake (a hibernating space's unmirrored |
| 239 | // tail waits for its next wake — the honest Phase A window). |
| 240 | func (s *Server) kickBlobMirror() { |
| 241 | if s.blobMirrorKick == nil { |
| 242 | return |
| 243 | } |
| 244 | select { |
| 245 | case s.blobMirrorKick <- struct{}{}: |
| 246 | default: // a kick is already pending — the sweep drains everything anyway |
| 247 | } |
| 248 | } |
| 249 | |
| 250 | // startBlobMirror launches the mirror + audit loops when the store is |
| 251 | // mirrored. Cadences: the mirror ticks every 5m (plus upload kicks) and |
| 252 | // drains in batches; the audit samples a handful hourly — both idle at zero |
| 253 | // S3 requests when there's nothing to do (#836's rule). |
| 254 | func (s *Server) startBlobMirror() { |
| 255 | if _, ok := s.db.blobs.(*mirroredBlobStore); !ok { |
| 256 | return |
| 257 | } |
| 258 | s.blobMirrorKick = make(chan struct{}, 1) |
| 259 | go func() { |
| 260 | tick := time.NewTicker(5 * time.Minute) |
| 261 | defer tick.Stop() |
| 262 | for { |
| 263 | select { |
| 264 | case <-tick.C: |
| 265 | case <-s.blobMirrorKick: |
| 266 | } |
| 267 | for { |
| 268 | n, err := s.db.mirrorOnce(32) |
| 269 | if err != nil { |
| 270 | log.Printf("blobs: mirror sweep: %v (retrying next tick)", err) |
| 271 | break |
| 272 | } |
| 273 | if n < 32 { |
| 274 | break // drained |
| 275 | } |
| 276 | } |
| 277 | } |
| 278 | }() |
| 279 | go func() { |
| 280 | tick := time.NewTicker(1 * time.Hour) |
| 281 | defer tick.Stop() |
| 282 | for range tick.C { |
| 283 | if res, err := s.db.auditBlobsOnce(3); err != nil { |
| 284 | log.Printf("blobs: audit sweep: %v", err) |
| 285 | } else if res.BadPrimary+res.BadMirror+res.MissingPrimary+res.MissingMirror > 0 { |
| 286 | log.Printf("blobs: AUDIT FAILURES this pass: %+v", res) |
| 287 | } |
| 288 | } |
| 289 | }() |
| 290 | log.Printf("blobs: mirror + audit sweeps on") |
| 291 | } |
| 292 | |
amadan