| 1 | package ops |
| 2 | |
| 3 | // Is the replica CURRENT, not merely restorable? |
| 4 | // |
| 5 | // The drill's original question — "does this replica restore clean?" — has a |
| 6 | // blind spot that only shows up in the failure it is meant to catch. A |
| 7 | // replicator that dies leaves behind a replica that is valid SQLite, restores |
| 8 | // perfectly, and passes `integrity_check` forever, while the database it |
| 9 | // mirrors moves on without it. The drill stays green; the backup quietly |
| 10 | // becomes a snapshot of the day the replicator died. |
| 11 | // |
| 12 | // This is not hypothetical. On 2026-08-11 the flagship's litestream exited |
| 13 | // with "permission denied" writing its shadow WAL into a DynamicUser tenant's |
| 14 | // state directory (a root drop-in fixed it the same minute). Had nobody been |
| 15 | // watching the journal, nothing in the platform would have said a word: the |
| 16 | // next morning's drill would have restored four frozen replicas and reported |
| 17 | // them clean. |
| 18 | // |
| 19 | // The signal used here is SQLite's own: the file-change counter in the |
| 20 | // database header, which advances on every write transaction that reaches the |
| 21 | // main database file. Compare the source's counter with the restored copy's, |
| 22 | // and a replica that has stopped tracking its source shows up as a number that |
| 23 | // stopped moving. No S3 layout knowledge, no litestream version dependency, |
| 24 | // no clock skew between machines — the comparison is between two files this |
| 25 | // process just read. |
| 26 | |
| 27 | import ( |
| 28 | "fmt" |
| 29 | "os" |
| 30 | "time" |
| 31 | |
| 32 | "github.com/carlosframework/platform/internal/lease" |
| 33 | ) |
| 34 | |
| 35 | // staleGrace is how long a source must have been QUIET before a replica that |
| 36 | // trails it is called stale. It exists to make false alarms impossible in the |
| 37 | // ordinary case: a database written seconds before the drill ran is expected |
| 38 | // to be a transaction or two ahead of its replica, and reporting that as a |
| 39 | // fault would put the drill permanently red on the busiest — most valuable — |
| 40 | // databases on the box, which is precisely how a check stops being read. |
| 41 | // |
| 42 | // 15 minutes is far longer than any replication lag this fleet has (litestream |
| 43 | // syncs on change, seconds at most) and far shorter than the gap that matters |
| 44 | // (a replicator that died). |
| 45 | const staleGrace = 15 * time.Minute |
| 46 | |
| 47 | type freshness int |
| 48 | |
| 49 | const ( |
| 50 | // freshnessOK: the replica is level with its source, or trailing a source |
| 51 | // that was written too recently to judge. |
| 52 | freshnessOK freshness = iota |
| 53 | // freshnessStale: the source has been quiet for longer than staleGrace and |
| 54 | // the replica is STILL behind it. Replication is not happening. |
| 55 | freshnessStale |
| 56 | // freshnessNoSource: the config names a database that is not on disk. The |
| 57 | // replica cannot ever advance again, and nothing is watching the gap. |
| 58 | freshnessNoSource |
| 59 | // freshnessUnknown: one of the two headers could not be read. Say so; |
| 60 | // never let "could not tell" read as "fine". |
| 61 | freshnessUnknown |
| 62 | ) |
| 63 | |
| 64 | func (f freshness) String() string { |
| 65 | switch f { |
| 66 | case freshnessOK: |
| 67 | return "ok" |
| 68 | case freshnessStale: |
| 69 | return "STALE" |
| 70 | case freshnessNoSource: |
| 71 | return "no-source" |
| 72 | default: |
| 73 | return "unknown" |
| 74 | } |
| 75 | } |
| 76 | |
| 77 | // ChangeCounter is lease.ChangeCounter — one definition of the freshness |
| 78 | // signal, shared by both drills. Kept exported here because this package's |
| 79 | // tests and callers are its documented door. |
| 80 | func ChangeCounter(path string) (uint32, error) { return lease.ChangeCounter(path) } |
| 81 | |
| 82 | // replicaFreshness compares a restored replica against the live database it |
| 83 | // mirrors. srcModTime is the source's last-modified time, passed in so the |
| 84 | // caller stats once and tests can drive the grace window. |
| 85 | func replicaFreshness(srcPath, restoredPath string, srcModTime time.Time) (freshness, string) { |
| 86 | restoredCounter, err := ChangeCounter(restoredPath) |
| 87 | if err != nil { |
| 88 | return freshnessUnknown, fmt.Sprintf("restored header unreadable: %v", err) |
| 89 | } |
| 90 | return replicaFreshnessCounter(srcPath, restoredCounter, srcModTime) |
| 91 | } |
| 92 | |
| 93 | // replicaFreshnessCounter is replicaFreshness for a caller that already holds |
| 94 | // the restored copy's counter and no longer has the file — which is the |
| 95 | // hibernation drill, where lease.VerifyReplica reads the counter and then |
| 96 | // sweeps its throwaway copy in the same call. |
| 97 | func replicaFreshnessCounter(srcPath string, restoredCounter uint32, srcModTime time.Time) (freshness, string) { |
| 98 | srcCounter, err := ChangeCounter(srcPath) |
| 99 | if os.IsNotExist(err) { |
| 100 | return freshnessNoSource, "source database is not on disk — this replica can never advance again" |
| 101 | } |
| 102 | if err != nil { |
| 103 | return freshnessUnknown, fmt.Sprintf("source header unreadable: %v", err) |
| 104 | } |
| 105 | if restoredCounter >= srcCounter { |
| 106 | return freshnessOK, "" |
| 107 | } |
| 108 | if quiet := time.Since(srcModTime); quiet <= staleGrace { |
| 109 | // Behind, but the source was written within the grace window — this is |
| 110 | // ordinary lag, and the next pass will judge it properly. |
| 111 | return freshnessOK, "" |
| 112 | } |
| 113 | return freshnessStale, fmt.Sprintf( |
| 114 | "replica is at change %d, the live database at %d, and the database has been quiet since %s — replication has stopped", |
| 115 | restoredCounter, srcCounter, srcModTime.UTC().Format(time.RFC3339)) |
| 116 | } |
| 117 | |
| 118 | // sourceModTime is the source database's mtime, or the zero time when it |
| 119 | // cannot be stated (which replicaFreshness will treat through its own paths). |
| 120 | func sourceModTime(path string) time.Time { |
| 121 | fi, err := os.Stat(path) |
| 122 | if err != nil { |
| 123 | return time.Time{} |
| 124 | } |
| 125 | return fi.ModTime() |
| 126 | } |
| 127 | |
amadan