| 1 | package ops |
| 2 | |
| 3 | // The hibernation half of the restore drill. |
| 4 | // |
| 5 | // FleetDBs deliberately EXCLUDES hibernating exec routes from |
| 6 | // /etc/litestream.yml — the activator's embedded litestream owns their |
| 7 | // replication under the lease, and a second replicator on the same WAL would |
| 8 | // corrupt the replica (Eleven's lesson). The unintended consequence was that |
| 9 | // the fleet's LARGEST set of app databases was the one set nobody ever |
| 10 | // restore-drilled: their replicas live in the instances bucket under |
| 11 | // <CARLOS_HIBERNATE_PREFIX>/<host>/, and the first thing to find out whether |
| 12 | // one restores clean was a real user's wake. |
| 13 | // |
| 14 | // This pass closes that. It is a READER: it restores each replica into a |
| 15 | // throwaway file and integrity-checks it (see lease.VerifyReplica for why |
| 16 | // that needs no lease and cannot shadow the true holder's state), so it can |
| 17 | // run against instances that are asleep, awake and actively replicating, or |
| 18 | // anywhere in between. |
| 19 | |
| 20 | import ( |
| 21 | "context" |
| 22 | "errors" |
| 23 | "fmt" |
| 24 | "sort" |
| 25 | "time" |
| 26 | |
| 27 | "github.com/carlosframework/platform/internal/activator" |
| 28 | "github.com/carlosframework/platform/internal/dbverify" |
| 29 | "github.com/carlosframework/platform/internal/lease" |
| 30 | "github.com/carlosframework/platform/internal/registry" |
| 31 | ) |
| 32 | |
| 33 | // HibernatingRoute is one instance this drill owns: the host names its replica |
| 34 | // prefix, and DBPath is the live database the replica is supposed to be |
| 35 | // tracking — carried because "restores clean" and "is current" are two |
| 36 | // different questions and the second one needs the source. |
| 37 | type HibernatingRoute struct { |
| 38 | Host string |
| 39 | DBPath string |
| 40 | } |
| 41 | |
| 42 | // HibernatingHosts lists the routes whose replication the activator owns — |
| 43 | // exactly the set FleetDBs filters out, derived with the SAME conditions so |
| 44 | // the two cannot drift into a gap that belongs to neither pass. |
| 45 | func HibernatingHosts(reg *registry.Registry) ([]HibernatingRoute, error) { |
| 46 | routes, err := reg.List() |
| 47 | if err != nil { |
| 48 | return nil, err |
| 49 | } |
| 50 | var hosts []HibernatingRoute |
| 51 | for _, r := range routes { |
| 52 | // BackingUnit tenants are the box-wide replicator's job (their wake is |
| 53 | // systemctl and the activator never replicates them) — FleetDBs adds |
| 54 | // them, so this pass must not claim them. |
| 55 | if r.Backing == registry.BackingUnit { |
| 56 | continue |
| 57 | } |
| 58 | if r.Hibernate && r.DBPath != "" { |
| 59 | hosts = append(hosts, HibernatingRoute{Host: r.Host, DBPath: r.DBPath}) |
| 60 | } |
| 61 | } |
| 62 | sort.Slice(hosts, func(i, j int) bool { return hosts[i].Host < hosts[j].Host }) |
| 63 | return hosts, nil |
| 64 | } |
| 65 | |
| 66 | // verifyOne is the seam the tests drive; production points it at S3. |
| 67 | var verifyOne = lease.VerifyReplica |
| 68 | |
| 69 | // VerifyHibernating restore-drills up to max of the given hosts and returns a |
| 70 | // verdict per host it VISITED. Hosts beyond the cap are simply absent from |
| 71 | // the result — Merge keeps their previous verdict and its old timestamp |
| 72 | // rather than inventing coverage for them. |
| 73 | // |
| 74 | // Order is the caller's: pass hosts oldest-verified-first so a fleet larger |
| 75 | // than the cap is still covered completely, just over several days. |
| 76 | // |
| 77 | // The cap is not squeamishness — it is the same request-cost discipline that |
| 78 | // set LeaseTTL. Each verification is a full restore: in-region S3→EC2 |
| 79 | // transfer is free, but the GETs and the restore CPU are not, and an |
| 80 | // unbounded nightly drill over a growing fleet is a bill that grows with it. |
| 81 | func VerifyHibernating(ctx context.Context, cfg *activator.Config, hosts []HibernatingRoute, max int, now time.Time, tmpDir string) map[string]dbverify.Verdict { |
| 82 | out := map[string]dbverify.Verdict{} |
| 83 | for i, rt := range hosts { |
| 84 | if max > 0 && i >= max { |
| 85 | break |
| 86 | } |
| 87 | host := rt.Host |
| 88 | prefix := activator.HibernatePrefix(cfg.Prefix, host) |
| 89 | counter, err := verifyOne(ctx, cfg.Bucket, prefix, cfg.Region, tmpDir) |
| 90 | switch { |
| 91 | case err == nil: |
| 92 | // Restores clean — now the second question: is it CURRENT? A |
| 93 | // hibernating instance's replica has a live owner (the activator's |
| 94 | // embedded replicator, under the lease), and an owner that has |
| 95 | // stopped writing leaves a replica that still restores perfectly. |
| 96 | if f, detail := hibernateFreshness(rt.DBPath, counter); f == freshnessStale || f == freshnessNoSource { |
| 97 | out[host] = dbverify.Verdict{VerifiedAt: now.UTC(), Detail: detail} |
| 98 | fmt.Printf("STALE %s: %s\n", host, detail) |
| 99 | continue |
| 100 | } |
| 101 | out[host] = dbverify.Verdict{OK: true, VerifiedAt: now.UTC()} |
| 102 | fmt.Printf("ok %s\n", host) |
| 103 | case errors.Is(err, lease.ErrNoReplica): |
| 104 | // Never replicated: a registered instance that has not woken yet. |
| 105 | // Not a failure — but not a pass either, because nothing was |
| 106 | // proven about anything. |
| 107 | out[host] = dbverify.Verdict{Unverified: true, VerifiedAt: now.UTC(), Detail: "no replica yet"} |
| 108 | fmt.Printf("unverified %s: no replica yet (never woken)\n", host) |
| 109 | case ctx.Err() != nil: |
| 110 | // The run is being torn down; what we could not reach is |
| 111 | // unverified, never failed. |
| 112 | out[host] = dbverify.Verdict{Unverified: true, VerifiedAt: now.UTC(), Detail: err.Error()} |
| 113 | fmt.Printf("unverified %s: %v\n", host, err) |
| 114 | default: |
| 115 | out[host] = dbverify.Verdict{VerifiedAt: now.UTC(), Detail: err.Error()} |
| 116 | fmt.Printf("FAIL %s: %v\n", host, err) |
| 117 | } |
| 118 | } |
| 119 | if max > 0 && len(hosts) > max { |
| 120 | // A silent cap reads as full coverage — say what was left. |
| 121 | fmt.Printf("hibernate-verify: %d of %d hosts this run (--max %d); the rest keep their previous verdict\n", |
| 122 | max, len(hosts), max) |
| 123 | } |
| 124 | return out |
| 125 | } |
| 126 | |
| 127 | // hibernateFreshness asks whether this replica is still tracking its source. |
| 128 | // |
| 129 | // A route with no recorded DBPath yields "ok": there is nothing to compare |
| 130 | // against, and mistaking "we do not know where the source lives" for "the |
| 131 | // source is gone" would raise a false alarm on a healthy instance. |
| 132 | // HibernatingHosts only yields routes with a DBPath, so this is a guard rather |
| 133 | // than a path production takes. |
| 134 | func hibernateFreshness(dbPath string, restoredCounter uint32) (freshness, string) { |
| 135 | if dbPath == "" { |
| 136 | return freshnessOK, "" |
| 137 | } |
| 138 | return replicaFreshnessCounter(dbPath, restoredCounter, sourceModTime(dbPath)) |
| 139 | } |
| 140 | |
| 141 | // OldestFirst orders hosts by how long ago they were last verified, most |
| 142 | // stale first, so a capped run works through the fleet instead of drilling |
| 143 | // the same alphabetical prefix every night. Hosts with no verdict at all sort |
| 144 | // first — never verified is staler than any timestamp. |
| 145 | func OldestFirst(hosts []HibernatingRoute, prior *dbverify.Record) []HibernatingRoute { |
| 146 | stamp := map[string]time.Time{} |
| 147 | if prior != nil { |
| 148 | for h, v := range prior.Hosts { |
| 149 | stamp[h] = v.VerifiedAt |
| 150 | } |
| 151 | } |
| 152 | ordered := append([]HibernatingRoute(nil), hosts...) |
| 153 | sort.SliceStable(ordered, func(i, j int) bool { |
| 154 | a, aok := stamp[ordered[i].Host] |
| 155 | b, bok := stamp[ordered[j].Host] |
| 156 | if aok != bok { |
| 157 | return !aok // never-verified first |
| 158 | } |
| 159 | if a.Equal(b) { |
| 160 | return ordered[i].Host < ordered[j].Host // stable, deterministic in tests |
| 161 | } |
| 162 | return a.Before(b) |
| 163 | }) |
| 164 | return ordered |
| 165 | } |
| 166 | |
amadan