| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "fmt" |
| 7 | "path/filepath" |
| 8 | "testing" |
| 9 | "time" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/registry" |
| 12 | "github.com/carlosframework/platform/internal/release" |
| 13 | ) |
| 14 | |
| 15 | func snapTestRegistry(t *testing.T) *registry.Registry { |
| 16 | t.Helper() |
| 17 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 18 | if err != nil { |
| 19 | t.Fatal(err) |
| 20 | } |
| 21 | t.Cleanup(func() { reg.Close() }) |
| 22 | if err := reg.Upsert(registry.Route{Host: "a.example.com", Kind: "service", Addr: "10.0.0.1:3000", Status: "ready"}); err != nil { |
| 23 | t.Fatal(err) |
| 24 | } |
| 25 | return reg |
| 26 | } |
| 27 | |
| 28 | func TestSnapshotPassWritesCurrentSidecarAndGeneration(t *testing.T) { |
| 29 | s := release.NewDirStore(t.TempDir()) |
| 30 | ctx := context.Background() |
| 31 | reg := snapTestRegistry(t) |
| 32 | at := time.Date(2026, 8, 12, 3, 0, 0, 0, time.UTC) |
| 33 | |
| 34 | var lastFP string |
| 35 | if err := registrySnapshotPass(ctx, s, reg, "box1", at, &lastFP); err != nil { |
| 36 | t.Fatal(err) |
| 37 | } |
| 38 | |
| 39 | raw, _, err := s.Get(ctx, snapshotDBKey("box1")) |
| 40 | if err != nil { |
| 41 | t.Fatalf("current snapshot missing: %v", err) |
| 42 | } |
| 43 | if len(raw) == 0 { |
| 44 | t.Fatal("current snapshot is empty") |
| 45 | } |
| 46 | |
| 47 | metaRaw, _, err := s.Get(ctx, snapshotMetaKey("box1")) |
| 48 | if err != nil { |
| 49 | t.Fatalf("sidecar missing: %v", err) |
| 50 | } |
| 51 | var meta snapshotMeta |
| 52 | if err := json.Unmarshal(metaRaw, &meta); err != nil { |
| 53 | t.Fatal(err) |
| 54 | } |
| 55 | if meta.V != 1 || meta.Box != "box1" || meta.Fingerprint == "" || meta.Bytes != len(raw) { |
| 56 | t.Fatalf("sidecar = %+v (snapshot %d bytes)", meta, len(raw)) |
| 57 | } |
| 58 | if lastFP != meta.Fingerprint { |
| 59 | t.Fatalf("lastFP %q not updated to %q", lastFP, meta.Fingerprint) |
| 60 | } |
| 61 | |
| 62 | gens, err := s.List(ctx, snapshotGenPrefix("box1")) |
| 63 | if err != nil { |
| 64 | t.Fatal(err) |
| 65 | } |
| 66 | if len(gens) != 1 || gens[0] != snapshotGenPrefix("box1")+"20260812T030000Z.db" { |
| 67 | t.Fatalf("generations = %v", gens) |
| 68 | } |
| 69 | } |
| 70 | |
| 71 | func TestSnapshotPassSkipsWhenFingerprintUnchanged(t *testing.T) { |
| 72 | t.Parallel() |
| 73 | s := release.NewDirStore(t.TempDir()) |
| 74 | ctx := context.Background() |
| 75 | reg := snapTestRegistry(t) |
| 76 | at := time.Date(2026, 8, 12, 3, 0, 0, 0, time.UTC) |
| 77 | |
| 78 | var lastFP string |
| 79 | if err := registrySnapshotPass(ctx, s, reg, "box1", at, &lastFP); err != nil { |
| 80 | t.Fatal(err) |
| 81 | } |
| 82 | if err := registrySnapshotPass(ctx, s, reg, "box1", at.Add(5*time.Minute), &lastFP); err != nil { |
| 83 | t.Fatal(err) |
| 84 | } |
| 85 | gens, err := s.List(ctx, snapshotGenPrefix("box1")) |
| 86 | if err != nil { |
| 87 | t.Fatal(err) |
| 88 | } |
| 89 | if len(gens) != 1 { |
| 90 | t.Fatalf("unchanged registry grew generations: %v", gens) |
| 91 | } |
| 92 | |
| 93 | // A registry change makes the next pass write again. |
| 94 | if err := reg.Upsert(registry.Route{Host: "b.example.com", Kind: "service", Addr: "10.0.0.2:3000", Status: "ready"}); err != nil { |
| 95 | t.Fatal(err) |
| 96 | } |
| 97 | if err := registrySnapshotPass(ctx, s, reg, "box1", at.Add(10*time.Minute), &lastFP); err != nil { |
| 98 | t.Fatal(err) |
| 99 | } |
| 100 | gens, err = s.List(ctx, snapshotGenPrefix("box1")) |
| 101 | if err != nil { |
| 102 | t.Fatal(err) |
| 103 | } |
| 104 | if len(gens) != 2 { |
| 105 | t.Fatalf("changed registry did not write a new generation: %v", gens) |
| 106 | } |
| 107 | |
| 108 | // A caps grant touches no edge-reload column, so registry.Fingerprint |
| 109 | // sits still for it — which is why this pass gates on |
| 110 | // SnapshotFingerprint instead. Caps and unit stamps are exactly the |
| 111 | // box-scoped rows only the snapshot can rebuild; if they did not trigger |
| 112 | // a pass, the standby would restore a registry missing them. |
| 113 | if err := reg.SetCaps("a.example.com", "", "net_bind_service"); err != nil { |
| 114 | t.Fatal(err) |
| 115 | } |
| 116 | if err := registrySnapshotPass(ctx, s, reg, "box1", at.Add(15*time.Minute), &lastFP); err != nil { |
| 117 | t.Fatal(err) |
| 118 | } |
| 119 | gens, err = s.List(ctx, snapshotGenPrefix("box1")) |
| 120 | if err != nil { |
| 121 | t.Fatal(err) |
| 122 | } |
| 123 | if len(gens) != 3 { |
| 124 | t.Fatalf("caps-only change did not snapshot: generations = %v (want 3) — a caps grant would never reach the bucket", gens) |
| 125 | } |
| 126 | |
| 127 | // Same for a console unit stamp. |
| 128 | if err := reg.SetUnit("a.example.com", "console.service"); err != nil { |
| 129 | t.Fatal(err) |
| 130 | } |
| 131 | if err := registrySnapshotPass(ctx, s, reg, "box1", at.Add(20*time.Minute), &lastFP); err != nil { |
| 132 | t.Fatal(err) |
| 133 | } |
| 134 | gens, err = s.List(ctx, snapshotGenPrefix("box1")) |
| 135 | if err != nil { |
| 136 | t.Fatal(err) |
| 137 | } |
| 138 | if len(gens) != 4 { |
| 139 | t.Fatalf("unit-only change did not snapshot: generations = %v (want 4)", gens) |
| 140 | } |
| 141 | } |
| 142 | |
| 143 | // The sidecar's fingerprint field must be the digest the pass actually gates |
| 144 | // on, or seedSnapshotFingerprint would seed lastFP with a value the next |
| 145 | // pass can never match and every restart would re-snapshot. |
| 146 | func TestSnapshotSidecarCarriesSnapshotFingerprint(t *testing.T) { |
| 147 | s := release.NewDirStore(t.TempDir()) |
| 148 | ctx := context.Background() |
| 149 | reg := snapTestRegistry(t) |
| 150 | |
| 151 | var lastFP string |
| 152 | if err := registrySnapshotPass(ctx, s, reg, "box1", time.Date(2026, 8, 12, 3, 0, 0, 0, time.UTC), &lastFP); err != nil { |
| 153 | t.Fatal(err) |
| 154 | } |
| 155 | want, err := reg.SnapshotFingerprint() |
| 156 | if err != nil { |
| 157 | t.Fatal(err) |
| 158 | } |
| 159 | metaRaw, _, err := s.Get(ctx, snapshotMetaKey("box1")) |
| 160 | if err != nil { |
| 161 | t.Fatal(err) |
| 162 | } |
| 163 | var meta snapshotMeta |
| 164 | if err := json.Unmarshal(metaRaw, &meta); err != nil { |
| 165 | t.Fatal(err) |
| 166 | } |
| 167 | if meta.Fingerprint != want { |
| 168 | t.Fatalf("sidecar fingerprint = %q, want SnapshotFingerprint %q", meta.Fingerprint, want) |
| 169 | } |
| 170 | } |
| 171 | |
| 172 | // readSnapshotMeta is box-scoped: it must report the prior writer at THIS |
| 173 | // box's own key, degrade to nil when there is no sidecar there yet, and |
| 174 | // never see another box's sidecar at all — per-box keys is exactly what |
| 175 | // makes a foreign-differently-named box's write harmless (it lands under a |
| 176 | // different key entirely, unlike the pre-multi-region shared key this |
| 177 | // guard was written against). |
| 178 | func TestReadSnapshotMetaReportsPriorWriter(t *testing.T) { |
| 179 | t.Parallel() |
| 180 | s := release.NewDirStore(t.TempDir()) |
| 181 | ctx := context.Background() |
| 182 | reg := snapTestRegistry(t) |
| 183 | at := time.Date(2026, 8, 12, 3, 0, 0, 0, time.UTC) |
| 184 | |
| 185 | if got := readSnapshotMeta(ctx, s, "dead-box"); got != nil { |
| 186 | t.Fatalf("readSnapshotMeta with no sidecar = %+v, want nil", got) |
| 187 | } |
| 188 | |
| 189 | var lastFP string |
| 190 | if err := registrySnapshotPass(ctx, s, reg, "dead-box", at, &lastFP); err != nil { |
| 191 | t.Fatal(err) |
| 192 | } |
| 193 | got := readSnapshotMeta(ctx, s, "dead-box") |
| 194 | if got == nil || got.Box != "dead-box" { |
| 195 | t.Fatalf("readSnapshotMeta = %+v, want Box \"dead-box\"", got) |
| 196 | } |
| 197 | |
| 198 | // A different box name never sees dead-box's sidecar — per-box keys |
| 199 | // isolate them, unlike the pre-multi-region shared key. |
| 200 | if got := readSnapshotMeta(ctx, s, "standby-box"); got != nil { |
| 201 | t.Fatalf("readSnapshotMeta(\"standby-box\") = %+v, want nil (no cross-box read)", got) |
| 202 | } |
| 203 | |
| 204 | // The runbook's restore path: a rebuilt standby deliberately reuses the |
| 205 | // dead box's name, so its pass writes to the SAME key. The write must |
| 206 | // still land (the pass never refuses it — a legitimate promotion looks |
| 207 | // identical from here), and readSnapshotMeta for that name reports the |
| 208 | // newest sidecar. |
| 209 | if err := reg.Upsert(registry.Route{Host: "c.example.com", Kind: "service", Addr: "10.0.0.3:3000", Status: "ready"}); err != nil { |
| 210 | t.Fatal(err) |
| 211 | } |
| 212 | if err := registrySnapshotPass(ctx, s, reg, "dead-box", at.Add(5*time.Minute), &lastFP); err != nil { |
| 213 | t.Fatal(err) |
| 214 | } |
| 215 | got = readSnapshotMeta(ctx, s, "dead-box") |
| 216 | if got == nil || !got.CreatedAt.Equal(at.Add(5*time.Minute)) { |
| 217 | t.Fatalf("after the reused-name write readSnapshotMeta = %+v, want CreatedAt %s", got, at.Add(5*time.Minute)) |
| 218 | } |
| 219 | } |
| 220 | |
| 221 | func TestSnapshotPassPrunesGenerationsBeyondKeep(t *testing.T) { |
| 222 | t.Parallel() |
| 223 | s := release.NewDirStore(t.TempDir()) |
| 224 | ctx := context.Background() |
| 225 | reg := snapTestRegistry(t) |
| 226 | at := time.Date(2026, 8, 12, 0, 0, 0, 0, time.UTC) |
| 227 | |
| 228 | var lastFP string |
| 229 | for i := 0; i < snapshotKeep+3; i++ { |
| 230 | // Change the registry each round so no pass skips. |
| 231 | if err := reg.Upsert(registry.Route{Host: "a.example.com", Kind: "service", Addr: "10.0.0.1:3000", Status: "ready", Version: string(rune('a' + i))}); err != nil { |
| 232 | t.Fatal(err) |
| 233 | } |
| 234 | if err := registrySnapshotPass(ctx, s, reg, "box1", at.Add(time.Duration(i)*5*time.Minute), &lastFP); err != nil { |
| 235 | t.Fatal(err) |
| 236 | } |
| 237 | } |
| 238 | gens, err := s.List(ctx, snapshotGenPrefix("box1")) |
| 239 | if err != nil { |
| 240 | t.Fatal(err) |
| 241 | } |
| 242 | if len(gens) != snapshotKeep { |
| 243 | t.Fatalf("kept %d generations, want %d: %v", len(gens), snapshotKeep, gens) |
| 244 | } |
| 245 | // The survivors must be the newest ones: the earliest kept stamp is round 3's. |
| 246 | if gens[0] != snapshotGenPrefix("box1")+"20260812T001500Z.db" { |
| 247 | t.Fatalf("oldest kept generation = %q, want the round-3 stamp", gens[0]) |
| 248 | } |
| 249 | } |
| 250 | |
| 251 | func TestSnapshotKeysArePerBox(t *testing.T) { |
| 252 | if got, want := snapshotDBKey("annie"), "control/registry-snapshot/annie.db"; got != want { |
| 253 | t.Fatalf("db key = %q, want %q", got, want) |
| 254 | } |
| 255 | if got, want := snapshotMetaKey("annie"), "control/registry-snapshot/annie.json"; got != want { |
| 256 | t.Fatalf("meta key = %q, want %q", got, want) |
| 257 | } |
| 258 | if got, want := snapshotGenPrefix("annie"), "control/registry-snapshots/annie/"; got != want { |
| 259 | t.Fatalf("gen prefix = %q, want %q", got, want) |
| 260 | } |
| 261 | } |
| 262 | |
| 263 | func TestTwoBoxesSnapshotWithoutColliding(t *testing.T) { |
| 264 | t.Parallel() |
| 265 | s := release.NewDirStore(t.TempDir()) |
| 266 | regA, regB := snapTestRegistry(t), snapTestRegistry(t) |
| 267 | var fpA, fpB string |
| 268 | now := time.Date(2026, 8, 20, 12, 0, 0, 0, time.UTC) |
| 269 | if err := registrySnapshotPass(context.Background(), s, regA, "annie", now, &fpA); err != nil { |
| 270 | t.Fatal(err) |
| 271 | } |
| 272 | if err := registrySnapshotPass(context.Background(), s, regB, "betty", now, &fpB); err != nil { |
| 273 | t.Fatal(err) |
| 274 | } |
| 275 | for _, key := range []string{ |
| 276 | "control/registry-snapshot/annie.db", "control/registry-snapshot/annie.json", |
| 277 | "control/registry-snapshot/betty.db", "control/registry-snapshot/betty.json", |
| 278 | } { |
| 279 | if _, _, err := s.Get(context.Background(), key); err != nil { |
| 280 | t.Fatalf("missing %s: %v", key, err) |
| 281 | } |
| 282 | } |
| 283 | // betty's pass must not have disturbed annie's generations. |
| 284 | gens, err := s.List(context.Background(), "control/registry-snapshots/annie/") |
| 285 | if err != nil || len(gens) != 1 { |
| 286 | t.Fatalf("annie generations = %v, %v; want exactly 1", gens, err) |
| 287 | } |
| 288 | |
| 289 | // Prove the prune is scoped per box, not global. Interleave four more |
| 290 | // fingerprint-changing passes for each box (registry content varies |
| 291 | // each round so the fingerprint gate always writes). After this, each |
| 292 | // box has 5 generations of its own (1 above + 4 here) — under |
| 293 | // snapshotKeep (8), so a correctly per-box-scoped prune leaves both at |
| 294 | // exactly 5. But the AGGREGATE across both boxes is 10, which crosses |
| 295 | // snapshotKeep — so a prune that (wrongly) lists the GLOBAL |
| 296 | // "control/registry-snapshots/" prefix would prune down to 8 total, |
| 297 | // eating into one or both boxes' tails. This is exactly the mutation |
| 298 | // the reviewer flagged: snapshotGenPrefix(box) collapsed back to a |
| 299 | // shared "control/registry-snapshots/" would pass every other test in |
| 300 | // this file (each of which uses only one box) but must fail here. |
| 301 | for i := 0; i < 4; i++ { |
| 302 | at := now.Add(time.Duration(i+1) * time.Minute) |
| 303 | if err := regA.Upsert(registry.Route{Host: fmt.Sprintf("annie-%d.example.com", i), Kind: "service", Addr: "10.0.1.1:3000", Status: "ready"}); err != nil { |
| 304 | t.Fatal(err) |
| 305 | } |
| 306 | if err := registrySnapshotPass(context.Background(), s, regA, "annie", at, &fpA); err != nil { |
| 307 | t.Fatal(err) |
| 308 | } |
| 309 | if err := regB.Upsert(registry.Route{Host: fmt.Sprintf("betty-%d.example.com", i), Kind: "service", Addr: "10.0.2.1:3000", Status: "ready"}); err != nil { |
| 310 | t.Fatal(err) |
| 311 | } |
| 312 | if err := registrySnapshotPass(context.Background(), s, regB, "betty", at, &fpB); err != nil { |
| 313 | t.Fatal(err) |
| 314 | } |
| 315 | } |
| 316 | gensA, err := s.List(context.Background(), "control/registry-snapshots/annie/") |
| 317 | if err != nil || len(gensA) != 5 { |
| 318 | t.Fatalf("annie generations after interleaving = %v, %v; want exactly 5 (per-box prune must not see betty's writes)", gensA, err) |
| 319 | } |
| 320 | gensB, err := s.List(context.Background(), "control/registry-snapshots/betty/") |
| 321 | if err != nil || len(gensB) != 5 { |
| 322 | t.Fatalf("betty generations after interleaving = %v, %v; want exactly 5 (per-box prune must not see annie's writes)", gensB, err) |
| 323 | } |
| 324 | } |
| 325 | |
| 326 | func TestSeedSnapshotFingerprintEmptyWhenNoSidecar(t *testing.T) { |
| 327 | s := release.NewDirStore(t.TempDir()) |
| 328 | if got := seedSnapshotFingerprint(context.Background(), s, "box1"); got != "" { |
| 329 | t.Fatalf("seedSnapshotFingerprint = %q, want empty with no sidecar written yet", got) |
| 330 | } |
| 331 | } |
| 332 | |
| 333 | // TestSeedSnapshotFingerprintMatchingSidecarSkipsFirstPass proves the |
| 334 | // registrySnapshotLoop wiring this backs: reading lastFP from the bucket |
| 335 | // sidecar at loop start (rather than starting it empty in memory) makes the |
| 336 | // very first pass after a restart a no-op when the registry has not changed |
| 337 | // since the last snapshot — the crash-loop case addition 1 of the Task 2 |
| 338 | // review exists to fix. |
| 339 | func TestSeedSnapshotFingerprintMatchingSidecarSkipsFirstPass(t *testing.T) { |
| 340 | s := release.NewDirStore(t.TempDir()) |
| 341 | ctx := context.Background() |
| 342 | reg := snapTestRegistry(t) |
| 343 | at := time.Date(2026, 8, 12, 3, 0, 0, 0, time.UTC) |
| 344 | |
| 345 | // A prior process ran a pass and wrote the sidecar, then exited (or |
| 346 | // crashed) — its in-memory lastFP is gone. |
| 347 | var priorFP string |
| 348 | if err := registrySnapshotPass(ctx, s, reg, "box1", at, &priorFP); err != nil { |
| 349 | t.Fatal(err) |
| 350 | } |
| 351 | |
| 352 | // A fresh process seeds lastFP from the bucket instead of starting empty. |
| 353 | seeded := seedSnapshotFingerprint(ctx, s, "box1") |
| 354 | if seeded == "" || seeded != priorFP { |
| 355 | t.Fatalf("seeded fingerprint = %q, want %q", seeded, priorFP) |
| 356 | } |
| 357 | |
| 358 | lastFP := seeded |
| 359 | if err := registrySnapshotPass(ctx, s, reg, "box1", at.Add(5*time.Minute), &lastFP); err != nil { |
| 360 | t.Fatal(err) |
| 361 | } |
| 362 | gens, err := s.List(ctx, snapshotGenPrefix("box1")) |
| 363 | if err != nil { |
| 364 | t.Fatal(err) |
| 365 | } |
| 366 | if len(gens) != 1 { |
| 367 | t.Fatalf("seeded lastFP did not skip the unchanged registry: generations = %v (want 1, no new write)", gens) |
| 368 | } |
| 369 | } |
| 370 | |
amadan