| 1 | package provision |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "crypto/ed25519" |
| 7 | "encoding/json" |
| 8 | "errors" |
| 9 | "fmt" |
| 10 | "log" |
| 11 | "os" |
| 12 | "path/filepath" |
| 13 | "reflect" |
| 14 | "strings" |
| 15 | "testing" |
| 16 | "time" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/domains" |
| 19 | "github.com/carlosframework/platform/internal/registry" |
| 20 | "github.com/carlosframework/platform/internal/release" |
| 21 | ) |
| 22 | |
| 23 | var ctx = context.Background() |
| 24 | |
| 25 | // acctID is the id behind every test's account manifest. Its sqid is always |
| 26 | // computed with domains.Sqid, never hardcoded — the same discipline |
| 27 | // internal/domains/reconcile_test.go follows for the identical reason: a |
| 28 | // literal sqid string would silently stop agreeing with domains.Sqid the |
| 29 | // moment the alphabet or offset ever changed. |
| 30 | const acctID int64 = 2 |
| 31 | |
| 32 | func dirStore(t *testing.T) *release.DirStore { |
| 33 | t.Helper() |
| 34 | return release.NewDirStore(t.TempDir()) |
| 35 | } |
| 36 | |
| 37 | func regWith(t *testing.T, routes ...registry.Route) *registry.Registry { |
| 38 | t.Helper() |
| 39 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 40 | if err != nil { |
| 41 | t.Fatal(err) |
| 42 | } |
| 43 | for _, rt := range routes { |
| 44 | if err := reg.Upsert(rt); err != nil { |
| 45 | t.Fatal(err) |
| 46 | } |
| 47 | } |
| 48 | return reg |
| 49 | } |
| 50 | |
| 51 | func mustGet(t *testing.T, reg *registry.Registry, host string) *registry.Route { |
| 52 | t.Helper() |
| 53 | rt, err := reg.Get(host) |
| 54 | if err != nil { |
| 55 | t.Fatal(err) |
| 56 | } |
| 57 | if rt == nil { |
| 58 | t.Fatalf("route %s missing", host) |
| 59 | } |
| 60 | return rt |
| 61 | } |
| 62 | |
| 63 | func accountManifestBody(id int64, sqid, name string) []byte { |
| 64 | return []byte(`{"id":` + fmt.Sprint(id) + `,"sqid":"` + sqid + `","name":"` + name + |
| 65 | `","created_at":"2026-08-02T09:00:00Z"}`) |
| 66 | } |
| 67 | |
| 68 | // writeAccountManifest writes a fresh (create-only) account manifest and |
| 69 | // hands back its sqid — the frozen encoding of id, computed by |
| 70 | // domains.Sqid, never a literal. |
| 71 | func writeAccountManifest(t *testing.T, s release.Store, id int64, name string) string { |
| 72 | t.Helper() |
| 73 | sqid := domains.Sqid(id) |
| 74 | if _, err := s.Put(ctx, "accounts/"+sqid+"/account.json", accountManifestBody(id, sqid, name), ""); err != nil { |
| 75 | t.Fatal(err) |
| 76 | } |
| 77 | return sqid |
| 78 | } |
| 79 | |
| 80 | // putReplace overwrites an existing object (read-etag-then-put), for tests |
| 81 | // that need to corrupt or repair an object already written by |
| 82 | // writeAccountManifest's create-only Put. |
| 83 | func putReplace(t *testing.T, s release.Store, key string, body []byte) { |
| 84 | t.Helper() |
| 85 | _, etag, err := s.Get(ctx, key) |
| 86 | if err != nil { |
| 87 | t.Fatal(err) |
| 88 | } |
| 89 | if _, err := s.Put(ctx, key, body, etag); err != nil { |
| 90 | t.Fatal(err) |
| 91 | } |
| 92 | } |
| 93 | |
| 94 | // releaseKeys mints a throwaway release signing pair — the same helper |
| 95 | // internal/domains/reconcile_test.go carries, for the same reason: the |
| 96 | // production flag is signed, so a test that exercises the guard needs both |
| 97 | // halves of a real key. |
| 98 | func releaseKeys(t *testing.T) (ed25519.PrivateKey, ed25519.PublicKey) { |
| 99 | t.Helper() |
| 100 | seedB64, pubB64, err := release.GenerateKey() |
| 101 | if err != nil { |
| 102 | t.Fatal(err) |
| 103 | } |
| 104 | priv, err := release.ParseSigningKey(seedB64, "test") |
| 105 | if err != nil { |
| 106 | t.Fatal(err) |
| 107 | } |
| 108 | pub, err := release.ParsePublicKey(pubB64, "test") |
| 109 | if err != nil { |
| 110 | t.Fatal(err) |
| 111 | } |
| 112 | return priv, pub |
| 113 | } |
| 114 | |
| 115 | func mustJSON(t *testing.T, v any) []byte { |
| 116 | t.Helper() |
| 117 | b, err := json.Marshal(v) |
| 118 | if err != nil { |
| 119 | t.Fatal(err) |
| 120 | } |
| 121 | return b |
| 122 | } |
| 123 | |
| 124 | // setup writes a valid account manifest (id acctID) and that account's |
| 125 | // "tito" app instances manifest (domain go.tito.io, pool tito, seed |
| 126 | // ["seed"]) — the fixture every test case below builds records on top of. |
| 127 | func setup(t *testing.T) (*release.DirStore, string) { |
| 128 | t.Helper() |
| 129 | s := dirStore(t) |
| 130 | sqid := writeAccountManifest(t, s, acctID, "Tito") |
| 131 | if err := PutManifest(ctx, s, sqid, "tito", Manifest{ |
| 132 | Schema: 1, App: "tito", Domain: "go.tito.io", Pool: "tito", Seed: []string{"seed"}, |
| 133 | }); err != nil { |
| 134 | t.Fatal(err) |
| 135 | } |
| 136 | return s, sqid |
| 137 | } |
| 138 | |
| 139 | func acmeRecord(sqid string) Record { |
| 140 | return Record{Schema: 1, Host: "acme.go.tito.io", Slug: "acme", App: "tito", |
| 141 | Account: sqid, Pool: "tito", Region: "ie", Channel: "stable", CreatedAt: time.Now()} |
| 142 | } |
| 143 | |
| 144 | // 1. Mint. |
| 145 | func TestReconcileMints(t *testing.T) { |
| 146 | s, sqid := setup(t) |
| 147 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 148 | t.Fatal(err) |
| 149 | } |
| 150 | reg := regWith(t) |
| 151 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 152 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 153 | t.Fatal(err) |
| 154 | } |
| 155 | got := *mustGet(t, reg, "acme.go.tito.io") |
| 156 | got.CreatedAt = 0 // stamped by Upsert; not part of the contract under test |
| 157 | want := registry.Route{ |
| 158 | Host: "acme.go.tito.io", Slug: "acme", Kind: "instance", |
| 159 | Socket: "/run/carlos/acme.go.tito.io.sock", DBPath: "/data/acme.go.tito.io.db", |
| 160 | App: "tito", Account: sqid, Pool: "tito", Region: "ie", Channel: "stable", |
| 161 | Hibernate: true, Provisioned: true, Seed: `["seed"]`, Status: "ready", |
| 162 | } |
| 163 | if got != want { |
| 164 | t.Errorf("minted route = %+v, want %+v", got, want) |
| 165 | } |
| 166 | } |
| 167 | |
| 168 | // Every shape the record can now describe, minted in one pass. The four |
| 169 | // cases are the production routes the old vocabulary could not express: |
| 170 | // a console (addr-backed, unit-owned, never hibernating), a whole-app TCP |
| 171 | // tenant, a static site, and a sibling service. |
| 172 | func TestReconcileMintsEveryShape(t *testing.T) { |
| 173 | s, sqid := setup(t) |
| 174 | off := false |
| 175 | on := true |
| 176 | recs := []Record{ |
| 177 | {Schema: 1, Host: "console.go.tito.io", Slug: "console", App: "tito", Account: sqid, |
| 178 | Pool: "tito", Region: "ie", Channel: "stable", Kind: "instance", |
| 179 | Addr: "127.0.0.1:8090", Unit: "console.service", CreatedAt: time.Now()}, |
| 180 | {Schema: 1, Host: "whole.go.tito.io", Slug: "whole", App: "tito", Account: sqid, |
| 181 | Pool: "tito", Region: "ie", Channel: "stable", Addr: "127.0.0.1:9443", |
| 182 | Hibernate: &off, CreatedAt: time.Now()}, |
| 183 | {Schema: 1, Host: "site.go.tito.io", Slug: "site", App: "tito", Account: sqid, |
| 184 | Pool: "tito", Region: "ie", Channel: "stable", Kind: "static", |
| 185 | StatusNamed: &on, CreatedAt: time.Now()}, |
| 186 | {Schema: 1, Host: "svc.go.tito.io", Slug: "svc", App: "tito", Account: sqid, |
| 187 | Pool: "tito", Region: "ie", Channel: "stable", Kind: "service", |
| 188 | Addr: "127.0.0.1:7000", Environment: "staging", CreatedAt: time.Now()}, |
| 189 | {Schema: 1, Host: "ha.go.tito.io", Slug: "ha", App: "tito", Account: sqid, |
| 190 | Pool: "tito", Region: "ie", Channel: "stable", Kind: "service", |
| 191 | Addrs: []string{"127.0.0.1:7001", "127.0.0.1:7002"}, CreatedAt: time.Now()}, |
| 192 | } |
| 193 | for _, rec := range recs { |
| 194 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 195 | t.Fatal(err) |
| 196 | } |
| 197 | } |
| 198 | reg := regWith(t) |
| 199 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 200 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 201 | t.Fatal(err) |
| 202 | } |
| 203 | |
| 204 | console := *mustGet(t, reg, "console.go.tito.io") |
| 205 | console.CreatedAt = 0 |
| 206 | wantConsole := registry.Route{ |
| 207 | Host: "console.go.tito.io", Slug: "console", Kind: "instance", |
| 208 | Addr: "127.0.0.1:8090", Unit: "console.service", |
| 209 | App: "tito", Account: sqid, Pool: "tito", Region: "ie", Channel: "stable", |
| 210 | Provisioned: true, Seed: `["seed"]`, Status: "ready", |
| 211 | } |
| 212 | if console != wantConsole { |
| 213 | t.Errorf("console route = %+v, want %+v", console, wantConsole) |
| 214 | } |
| 215 | |
| 216 | whole := *mustGet(t, reg, "whole.go.tito.io") |
| 217 | if whole.Socket != "" || whole.DBPath != "" { |
| 218 | t.Errorf("an addr-backed tenant derived filesystem paths: socket %q db %q", whole.Socket, whole.DBPath) |
| 219 | } |
| 220 | if whole.Hibernate { |
| 221 | t.Error("an explicitly non-hibernating record minted a hibernating route") |
| 222 | } |
| 223 | |
| 224 | site := *mustGet(t, reg, "site.go.tito.io") |
| 225 | if site.Kind != "static" { |
| 226 | t.Errorf("static route Kind = %q", site.Kind) |
| 227 | } |
| 228 | if site.Socket != "" || site.DBPath != "" { |
| 229 | t.Errorf("a static route derived filesystem paths: socket %q db %q", site.Socket, site.DBPath) |
| 230 | } |
| 231 | if site.Hibernate { |
| 232 | t.Error("a static route was armed for hibernation — it has no process to wake") |
| 233 | } |
| 234 | if !site.StatusNamed { |
| 235 | t.Error("StatusNamed did not reach the row") |
| 236 | } |
| 237 | |
| 238 | // A sibling service, and the INSERT path for Environment. On the conflict |
| 239 | // path Upsert deliberately preserves a non-empty environment (`CASE WHEN |
| 240 | // excluded.environment = ''`), so the insert is the only place a mint's |
| 241 | // Environment can land at all — and every other assertion in this file |
| 242 | // would pass just the same if it never did. |
| 243 | svc := *mustGet(t, reg, "svc.go.tito.io") |
| 244 | svc.CreatedAt = 0 |
| 245 | wantSvc := registry.Route{ |
| 246 | Host: "svc.go.tito.io", Slug: "svc", Kind: "service", Addr: "127.0.0.1:7000", |
| 247 | App: "tito", Account: sqid, Pool: "tito", Region: "ie", Channel: "stable", |
| 248 | Environment: "staging", Provisioned: true, Seed: `["seed"]`, Status: "ready", |
| 249 | } |
| 250 | if svc != wantSvc { |
| 251 | t.Errorf("service route = %+v, want %+v", svc, wantSvc) |
| 252 | } |
| 253 | |
| 254 | // The routeFor -> reg.Upsert hand-off, pinned end-to-end through a real |
| 255 | // mint pass: a multi-addr record's Addrs (and its implied default |
| 256 | // Health) must reach the row Reconcile actually wrote, not just the |
| 257 | // value routeFor returns in isolation (TestRouteForCarriesAddrsAndHealth |
| 258 | // covers that half). This is the fix for the hazard Task 2 flagged — |
| 259 | // SetUpstreams was safe in isolation, but every mint-path Upsert would |
| 260 | // have blanked it, had routeFor not been taught to carry the columns. |
| 261 | ha := *mustGet(t, reg, "ha.go.tito.io") |
| 262 | if ha.Addrs != "127.0.0.1:7001,127.0.0.1:7002" { |
| 263 | t.Errorf("multi-addr record's Addrs did not reach the registry row: %q", ha.Addrs) |
| 264 | } |
| 265 | if ha.Health == "" { |
| 266 | t.Error("multi-addr record's default Health did not reach the registry row") |
| 267 | } |
| 268 | } |
| 269 | |
| 270 | // 2. Scope. |
| 271 | func TestReconcileScope(t *testing.T) { |
| 272 | s, sqid := setup(t) |
| 273 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 274 | t.Fatal(err) |
| 275 | } |
| 276 | reg := regWith(t) |
| 277 | |
| 278 | // Wrong region entirely. |
| 279 | if err := Reconcile(ctx, s, reg, ReconcileConfig{Region: "us", Pools: []string{"tito"}, DataDir: "/data"}); err != nil { |
| 280 | t.Fatal(err) |
| 281 | } |
| 282 | if rt, err := reg.Get("acme.go.tito.io"); err != nil || rt != nil { |
| 283 | t.Errorf("minted for a foreign region: %+v %v", rt, err) |
| 284 | } |
| 285 | |
| 286 | // Right region, but this box serves no pools (nil = the shared "" pool |
| 287 | // only) — "tito" is not among them. |
| 288 | if err := Reconcile(ctx, s, reg, ReconcileConfig{Region: "ie", DataDir: "/data"}); err != nil { |
| 289 | t.Fatal(err) |
| 290 | } |
| 291 | if rt, err := reg.Get("acme.go.tito.io"); err != nil || rt != nil { |
| 292 | t.Errorf("minted for an unserved pool: %+v %v", rt, err) |
| 293 | } |
| 294 | |
| 295 | // A record with empty pool+region mints under a zero-value cfg (the |
| 296 | // unconstrained region, the shared pool). |
| 297 | unscoped := Record{Schema: 1, Host: "shared.go.tito.io", Slug: "shared", App: "tito", |
| 298 | Account: sqid, Channel: "stable", CreatedAt: time.Now()} |
| 299 | if err := CreateRecord(ctx, s, unscoped); err != nil { |
| 300 | t.Fatal(err) |
| 301 | } |
| 302 | if err := Reconcile(ctx, s, reg, ReconcileConfig{}); err != nil { |
| 303 | t.Fatal(err) |
| 304 | } |
| 305 | mustGet(t, reg, "shared.go.tito.io") |
| 306 | } |
| 307 | |
| 308 | // 2b. Fleet placement rides the same Region filter, string-equal, unchanged. |
| 309 | // |
| 310 | // TestReconcileFleetRegionMatchesByStringEquality pins the customer-fleets |
| 311 | // plan's Task 5 Step 1: a record placed on a fleet (Region: "fleet/pi") is |
| 312 | // adopted by a reconcile pass configured for that exact fleet region, and |
| 313 | // left alone by a pass configured for a different (platform) region — |
| 314 | // reconcile.go:125's `rec.Region != cfg.Region` is plain string equality, |
| 315 | // so "fleet/pi" needs no new grammar to be recognized: it's simply another |
| 316 | // opaque region string, exactly like "ie" or "us" already are. Exercises |
| 317 | // the real Reconcile entry point (not a reimplementation of the filter) so |
| 318 | // a future refactor of the scope check can't silently narrow it. |
| 319 | func TestReconcileFleetRegionMatchesByStringEquality(t *testing.T) { |
| 320 | s, sqid := setup(t) |
| 321 | rec := Record{Schema: 1, Host: "acme.go.tito.io", Slug: "acme", App: "tito", |
| 322 | Account: sqid, Pool: "tito", Region: "fleet/pi", Channel: "stable", CreatedAt: time.Now()} |
| 323 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 324 | t.Fatal(err) |
| 325 | } |
| 326 | reg := regWith(t) |
| 327 | |
| 328 | // A pass configured for a platform region must not adopt a fleet-placed |
| 329 | // record — "eu" != "fleet/pi", same as any other region mismatch. |
| 330 | if err := Reconcile(ctx, s, reg, ReconcileConfig{Region: "eu", Pools: []string{"tito"}, DataDir: "/data"}); err != nil { |
| 331 | t.Fatal(err) |
| 332 | } |
| 333 | if rt, err := reg.Get("acme.go.tito.io"); err != nil || rt != nil { |
| 334 | t.Errorf("minted for the wrong region (eu != fleet/pi): %+v %v", rt, err) |
| 335 | } |
| 336 | |
| 337 | // The fleet's own box — configured with the exact "fleet/pi" region — |
| 338 | // adopts it with zero code change: the same Region == cfg.Region check |
| 339 | // every platform region already uses. |
| 340 | if err := Reconcile(ctx, s, reg, ReconcileConfig{Region: "fleet/pi", Pools: []string{"tito"}, DataDir: "/data"}); err != nil { |
| 341 | t.Fatal(err) |
| 342 | } |
| 343 | got := mustGet(t, reg, "acme.go.tito.io") |
| 344 | if got.Region != "fleet/pi" { |
| 345 | t.Errorf("adopted route Region = %q, want fleet/pi", got.Region) |
| 346 | } |
| 347 | } |
| 348 | |
| 349 | // 3. Idempotent. |
| 350 | func TestReconcileIdempotent(t *testing.T) { |
| 351 | s, sqid := setup(t) |
| 352 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 353 | t.Fatal(err) |
| 354 | } |
| 355 | reg := regWith(t) |
| 356 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 357 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 358 | t.Fatal(err) |
| 359 | } |
| 360 | before, err := reg.List() |
| 361 | if err != nil { |
| 362 | t.Fatal(err) |
| 363 | } |
| 364 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 365 | t.Fatal(err) |
| 366 | } |
| 367 | after, err := reg.List() |
| 368 | if err != nil { |
| 369 | t.Fatal(err) |
| 370 | } |
| 371 | if !reflect.DeepEqual(before, after) { |
| 372 | t.Errorf("second pass changed the registry:\nbefore %+v\nafter %+v", before, after) |
| 373 | } |
| 374 | } |
| 375 | |
| 376 | // 4. Converge seed. |
| 377 | func TestReconcileConvergesSeed(t *testing.T) { |
| 378 | s, sqid := setup(t) |
| 379 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 380 | t.Fatal(err) |
| 381 | } |
| 382 | reg := regWith(t) |
| 383 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 384 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 385 | t.Fatal(err) |
| 386 | } |
| 387 | before := *mustGet(t, reg, "acme.go.tito.io") |
| 388 | |
| 389 | if err := PutManifest(ctx, s, sqid, "tito", Manifest{ |
| 390 | Schema: 1, App: "tito", Domain: "go.tito.io", Pool: "tito", Seed: []string{"seed", "-v"}, |
| 391 | }); err != nil { |
| 392 | t.Fatal(err) |
| 393 | } |
| 394 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 395 | t.Fatal(err) |
| 396 | } |
| 397 | after := *mustGet(t, reg, "acme.go.tito.io") |
| 398 | if after.Seed != `["seed","-v"]` { |
| 399 | t.Errorf("seed = %q, want the converged manifest seed", after.Seed) |
| 400 | } |
| 401 | after.Seed = before.Seed |
| 402 | if after != before { |
| 403 | t.Errorf("converge touched more than Seed: before %+v after %+v", before, after) |
| 404 | } |
| 405 | } |
| 406 | |
| 407 | // Issue #130, as it happened in production on 2026-08-09: a record created |
| 408 | // on stable, deleted and recreated on edge, and the route row left on |
| 409 | // stable — a route pointing at a channel with no pointer, so every wake |
| 410 | // failed "no adoptable head" while carlos channels cheerfully showed edge |
| 411 | // promoted. Reconcile created from a record and never converged to one. |
| 412 | func TestReconcileConvergesChannel(t *testing.T) { |
| 413 | s, sqid := setup(t) |
| 414 | rec := acmeRecord(sqid) |
| 415 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 416 | t.Fatal(err) |
| 417 | } |
| 418 | reg := regWith(t) |
| 419 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 420 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 421 | t.Fatal(err) |
| 422 | } |
| 423 | before := *mustGet(t, reg, "acme.go.tito.io") |
| 424 | |
| 425 | rec.Channel = "edge" |
| 426 | putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec)) |
| 427 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 428 | t.Fatal(err) |
| 429 | } |
| 430 | after := *mustGet(t, reg, "acme.go.tito.io") |
| 431 | if after.Channel != "edge" { |
| 432 | t.Errorf("channel = %q, want edge — the record moved and the row did not", after.Channel) |
| 433 | } |
| 434 | after.Channel = before.Channel |
| 435 | if after != before { |
| 436 | t.Errorf("converge touched more than Channel: before %+v after %+v", before, after) |
| 437 | } |
| 438 | |
| 439 | // The fleet runs on canary/* channels today (pinned in |
| 440 | // internal/release/laddercheck_test.go); a converge that could not carry |
| 441 | // one would stop deploys. |
| 442 | rec.Channel = "canary/rehearsal" |
| 443 | putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec)) |
| 444 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 445 | t.Fatal(err) |
| 446 | } |
| 447 | if got := mustGet(t, reg, "acme.go.tito.io").Channel; got != "canary/rehearsal" { |
| 448 | t.Errorf("channel = %q, want canary/rehearsal", got) |
| 449 | } |
| 450 | } |
| 451 | |
| 452 | // The converge guard is load-bearing: an empty Channel means a malformed |
| 453 | // record, not "clear the channel" — an empty channel would pin the route to |
| 454 | // nothing, exactly the #130 failure mode. A route already provisioned on a |
| 455 | // real channel must keep it when its record goes channel-less, not have it |
| 456 | // blanked. |
| 457 | func TestReconcileConvergeGuardEmptyChannelLeavesRouteAlone(t *testing.T) { |
| 458 | s, sqid := setup(t) |
| 459 | rec := acmeRecord(sqid) |
| 460 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 461 | t.Fatal(err) |
| 462 | } |
| 463 | reg := regWith(t) |
| 464 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 465 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 466 | t.Fatal(err) |
| 467 | } |
| 468 | before := *mustGet(t, reg, "acme.go.tito.io") |
| 469 | if before.Channel != "stable" { |
| 470 | t.Fatalf("fixture's route didn't channel as expected: %+v", before) |
| 471 | } |
| 472 | |
| 473 | rec.Channel = "" |
| 474 | putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec)) |
| 475 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 476 | t.Fatal(err) |
| 477 | } |
| 478 | after := *mustGet(t, reg, "acme.go.tito.io") |
| 479 | if after != before { |
| 480 | t.Errorf("channel-less record changed the row: before %+v after %+v", before, after) |
| 481 | } |
| 482 | } |
| 483 | |
| 484 | func TestReconcileConvergesEnvironment(t *testing.T) { |
| 485 | s, sqid := setup(t) |
| 486 | rec := acmeRecord(sqid) |
| 487 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 488 | t.Fatal(err) |
| 489 | } |
| 490 | reg := regWith(t) |
| 491 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 492 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 493 | t.Fatal(err) |
| 494 | } |
| 495 | |
| 496 | rec.Environment = "staging" |
| 497 | putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec)) |
| 498 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 499 | t.Fatal(err) |
| 500 | } |
| 501 | if got := mustGet(t, reg, "acme.go.tito.io").Environment; got != "staging" { |
| 502 | t.Errorf("environment = %q, want staging", got) |
| 503 | } |
| 504 | |
| 505 | // Clearing must reach the row too. Upsert deliberately PRESERVES a |
| 506 | // non-empty environment (it cannot tell "untouched" from "cleared"), so |
| 507 | // the clear has to go through the narrow setter — which is exactly why |
| 508 | // this converge does not re-Upsert. |
| 509 | rec.Environment = "" |
| 510 | putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec)) |
| 511 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 512 | t.Fatal(err) |
| 513 | } |
| 514 | if got := mustGet(t, reg, "acme.go.tito.io").Environment; got != "" { |
| 515 | t.Errorf("environment = %q, want it cleared", got) |
| 516 | } |
| 517 | } |
| 518 | |
| 519 | // TestReconcileConvergesSteering: a record's steering opt-in changing after |
| 520 | // mint converges onto the owned row within one pass, exactly as Channel and |
| 521 | // Environment do — the console's PATCH door (Task 3) writes the record, and |
| 522 | // this is the pass that has to notice. |
| 523 | func TestReconcileConvergesSteering(t *testing.T) { |
| 524 | s, sqid := setup(t) |
| 525 | rec := acmeRecord(sqid) |
| 526 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 527 | t.Fatal(err) |
| 528 | } |
| 529 | reg := regWith(t) |
| 530 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 531 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 532 | t.Fatal(err) |
| 533 | } |
| 534 | before := *mustGet(t, reg, "acme.go.tito.io") |
| 535 | if before.Steering != "" { |
| 536 | t.Fatalf("minted Steering = %q, want empty", before.Steering) |
| 537 | } |
| 538 | |
| 539 | rec.Steering = "latency" |
| 540 | putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec)) |
| 541 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 542 | t.Fatal(err) |
| 543 | } |
| 544 | after := *mustGet(t, reg, "acme.go.tito.io") |
| 545 | if after.Steering != "latency" { |
| 546 | t.Errorf("steering = %q, want latency", after.Steering) |
| 547 | } |
| 548 | // The self-review question this test also answers: an edited record |
| 549 | // converges ONLY its steering column, through SetSteering's narrow |
| 550 | // UPDATE — not a full-row Upsert that could disturb anything else. |
| 551 | after.Steering = before.Steering |
| 552 | if after != before { |
| 553 | t.Errorf("converging steering touched another column:\n before %+v\n after %+v", before, after) |
| 554 | } |
| 555 | |
| 556 | // Clearing must reach the row too — same shape as Environment's clear: |
| 557 | // Upsert never runs on the owned branch, so only the narrow setter can |
| 558 | // move this column back to "". |
| 559 | rec.Steering = "" |
| 560 | putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec)) |
| 561 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 562 | t.Fatal(err) |
| 563 | } |
| 564 | if got := mustGet(t, reg, "acme.go.tito.io").Steering; got != "" { |
| 565 | t.Errorf("steering = %q, want it cleared", got) |
| 566 | } |
| 567 | } |
| 568 | |
| 569 | func TestReconcileConvergesGate(t *testing.T) { |
| 570 | s, sqid := setup(t) |
| 571 | rec := acmeRecord(sqid) |
| 572 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 573 | t.Fatal(err) |
| 574 | } |
| 575 | reg := regWith(t) |
| 576 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 577 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 578 | t.Fatal(err) |
| 579 | } |
| 580 | before := *mustGet(t, reg, "acme.go.tito.io") |
| 581 | if before.Gate != "" { |
| 582 | t.Fatalf("minted Gate = %q, want open", before.Gate) |
| 583 | } |
| 584 | |
| 585 | // From here on the log is captured: the converge line names the STATE, |
| 586 | // never the secret. A bcrypt hash is not a password, but it is the |
| 587 | // material an offline cracker works on, and an operations log is copied, |
| 588 | // shipped and pasted far more freely than the registry row is. |
| 589 | var logged bytes.Buffer |
| 590 | log.SetOutput(&logged) |
| 591 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 592 | |
| 593 | const hash = "$2a$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy" |
| 594 | rec.Gate = hash |
| 595 | putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec)) |
| 596 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 597 | t.Fatal(err) |
| 598 | } |
| 599 | if !strings.Contains(logged.String(), "converged acme.go.tito.io gate open -> set") { |
| 600 | t.Errorf("no open -> set line logged; log was:\n%s", logged.String()) |
| 601 | } |
| 602 | if strings.Contains(logged.String(), hash) { |
| 603 | t.Errorf("the hash reached the log:\n%s", logged.String()) |
| 604 | } |
| 605 | after := *mustGet(t, reg, "acme.go.tito.io") |
| 606 | if after.Gate != hash { |
| 607 | t.Errorf("gate = %q, want the hash", after.Gate) |
| 608 | } |
| 609 | after.Gate = before.Gate |
| 610 | if after != before { |
| 611 | t.Errorf("converging gate touched another column:\n before %+v\n after %+v", before, after) |
| 612 | } |
| 613 | |
| 614 | // A record with no gate field is an OPEN host — it is a string, not a |
| 615 | // pointer: absent and "" are the same fact, and clearing must reach the |
| 616 | // row through SetGate. |
| 617 | rec.Gate = "" |
| 618 | putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec)) |
| 619 | logged.Reset() |
| 620 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 621 | t.Fatal(err) |
| 622 | } |
| 623 | if got := mustGet(t, reg, "acme.go.tito.io").Gate; got != "" { |
| 624 | t.Errorf("gate = %q, want it cleared", got) |
| 625 | } |
| 626 | if !strings.Contains(logged.String(), "converged acme.go.tito.io gate set -> open") { |
| 627 | t.Errorf("no set -> open line logged; log was:\n%s", logged.String()) |
| 628 | } |
| 629 | if strings.Contains(logged.String(), hash) { |
| 630 | t.Errorf("the hash reached the log:\n%s", logged.String()) |
| 631 | } |
| 632 | } |
| 633 | |
| 634 | // An explicit Hibernate converges; an absent one never does. Absent means |
| 635 | // "the record does not speak to this", and a pass that stamped its |
| 636 | // derivation over an operator's deliberate SetHibernate would be the silent |
| 637 | // stomp this design exists to end. |
| 638 | func TestReconcileConvergesHibernateOnlyWhenExplicit(t *testing.T) { |
| 639 | s, sqid := setup(t) |
| 640 | rec := acmeRecord(sqid) |
| 641 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 642 | t.Fatal(err) |
| 643 | } |
| 644 | reg := regWith(t) |
| 645 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 646 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 647 | t.Fatal(err) |
| 648 | } |
| 649 | if err := reg.SetHibernate("acme.go.tito.io", false); err != nil { |
| 650 | t.Fatal(err) |
| 651 | } |
| 652 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 653 | t.Fatal(err) |
| 654 | } |
| 655 | if mustGet(t, reg, "acme.go.tito.io").Hibernate { |
| 656 | t.Error("an absent Hibernate re-asserted its derivation over an operator's setting") |
| 657 | } |
| 658 | |
| 659 | on := true |
| 660 | rec.Hibernate = &on |
| 661 | putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec)) |
| 662 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 663 | t.Fatal(err) |
| 664 | } |
| 665 | if !mustGet(t, reg, "acme.go.tito.io").Hibernate { |
| 666 | t.Error("an explicit Hibernate did not converge") |
| 667 | } |
| 668 | } |
| 669 | |
| 670 | // Converging hibernation OFF drops the route's replication rung with it. |
| 671 | // This is the quietest of the three doors that flip the column — no operator |
| 672 | // types it, a record edit somewhere else does — and it is the one that made |
| 673 | // `carlos route --replication`'s rail point-in-time: the rung is read only on |
| 674 | // the activator's wake path, so the moment this pass stops a route sleeping, |
| 675 | // box-wide /etc/litestream.yml owns that database and a stored `daily` is a |
| 676 | // column `carlos routes` prints and nothing honours (discussion #10). |
| 677 | // |
| 678 | // A record cannot express a rung — it is box-scoped, like caps and tunnel — |
| 679 | // so this pass can only drop one, never restore it. That is why the drop is |
| 680 | // logged beside the flip rather than left to be discovered. |
| 681 | func TestReconcileConvergingHibernateOffClearsTheReplicationRung(t *testing.T) { |
| 682 | s, sqid := setup(t) |
| 683 | rec := acmeRecord(sqid) |
| 684 | on := true |
| 685 | rec.Hibernate = &on |
| 686 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 687 | t.Fatal(err) |
| 688 | } |
| 689 | reg := regWith(t) |
| 690 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 691 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 692 | t.Fatal(err) |
| 693 | } |
| 694 | if !mustGet(t, reg, "acme.go.tito.io").Hibernate { |
| 695 | t.Fatal("setup: the minted route does not hibernate") |
| 696 | } |
| 697 | // The operator's own write, on the box, through the only door there is. |
| 698 | if err := reg.SetReplication("acme.go.tito.io", registry.ReplicationDaily); err != nil { |
| 699 | t.Fatal(err) |
| 700 | } |
| 701 | |
| 702 | off := false |
| 703 | rec.Hibernate = &off |
| 704 | putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec)) |
| 705 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 706 | t.Fatal(err) |
| 707 | } |
| 708 | rt := mustGet(t, reg, "acme.go.tito.io") |
| 709 | if rt.Hibernate { |
| 710 | t.Fatal("setup: hibernate did not converge off") |
| 711 | } |
| 712 | if rt.Replication != "" { |
| 713 | t.Errorf("replication = %q after the record stopped the route hibernating — the column now names an interval nothing reads", rt.Replication) |
| 714 | } |
| 715 | } |
| 716 | |
| 717 | // nestRow moves a minted flat row onto the nested layout the way the layout |
| 718 | // pass's compare-and-swap does, so a test can meet a nested row without the |
| 719 | // pass itself. |
| 720 | func nestRow(t *testing.T, reg *registry.Registry, dataDir, host string) { |
| 721 | t.Helper() |
| 722 | if err := reg.SetInstancePaths(host, DefaultDBPath(dataDir, host), DefaultSocket(host), |
| 723 | NestedDBPath(dataDir, host), NestedSocket(host)); err != nil { |
| 724 | t.Fatal(err) |
| 725 | } |
| 726 | } |
| 727 | |
| 728 | // A record that stops a nested row hibernating is not applied: the row would |
| 729 | // leave the activator's management with its files under the instance tree, |
| 730 | // and the revert pass only quiesces rows the activator manages. |
| 731 | func TestReconcileKeepsANestedRowFlagged(t *testing.T) { |
| 732 | s, sqid := setup(t) |
| 733 | rec := acmeRecord(sqid) |
| 734 | on := true |
| 735 | rec.Hibernate = &on |
| 736 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 737 | t.Fatal(err) |
| 738 | } |
| 739 | reg := regWith(t) |
| 740 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 741 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 742 | t.Fatal(err) |
| 743 | } |
| 744 | nestRow(t, reg, cfg.DataDir, rec.Host) |
| 745 | |
| 746 | var logged bytes.Buffer |
| 747 | log.SetOutput(&logged) |
| 748 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 749 | |
| 750 | off := false |
| 751 | rec.Hibernate = &off |
| 752 | putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec)) |
| 753 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 754 | t.Fatal(err) |
| 755 | } |
| 756 | if !mustGet(t, reg, rec.Host).Hibernate { |
| 757 | t.Fatal("the record cleared hibernate on a nested row") |
| 758 | } |
| 759 | want := "provision: acme.go.tito.io: record asks to clear hibernate but the row's paths are nested; revert the layout first" |
| 760 | if !strings.Contains(logged.String(), want) { |
| 761 | t.Errorf("no refusal logged; want %q in:\n%s", want, logged.String()) |
| 762 | } |
| 763 | if strings.Contains(logged.String(), "converged acme.go.tito.io hibernate") { |
| 764 | t.Errorf("a converge was logged over the refusal:\n%s", logged.String()) |
| 765 | } |
| 766 | |
| 767 | // A flat row still converges off: the refusal is by path shape. |
| 768 | flat := acmeRecord(sqid) |
| 769 | flat.Host, flat.Slug = "flat.go.tito.io", "flat" |
| 770 | flat.Hibernate = &on |
| 771 | if err := CreateRecord(ctx, s, flat); err != nil { |
| 772 | t.Fatal(err) |
| 773 | } |
| 774 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 775 | t.Fatal(err) |
| 776 | } |
| 777 | flat.Hibernate = &off |
| 778 | putReplace(t, s, RecordKey(sqid, "tito", flat.Host), mustJSON(t, flat)) |
| 779 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 780 | t.Fatal(err) |
| 781 | } |
| 782 | if mustGet(t, reg, flat.Host).Hibernate { |
| 783 | t.Error("a flat row's hibernate did not converge off") |
| 784 | } |
| 785 | } |
| 786 | |
| 787 | // A flat row with a layout journal is mid-move, and clearing its flag |
| 788 | // would strand it the same way, so the record is not applied and the same |
| 789 | // approved line is logged. |
| 790 | func TestReconcileKeepsAFlatRowFlaggedWhileAJournalExists(t *testing.T) { |
| 791 | s, sqid := setup(t) |
| 792 | rec := acmeRecord(sqid) |
| 793 | on := true |
| 794 | rec.Hibernate = &on |
| 795 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 796 | t.Fatal(err) |
| 797 | } |
| 798 | reg := regWith(t) |
| 799 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: t.TempDir()} |
| 800 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 801 | t.Fatal(err) |
| 802 | } |
| 803 | journal := LayoutJournalPath(cfg.DataDir, rec.Host) |
| 804 | if err := os.MkdirAll(filepath.Dir(journal), 0o700); err != nil { |
| 805 | t.Fatal(err) |
| 806 | } |
| 807 | if err := os.WriteFile(journal, []byte("{}"), 0o600); err != nil { |
| 808 | t.Fatal(err) |
| 809 | } |
| 810 | |
| 811 | var logged bytes.Buffer |
| 812 | log.SetOutput(&logged) |
| 813 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 814 | |
| 815 | off := false |
| 816 | rec.Hibernate = &off |
| 817 | putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec)) |
| 818 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 819 | t.Fatal(err) |
| 820 | } |
| 821 | if NestedPaths(cfg.DataDir, *mustGet(t, reg, rec.Host)) { |
| 822 | t.Fatal("setup: the row is nested; this test is about a flat row") |
| 823 | } |
| 824 | if !mustGet(t, reg, rec.Host).Hibernate { |
| 825 | t.Fatal("the record cleared hibernate on a row with a layout journal") |
| 826 | } |
| 827 | want := "provision: acme.go.tito.io: record asks to clear hibernate but the row's paths are nested; revert the layout first" |
| 828 | if !strings.Contains(logged.String(), want) { |
| 829 | t.Errorf("no refusal logged; want %q in:\n%s", want, logged.String()) |
| 830 | } |
| 831 | } |
| 832 | |
| 833 | // A nested row's paths differ from the record's flat derivation by design, |
| 834 | // so the drift log names the layout rather than a shape mismatch that |
| 835 | // points the operator at `carlos remove`. |
| 836 | func TestReconcileNestedRowIsNotShapeDrift(t *testing.T) { |
| 837 | s, sqid := setup(t) |
| 838 | rec := acmeRecord(sqid) |
| 839 | on := true |
| 840 | rec.Hibernate = &on |
| 841 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 842 | t.Fatal(err) |
| 843 | } |
| 844 | reg := regWith(t) |
| 845 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 846 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 847 | t.Fatal(err) |
| 848 | } |
| 849 | nestRow(t, reg, cfg.DataDir, rec.Host) |
| 850 | |
| 851 | var logged bytes.Buffer |
| 852 | log.SetOutput(&logged) |
| 853 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 854 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 855 | t.Fatal(err) |
| 856 | } |
| 857 | want := fmt.Sprintf("provision: acme.go.tito.io: row is on the nested layout (db %q socket %q); the record's shared paths are not drift", |
| 858 | NestedDBPath(cfg.DataDir, rec.Host), NestedSocket(rec.Host)) |
| 859 | if !strings.Contains(logged.String(), want) { |
| 860 | t.Errorf("want %q in:\n%s", want, logged.String()) |
| 861 | } |
| 862 | if strings.Contains(logged.String(), "different route SHAPE") { |
| 863 | t.Errorf("a nested row was reported as shape drift:\n%s", logged.String()) |
| 864 | } |
| 865 | } |
| 866 | |
| 867 | // StatusNamed is converged, so it carries Hibernate's tri-state for |
| 868 | // Hibernate's reason: explicit true reaches the row, explicit false CLEARS |
| 869 | // it, and absent is not an answer at all. |
| 870 | func TestReconcileConvergesStatusNamedWhenExplicit(t *testing.T) { |
| 871 | s, sqid := setup(t) |
| 872 | rec := acmeRecord(sqid) |
| 873 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 874 | t.Fatal(err) |
| 875 | } |
| 876 | reg := regWith(t) |
| 877 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 878 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 879 | t.Fatal(err) |
| 880 | } |
| 881 | if mustGet(t, reg, "acme.go.tito.io").StatusNamed { |
| 882 | t.Fatal("a record with no status_named minted an opted-in route") |
| 883 | } |
| 884 | |
| 885 | on := true |
| 886 | rec.StatusNamed = &on |
| 887 | putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec)) |
| 888 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 889 | t.Fatal(err) |
| 890 | } |
| 891 | if !mustGet(t, reg, "acme.go.tito.io").StatusNamed { |
| 892 | t.Error("an explicit status_named:true did not converge onto the row") |
| 893 | } |
| 894 | |
| 895 | off := false |
| 896 | rec.StatusNamed = &off |
| 897 | putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec)) |
| 898 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 899 | t.Fatal(err) |
| 900 | } |
| 901 | if mustGet(t, reg, "acme.go.tito.io").StatusNamed { |
| 902 | t.Error("an explicit status_named:false did not clear the row's opt-in") |
| 903 | } |
| 904 | } |
| 905 | |
| 906 | // The whole reason StatusNamed is a pointer: an operator's `carlos add |
| 907 | // -status-named` opt-in survives every pass of a record that never mentions |
| 908 | // the field. A plain bool would read absent as an explicit false and quietly |
| 909 | // pull the app's name off the public status page. |
| 910 | func TestReconcileAbsentStatusNamedDoesNotStompOptIn(t *testing.T) { |
| 911 | s, sqid := setup(t) |
| 912 | rec := acmeRecord(sqid) |
| 913 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 914 | t.Fatal(err) |
| 915 | } |
| 916 | reg := regWith(t) |
| 917 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 918 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 919 | t.Fatal(err) |
| 920 | } |
| 921 | if err := reg.SetStatusNamed("acme.go.tito.io", true); err != nil { |
| 922 | t.Fatal(err) |
| 923 | } |
| 924 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 925 | t.Fatal(err) |
| 926 | } |
| 927 | if !mustGet(t, reg, "acme.go.tito.io").StatusNamed { |
| 928 | t.Error("an absent StatusNamed stamped its default over an operator's opt-in") |
| 929 | } |
| 930 | } |
| 931 | |
| 932 | // haRecord is the multi-upstream fixture: a service route fronting two |
| 933 | // processes, the shape the HA design (issue #152) added Addrs/Health for. |
| 934 | func haRecord(sqid string) Record { |
| 935 | return Record{Schema: 1, Host: "ha.go.tito.io", Slug: "ha", App: "tito", |
| 936 | Account: sqid, Pool: "tito", Region: "ie", Channel: "stable", Kind: "service", |
| 937 | Addrs: []string{"127.0.0.1:7001", "127.0.0.1:7002"}, CreatedAt: time.Now()} |
| 938 | } |
| 939 | |
| 940 | // TestReconcileConvergesUpstreams: a record whose upstream set (or probe |
| 941 | // config) changes AFTER the route was minted converges onto the owned row |
| 942 | // within one pass, exactly as Channel and Environment do. |
| 943 | // |
| 944 | // This is the drift Task 7's review found: registry.SetUpstreams existed with |
| 945 | // no non-test caller and the owned-row branch compared Kind/Addr/Unit/Socket/ |
| 946 | // DBPath only, so a record that gained, lost or reordered an upstream kept |
| 947 | // serving the set it was minted with forever — silently, since the drift log |
| 948 | // never looked at those columns either. The console's PATCH door writes |
| 949 | // exactly this record field, so without this converge that door would move |
| 950 | // the bucket and nothing else. |
| 951 | func TestReconcileConvergesUpstreams(t *testing.T) { |
| 952 | s, sqid := setup(t) |
| 953 | rec := haRecord(sqid) |
| 954 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 955 | t.Fatal(err) |
| 956 | } |
| 957 | reg := regWith(t) |
| 958 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 959 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 960 | t.Fatal(err) |
| 961 | } |
| 962 | before := *mustGet(t, reg, "ha.go.tito.io") |
| 963 | if before.Addrs != "127.0.0.1:7001,127.0.0.1:7002" { |
| 964 | t.Fatalf("minted Addrs = %q, want the pair", before.Addrs) |
| 965 | } |
| 966 | |
| 967 | // One upstream replaced: the row must follow the record. |
| 968 | rec.Addrs = []string{"127.0.0.1:7001", "127.0.0.1:7003"} |
| 969 | putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec)) |
| 970 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 971 | t.Fatal(err) |
| 972 | } |
| 973 | after := *mustGet(t, reg, "ha.go.tito.io") |
| 974 | if after.Addrs != "127.0.0.1:7001,127.0.0.1:7003" { |
| 975 | t.Errorf("Addrs = %q, want the record's new set — the record moved and the row did not", after.Addrs) |
| 976 | } |
| 977 | after.Addrs = before.Addrs |
| 978 | if after != before { |
| 979 | t.Errorf("the upstream converge touched more than Addrs: before %+v after %+v", before, after) |
| 980 | } |
| 981 | |
| 982 | // ORDER is shape-significant (shapeMismatch compares the joined column |
| 983 | // verbatim), so a reordered record is a converge, not a no-op. |
| 984 | rec.Addrs = []string{"127.0.0.1:7003", "127.0.0.1:7001"} |
| 985 | putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec)) |
| 986 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 987 | t.Fatal(err) |
| 988 | } |
| 989 | if got := mustGet(t, reg, "ha.go.tito.io").Addrs; got != "127.0.0.1:7003,127.0.0.1:7001" { |
| 990 | t.Errorf("Addrs = %q, want the record's order", got) |
| 991 | } |
| 992 | |
| 993 | // The probe config converges through the same setter, and lands |
| 994 | // fully-defaulted — an un-defaulted interval_ms of 0 reaching the edge |
| 995 | // would be a tight-loop probe against a customer upstream. |
| 996 | rec.Health = &Health{Path: "/healthz"} |
| 997 | putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec)) |
| 998 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 999 | t.Fatal(err) |
| 1000 | } |
| 1001 | var probe Health |
| 1002 | if err := json.Unmarshal([]byte(mustGet(t, reg, "ha.go.tito.io").Health), &probe); err != nil { |
| 1003 | t.Fatalf("row Health is not probe JSON: %v", err) |
| 1004 | } |
| 1005 | if probe.Path != "/healthz" || probe.IntervalMS != 2000 || probe.Status != 200 { |
| 1006 | t.Errorf("converged probe = %+v, want /healthz with the platform defaults filled in", probe) |
| 1007 | } |
| 1008 | |
| 1009 | // A pass over an unchanged record writes nothing further: converge is |
| 1010 | // idempotent, not a write every tick. |
| 1011 | steady := *mustGet(t, reg, "ha.go.tito.io") |
| 1012 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1013 | t.Fatal(err) |
| 1014 | } |
| 1015 | if again := *mustGet(t, reg, "ha.go.tito.io"); again != steady { |
| 1016 | t.Errorf("a second pass over an unchanged record moved the row: %+v -> %+v", steady, again) |
| 1017 | } |
| 1018 | } |
| 1019 | |
| 1020 | // TestReconcileConvergesLegacyAddrToAddrs is the migration this whole feature |
| 1021 | // exists to make possible: a tenant that has been serving on ONE addr since |
| 1022 | // before Addrs existed (kass, keymail, the consoles) gains a second upstream |
| 1023 | // through the console's PATCH door, which rewrites the record from Addr to |
| 1024 | // Addrs. The row must end up agreeing with the record on ALL THREE upstream |
| 1025 | // columns. |
| 1026 | // |
| 1027 | // Found in review, 2026-08-12: converging addrs while leaving the row's addr |
| 1028 | // standing cost three separate things — the pass logged "record describes a |
| 1029 | // different route SHAPE ... carlos remove" every tick forever, the row could |
| 1030 | // never be re-adopted (shapeMismatch compares addr), and clearing the set |
| 1031 | // later would resurrect the decommissioned upstream through AddrList's fold. |
| 1032 | func TestReconcileConvergesLegacyAddrToAddrs(t *testing.T) { |
| 1033 | s, sqid := setup(t) |
| 1034 | rec := Record{Schema: 1, Host: "whole.go.tito.io", Slug: "whole", App: "tito", Account: sqid, |
| 1035 | Pool: "tito", Region: "ie", Channel: "stable", Kind: "service", |
| 1036 | Addr: "127.0.0.1:9443", CreatedAt: time.Now()} |
| 1037 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 1038 | t.Fatal(err) |
| 1039 | } |
| 1040 | reg := regWith(t) |
| 1041 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 1042 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1043 | t.Fatal(err) |
| 1044 | } |
| 1045 | if got := mustGet(t, reg, "whole.go.tito.io"); got.Addr != "127.0.0.1:9443" || got.Addrs != "" { |
| 1046 | t.Fatalf("legacy mint = addr %q addrs %q, want the single addr alone", got.Addr, got.Addrs) |
| 1047 | } |
| 1048 | |
| 1049 | // What the PATCH door writes: the set stated whole, the legacy spelling |
| 1050 | // retired. |
| 1051 | rec.Addr = "" |
| 1052 | rec.Addrs = []string{"127.0.0.1:9443", "127.0.0.1:9444"} |
| 1053 | putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec)) |
| 1054 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1055 | t.Fatal(err) |
| 1056 | } |
| 1057 | after := *mustGet(t, reg, "whole.go.tito.io") |
| 1058 | if after.Addr != "" { |
| 1059 | t.Errorf("row Addr = %q after the migration, want it retired — AddrList would fold it back in", after.Addr) |
| 1060 | } |
| 1061 | if after.Addrs != "127.0.0.1:9443,127.0.0.1:9444" { |
| 1062 | t.Errorf("row Addrs = %q, want the record's set", after.Addrs) |
| 1063 | } |
| 1064 | if after.Health == "" { |
| 1065 | t.Error("a multi-upstream row converged without a probe — the edge cannot tell which upstream is dark") |
| 1066 | } |
| 1067 | if list := after.AddrList(); len(list) != 2 { |
| 1068 | t.Errorf("AddrList = %v, want exactly the two upstreams (a stale addr folded back in is live traffic to a dead backend)", list) |
| 1069 | } |
| 1070 | |
| 1071 | // The row now IS the route the record describes: no drift, and adoptable |
| 1072 | // (shapeMismatch is the adoption gate AND the drift test — the two failures |
| 1073 | // the un-cleared addr caused, asserted at their source). |
| 1074 | seed := seedJSON(&Manifest{Seed: []string{"seed"}}) // setup()'s manifest |
| 1075 | if reason := shapeMismatch(after, RouteFor(rec, cfg.DataDir, seed, false, nil)); reason != "" { |
| 1076 | t.Errorf("converged row still diverges from its record (%s) — the pass would log this every tick and refuse re-adoption", reason) |
| 1077 | } |
| 1078 | |
| 1079 | // Clearing the set on a route that no longer has a legacy addr to fall |
| 1080 | // back on: the columns empty out together, and nothing resurrects. |
| 1081 | rec.Addrs = nil |
| 1082 | rec.Kind = "instance" // a service with no upstream is refused by ValidateShape |
| 1083 | putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec)) |
| 1084 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1085 | t.Fatal(err) |
| 1086 | } |
| 1087 | cleared := *mustGet(t, reg, "whole.go.tito.io") |
| 1088 | if cleared.Addr != "" || cleared.Addrs != "" || cleared.Health != "" { |
| 1089 | t.Errorf("cleared row = addr %q addrs %q health %q, want all three empty", cleared.Addr, cleared.Addrs, cleared.Health) |
| 1090 | } |
| 1091 | if cleared.AddrList() != nil { |
| 1092 | t.Errorf("AddrList = %v after clearing, want nil — a resurrected upstream is traffic to a decommissioned backend", cleared.AddrList()) |
| 1093 | } |
| 1094 | } |
| 1095 | |
| 1096 | // TestReconcileLeavesLegacySingleAddrUpstreamsAlone is the additivity guard |
| 1097 | // on the converge above: every record in a production bucket today is |
| 1098 | // single-Addr or socket-backed, carries no addrs and no probe, and must |
| 1099 | // converge to the same empty columns it has always had — never to a |
| 1100 | // one-element Addrs the fold-back (AddrsOrDefault) would produce. |
| 1101 | func TestReconcileLeavesLegacySingleAddrUpstreamsAlone(t *testing.T) { |
| 1102 | s, sqid := setup(t) |
| 1103 | for _, rec := range []Record{ |
| 1104 | acmeRecord(sqid), // socket-backed instance, no addr at all |
| 1105 | {Schema: 1, Host: "whole.go.tito.io", Slug: "whole", App: "tito", Account: sqid, |
| 1106 | Pool: "tito", Region: "ie", Channel: "stable", Kind: "service", |
| 1107 | Addr: "127.0.0.1:9443", CreatedAt: time.Now()}, |
| 1108 | } { |
| 1109 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 1110 | t.Fatal(err) |
| 1111 | } |
| 1112 | } |
| 1113 | reg := regWith(t) |
| 1114 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 1115 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1116 | t.Fatal(err) |
| 1117 | } |
| 1118 | before := map[string]registry.Route{} |
| 1119 | for _, h := range []string{"acme.go.tito.io", "whole.go.tito.io"} { |
| 1120 | before[h] = *mustGet(t, reg, h) |
| 1121 | if before[h].Addrs != "" || before[h].Health != "" { |
| 1122 | t.Fatalf("%s minted with upstream columns set: %+v", h, before[h]) |
| 1123 | } |
| 1124 | } |
| 1125 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1126 | t.Fatal(err) |
| 1127 | } |
| 1128 | for h, want := range before { |
| 1129 | if got := *mustGet(t, reg, h); got != want { |
| 1130 | t.Errorf("%s moved on a converge pass: %+v -> %+v", h, want, got) |
| 1131 | } |
| 1132 | } |
| 1133 | } |
| 1134 | |
| 1135 | // 5. Conflict. |
| 1136 | func TestReconcileLeavesConflictAlone(t *testing.T) { |
| 1137 | s, sqid := setup(t) |
| 1138 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 1139 | t.Fatal(err) |
| 1140 | } |
| 1141 | handAdded := registry.Route{Host: "acme.go.tito.io", Kind: "instance", App: "other", |
| 1142 | Status: "ready", Addr: "127.0.0.1:9000"} |
| 1143 | reg := regWith(t, handAdded) |
| 1144 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 1145 | before := *mustGet(t, reg, "acme.go.tito.io") |
| 1146 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1147 | t.Fatal(err) |
| 1148 | } |
| 1149 | after := *mustGet(t, reg, "acme.go.tito.io") |
| 1150 | if before != after { |
| 1151 | t.Errorf("conflicting hand-added route changed: before %+v after %+v", before, after) |
| 1152 | } |
| 1153 | if n, err := reg.List(); err != nil || len(n) != 1 { |
| 1154 | t.Errorf("pass minted something alongside the conflict: %+v %v", n, err) |
| 1155 | } |
| 1156 | } |
| 1157 | |
| 1158 | // 6. Remove. |
| 1159 | func TestReconcileRemoves(t *testing.T) { |
| 1160 | s, sqid := setup(t) |
| 1161 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 1162 | t.Fatal(err) |
| 1163 | } |
| 1164 | handAdded := registry.Route{Host: "neighbor.go.tito.io", Kind: "instance", App: "other", |
| 1165 | Status: "ready", Addr: "127.0.0.1:9001"} |
| 1166 | foreign := registry.Route{Host: "us.go.tito.io", Kind: "instance", App: "tito", |
| 1167 | Status: "ready", Provisioned: true, Region: "us", Pool: "tito", Account: sqid} |
| 1168 | reg := regWith(t, handAdded, foreign) |
| 1169 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 1170 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1171 | t.Fatal(err) |
| 1172 | } |
| 1173 | mustGet(t, reg, "acme.go.tito.io") // minted first |
| 1174 | |
| 1175 | if err := DeleteRecord(ctx, s, sqid, "tito", "acme.go.tito.io"); err != nil { |
| 1176 | t.Fatal(err) |
| 1177 | } |
| 1178 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1179 | t.Fatal(err) |
| 1180 | } |
| 1181 | if rt, err := reg.Get("acme.go.tito.io"); err != nil || rt != nil { |
| 1182 | t.Errorf("retired route survived: %+v %v", rt, err) |
| 1183 | } |
| 1184 | mustGet(t, reg, "neighbor.go.tito.io") // hand-added, never provisioned: survives |
| 1185 | mustGet(t, reg, "us.go.tito.io") // provisioned but out of this box's region: survives |
| 1186 | } |
| 1187 | |
| 1188 | // 7. Fail-safe. |
| 1189 | func TestReconcileFailSafeOnBrokenAccountManifest(t *testing.T) { |
| 1190 | s, sqid := setup(t) |
| 1191 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 1192 | t.Fatal(err) |
| 1193 | } |
| 1194 | reg := regWith(t) |
| 1195 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 1196 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1197 | t.Fatal(err) |
| 1198 | } |
| 1199 | mustGet(t, reg, "acme.go.tito.io") // minted while the manifest was good |
| 1200 | |
| 1201 | putReplace(t, s, "accounts/"+sqid+"/account.json", []byte("not json")) |
| 1202 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1203 | t.Fatal(err) |
| 1204 | } |
| 1205 | // Mints nothing new for the now-unreadable account, but the existing |
| 1206 | // provisioned route must NOT be retired: readFailed disables removal for |
| 1207 | // the whole pass, even though the record has (from this pass's point of |
| 1208 | // view, with the account unknown) effectively vanished from the diff set. |
| 1209 | mustGet(t, reg, "acme.go.tito.io") |
| 1210 | |
| 1211 | putReplace(t, s, "accounts/"+sqid+"/account.json", accountManifestBody(acctID, sqid, "Tito")) |
| 1212 | if err := DeleteRecord(ctx, s, sqid, "tito", "acme.go.tito.io"); err != nil { |
| 1213 | t.Fatal(err) |
| 1214 | } |
| 1215 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1216 | t.Fatal(err) |
| 1217 | } |
| 1218 | if rt, err := reg.Get("acme.go.tito.io"); err != nil || rt != nil { |
| 1219 | t.Errorf("route survived after the manifest was fixed and the record removed: %+v %v", rt, err) |
| 1220 | } |
| 1221 | } |
| 1222 | |
| 1223 | // A manifest READ FAILURE (as opposed to "no manifest") must not wipe an |
| 1224 | // already-provisioned route's seed: seedFor's "" fallback is only safe to |
| 1225 | // hand a brand-new mint (nothing existing to protect), never the converge |
| 1226 | // branch. This pins the self-review fix in reconcile.go's seedFor/seedIsKnown |
| 1227 | // split — the first passing implementation called SetSeed(host, "") here, |
| 1228 | // which would erase a working seed on a transient blip. |
| 1229 | // |
| 1230 | // It also confirms readFailed from a manifest read failure disables removal |
| 1231 | // for the WHOLE pass, not only the account-listing step the brief's step 1 |
| 1232 | // names explicitly: a second, in-scope provisioned route with no matching |
| 1233 | // record this pass (the removal phase's only candidate) must survive too. |
| 1234 | func TestReconcileWithholdsSeedOnManifestReadFailure(t *testing.T) { |
| 1235 | s, sqid := setup(t) |
| 1236 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 1237 | t.Fatal(err) |
| 1238 | } |
| 1239 | reg := regWith(t) |
| 1240 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 1241 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1242 | t.Fatal(err) |
| 1243 | } |
| 1244 | before := *mustGet(t, reg, "acme.go.tito.io") |
| 1245 | if before.Seed != `["seed"]` { |
| 1246 | t.Fatalf("fixture's route didn't seed as expected: %+v", before) |
| 1247 | } |
| 1248 | |
| 1249 | // A second provisioned, in-scope route with no matching record at all — |
| 1250 | // the removal phase's only candidate this pass. |
| 1251 | if err := reg.Upsert(registry.Route{Host: "ghost.go.tito.io", Kind: "instance", App: "tito", |
| 1252 | Status: "ready", Provisioned: true, Region: "ie", Pool: "tito", Account: sqid}); err != nil { |
| 1253 | t.Fatal(err) |
| 1254 | } |
| 1255 | |
| 1256 | // Break the app's instances.json: GetManifest now fails non-ENOENT. |
| 1257 | putReplace(t, s, ManifestKey(sqid, "tito"), []byte("not json")) |
| 1258 | |
| 1259 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1260 | t.Fatal(err) |
| 1261 | } |
| 1262 | |
| 1263 | after := *mustGet(t, reg, "acme.go.tito.io") |
| 1264 | if after.Seed != before.Seed { |
| 1265 | t.Errorf("manifest read failure wiped an existing route's seed: before %q after %q", before.Seed, after.Seed) |
| 1266 | } |
| 1267 | after.Seed = before.Seed |
| 1268 | if after != before { |
| 1269 | t.Errorf("manifest read failure touched more than Seed: before %+v after %+v", before, after) |
| 1270 | } |
| 1271 | |
| 1272 | mustGet(t, reg, "ghost.go.tito.io") // removal skipped for the whole pass |
| 1273 | } |
| 1274 | |
| 1275 | // 8. Malformed record. |
| 1276 | func TestReconcileSkipsMalformedRecord(t *testing.T) { |
| 1277 | s, sqid := setup(t) |
| 1278 | body, err := json.Marshal(Record{Schema: 1, Host: "wrong-host.go.tito.io", Slug: "wrong", |
| 1279 | App: "tito", Account: sqid, Pool: "tito", Region: "ie", Channel: "stable", CreatedAt: time.Now()}) |
| 1280 | if err != nil { |
| 1281 | t.Fatal(err) |
| 1282 | } |
| 1283 | // Written at the key for "acme.go.tito.io" but the body claims a |
| 1284 | // different host: the record disagrees with its own path. |
| 1285 | if _, err := s.Put(ctx, RecordKey(sqid, "tito", "acme.go.tito.io"), body, ""); err != nil { |
| 1286 | t.Fatal(err) |
| 1287 | } |
| 1288 | reg := regWith(t) |
| 1289 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 1290 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1291 | t.Fatal(err) |
| 1292 | } |
| 1293 | if rt, err := reg.Get("acme.go.tito.io"); err != nil || rt != nil { |
| 1294 | t.Errorf("minted from a malformed record's key host: %+v %v", rt, err) |
| 1295 | } |
| 1296 | if rt, err := reg.Get("wrong-host.go.tito.io"); err != nil || rt != nil { |
| 1297 | t.Errorf("minted from a malformed record's body host: %+v %v", rt, err) |
| 1298 | } |
| 1299 | } |
| 1300 | |
| 1301 | // The record is the authorization, and this pass builds a socket path and a |
| 1302 | // database path out of it — so the host-shape invariant the console's create |
| 1303 | // door enforces has to hold at THIS door too, not by trusting the other one. |
| 1304 | // A record written straight into the bucket (or by a future/older writer) |
| 1305 | // whose host is not a hostname must not become a route: nothing downstream |
| 1306 | // re-checks it. Skipped loudly, and protected from removal for the same |
| 1307 | // reason a corrupted body is — an unusable record is not a retired one. |
| 1308 | func TestReconcileSkipsRecordWithInvalidHostShape(t *testing.T) { |
| 1309 | s, sqid := setup(t) |
| 1310 | const bad = "acme$evil.go.tito.io" |
| 1311 | body, err := json.Marshal(Record{Schema: 1, Host: bad, Slug: "acme$evil", App: "tito", |
| 1312 | Account: sqid, Pool: "tito", Region: "ie", Channel: "stable", CreatedAt: time.Now()}) |
| 1313 | if err != nil { |
| 1314 | t.Fatal(err) |
| 1315 | } |
| 1316 | // Key and body agree — the only thing wrong here is the host itself. |
| 1317 | if _, err := s.Put(ctx, RecordKey(sqid, "tito", bad), body, ""); err != nil { |
| 1318 | t.Fatal(err) |
| 1319 | } |
| 1320 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 1321 | t.Fatal(err) |
| 1322 | } |
| 1323 | reg := regWith(t) |
| 1324 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 1325 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1326 | t.Fatal(err) |
| 1327 | } |
| 1328 | if rt, err := reg.Get(bad); err != nil || rt != nil { |
| 1329 | t.Errorf("minted a route from a record whose host is not a hostname: %+v %v", rt, err) |
| 1330 | } |
| 1331 | // The good neighbour in the same pass is untouched by the skip. |
| 1332 | mustGet(t, reg, "acme.go.tito.io") |
| 1333 | |
| 1334 | // And a route that somehow already carries that host is not treated as |
| 1335 | // retired: the record is there, it just cannot be used. |
| 1336 | if err := reg.Upsert(registry.Route{Host: bad, Kind: "instance", App: "tito", |
| 1337 | Status: "ready", Provisioned: true, Region: "ie", Pool: "tito", Account: sqid}); err != nil { |
| 1338 | t.Fatal(err) |
| 1339 | } |
| 1340 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1341 | t.Fatal(err) |
| 1342 | } |
| 1343 | mustGet(t, reg, bad) |
| 1344 | } |
| 1345 | |
| 1346 | // ValidateShape is a safety net, not just the console door's policy check: |
| 1347 | // a record can reach the bucket by paths other than the console (an older |
| 1348 | // or future writer, or direct bucket access), and this pass must not |
| 1349 | // materialize one whose Kind/Addr/Unit/Environment combination is |
| 1350 | // malformed. Skipped loudly, and protected from removal for the same |
| 1351 | // reason a corrupted body or an invalid host shape is — an unusable record |
| 1352 | // is not a retired one. |
| 1353 | func TestReconcileSkipsRecordWithInvalidShape(t *testing.T) { |
| 1354 | s, sqid := setup(t) |
| 1355 | // kind "service" with no addr: ValidateShape refuses this combination |
| 1356 | // (the platform only proxies to a service, so it must name its |
| 1357 | // upstream), but it agrees with its own key path and its host shape is |
| 1358 | // fine — the only thing wrong is the Kind/Addr combination itself. |
| 1359 | bad := Record{Schema: 1, Host: "bad-service.go.tito.io", Slug: "bad-service", App: "tito", |
| 1360 | Account: sqid, Pool: "tito", Region: "ie", Channel: "stable", Kind: "service", |
| 1361 | CreatedAt: time.Now()} |
| 1362 | body, err := json.Marshal(bad) |
| 1363 | if err != nil { |
| 1364 | t.Fatal(err) |
| 1365 | } |
| 1366 | if _, err := s.Put(ctx, RecordKey(sqid, "tito", bad.Host), body, ""); err != nil { |
| 1367 | t.Fatal(err) |
| 1368 | } |
| 1369 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 1370 | t.Fatal(err) |
| 1371 | } |
| 1372 | reg := regWith(t) |
| 1373 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 1374 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1375 | t.Fatal(err) |
| 1376 | } |
| 1377 | if rt, err := reg.Get(bad.Host); err != nil || rt != nil { |
| 1378 | t.Errorf("minted a route from a shape-invalid record: %+v %v", rt, err) |
| 1379 | } |
| 1380 | // The good neighbour in the same pass is untouched by the skip. |
| 1381 | mustGet(t, reg, "acme.go.tito.io") |
| 1382 | |
| 1383 | // And a route that somehow already carries that host is not treated as |
| 1384 | // retired: the record is there, it just cannot be used. |
| 1385 | if err := reg.Upsert(registry.Route{Host: bad.Host, Kind: "instance", App: "tito", |
| 1386 | Status: "ready", Provisioned: true, Region: "ie", Pool: "tito", Account: sqid}); err != nil { |
| 1387 | t.Fatal(err) |
| 1388 | } |
| 1389 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1390 | t.Fatal(err) |
| 1391 | } |
| 1392 | mustGet(t, reg, bad.Host) |
| 1393 | } |
| 1394 | |
| 1395 | // ValidateShape deliberately does not inspect Channel (it validates shape |
| 1396 | // fields, Channel is not one), but a channel-less record must not become a |
| 1397 | // route pinned to no channel — that is issue #130's failure mode (a route |
| 1398 | // whose channel resolves to nothing, waking with "no adoptable head") |
| 1399 | // arriving through the MINT door instead of the converge door Task 3 |
| 1400 | // closed. Skipped loudly at the read-path gate, and protected from removal |
| 1401 | // for the same reason a corrupted body is — an unusable record is not a |
| 1402 | // retired one. |
| 1403 | func TestReconcileSkipsRecordWithEmptyChannel(t *testing.T) { |
| 1404 | s, sqid := setup(t) |
| 1405 | bad := acmeRecord(sqid) |
| 1406 | bad.Host, bad.Slug, bad.Channel = "no-channel.go.tito.io", "no-channel", "" |
| 1407 | body, err := json.Marshal(bad) |
| 1408 | if err != nil { |
| 1409 | t.Fatal(err) |
| 1410 | } |
| 1411 | if _, err := s.Put(ctx, RecordKey(sqid, "tito", bad.Host), body, ""); err != nil { |
| 1412 | t.Fatal(err) |
| 1413 | } |
| 1414 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 1415 | t.Fatal(err) |
| 1416 | } |
| 1417 | reg := regWith(t) |
| 1418 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 1419 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1420 | t.Fatal(err) |
| 1421 | } |
| 1422 | if rt, err := reg.Get(bad.Host); err != nil || rt != nil { |
| 1423 | t.Errorf("minted a route from a channel-less record: %+v %v", rt, err) |
| 1424 | } |
| 1425 | // The good neighbour in the same pass is untouched by the skip. |
| 1426 | mustGet(t, reg, "acme.go.tito.io") |
| 1427 | |
| 1428 | // And a route that somehow already carries that host is not treated as |
| 1429 | // retired: the record is there, it just cannot be used. |
| 1430 | if err := reg.Upsert(registry.Route{Host: bad.Host, Kind: "instance", App: "tito", |
| 1431 | Status: "ready", Provisioned: true, Region: "ie", Pool: "tito", Account: sqid}); err != nil { |
| 1432 | t.Fatal(err) |
| 1433 | } |
| 1434 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1435 | t.Fatal(err) |
| 1436 | } |
| 1437 | mustGet(t, reg, bad.Host) |
| 1438 | } |
| 1439 | |
| 1440 | // CRITICAL regression (code review finding): an existing route's record |
| 1441 | // going unparseable — not gone, just unreadable this pass — must never read |
| 1442 | // as "the bucket retired it". Before the fix, json.Unmarshal failure and |
| 1443 | // path-disagreement both log-and-continue without setting readFailed and |
| 1444 | // without the host entering recs, so the removal phase deleted the route. |
| 1445 | // A single non-additive change to Record's wire shape reaching an older |
| 1446 | // binary would make EVERY existing record fail exactly this way, turning |
| 1447 | // one tick into a mass fleet retirement. The fix (protect map, keyed on the |
| 1448 | // record's KEY-derived host) is deliberately narrower than readFailed: it |
| 1449 | // holds back only the specific corrupted host, so a genuinely retired |
| 1450 | // neighbor is still removed in the very same pass — proven below. |
| 1451 | func TestReconcileProtectsRouteWithCorruptedRecordBody(t *testing.T) { |
| 1452 | s, sqid := setup(t) |
| 1453 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 1454 | t.Fatal(err) |
| 1455 | } |
| 1456 | reg := regWith(t) |
| 1457 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 1458 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1459 | t.Fatal(err) |
| 1460 | } |
| 1461 | mustGet(t, reg, "acme.go.tito.io") // minted while the record was good |
| 1462 | |
| 1463 | // Corrupt the record body IN PLACE (a replace, not a delete) — the |
| 1464 | // shape a non-additive encoding change would produce on an old binary. |
| 1465 | putReplace(t, s, RecordKey(sqid, "tito", "acme.go.tito.io"), []byte("not json")) |
| 1466 | |
| 1467 | // A second, genuinely retired route: its record is deleted outright |
| 1468 | // (not corrupted), so it must still be removed in the SAME pass — this |
| 1469 | // is what proves protect is narrow, not a readFailed-style pass-wide |
| 1470 | // veto. |
| 1471 | ghost := registry.Route{Host: "ghost.go.tito.io", Kind: "instance", App: "tito", |
| 1472 | Status: "ready", Provisioned: true, Region: "ie", Pool: "tito", Account: sqid} |
| 1473 | if err := reg.Upsert(ghost); err != nil { |
| 1474 | t.Fatal(err) |
| 1475 | } |
| 1476 | |
| 1477 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1478 | t.Fatal(err) |
| 1479 | } |
| 1480 | mustGet(t, reg, "acme.go.tito.io") // protected: an unparseable record is not "gone" |
| 1481 | if rt, err := reg.Get("ghost.go.tito.io"); err != nil || rt != nil { |
| 1482 | t.Errorf("a genuinely record-less route was not removed alongside a protected one: %+v %v", rt, err) |
| 1483 | } |
| 1484 | } |
| 1485 | |
| 1486 | // IMPORTANT regression (code review finding): two accounts holding a record |
| 1487 | // for the same host must not resolve by silent last-writer-wins map |
| 1488 | // overwrite (the losing tenant's record would vanish from the pass with no |
| 1489 | // diagnostic). First-seen (bucket listing order) wins; the loser is skipped |
| 1490 | // loudly and produces no second route, no crash. |
| 1491 | func TestReconcileDuplicateHostAcrossAccountsFirstSeenWins(t *testing.T) { |
| 1492 | s := dirStore(t) |
| 1493 | sqid1 := writeAccountManifest(t, s, 2, "Tito") |
| 1494 | sqid2 := writeAccountManifest(t, s, 3, "Other") |
| 1495 | rec1 := Record{Schema: 1, Host: "dup.go.tito.io", Slug: "dup", App: "tito", |
| 1496 | Account: sqid1, Pool: "tito", Region: "ie", Channel: "stable", CreatedAt: time.Now()} |
| 1497 | rec2 := Record{Schema: 1, Host: "dup.go.tito.io", Slug: "dup", App: "tito", |
| 1498 | Account: sqid2, Pool: "tito", Region: "ie", Channel: "stable", CreatedAt: time.Now()} |
| 1499 | if err := CreateRecord(ctx, s, rec1); err != nil { |
| 1500 | t.Fatal(err) |
| 1501 | } |
| 1502 | if err := CreateRecord(ctx, s, rec2); err != nil { |
| 1503 | t.Fatal(err) |
| 1504 | } |
| 1505 | reg := regWith(t) |
| 1506 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 1507 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1508 | t.Fatal(err) |
| 1509 | } |
| 1510 | |
| 1511 | // The winner is whichever record's KEY sorts first — that is the order |
| 1512 | // s.List("apps/") actually hands the reconcile loop, so this must be |
| 1513 | // derived, never assumed. |
| 1514 | key1, key2 := RecordKey(sqid1, "tito", rec1.Host), RecordKey(sqid2, "tito", rec2.Host) |
| 1515 | winner := sqid1 |
| 1516 | if key2 < key1 { |
| 1517 | winner = sqid2 |
| 1518 | } |
| 1519 | got := mustGet(t, reg, "dup.go.tito.io") |
| 1520 | if got.Account != winner { |
| 1521 | t.Errorf("duplicate host resolved to account %q, want first-seen %q", got.Account, winner) |
| 1522 | } |
| 1523 | if routes, err := reg.List(); err != nil || len(routes) != 1 { |
| 1524 | t.Errorf("duplicate host produced %d routes, want exactly 1: %+v %v", len(routes), routes, err) |
| 1525 | } |
| 1526 | } |
| 1527 | |
| 1528 | // Trash: an app in the trash provisions nothing. Its record may still be in |
| 1529 | // the bucket (purge is what deletes it), but minting a route for it would |
| 1530 | // fight domains.Reconcile's trash pass — which drops a trashed app's routes |
| 1531 | // every tick — forever. Restore lifts the refusal the very next pass. |
| 1532 | func TestReconcileDeclinesTrashedApp(t *testing.T) { |
| 1533 | s, sqid := setup(t) |
| 1534 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 1535 | t.Fatal(err) |
| 1536 | } |
| 1537 | if err := release.TrashApp(ctx, s, sqid, "tito", "paul@keymail.dev", time.Now()); err != nil { |
| 1538 | t.Fatal(err) |
| 1539 | } |
| 1540 | reg := regWith(t) |
| 1541 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 1542 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1543 | t.Fatal(err) |
| 1544 | } |
| 1545 | if rt, err := reg.Get("acme.go.tito.io"); err != nil || rt != nil { |
| 1546 | t.Fatalf("minted a trashed app's instance route: %+v %v", rt, err) |
| 1547 | } |
| 1548 | |
| 1549 | if err := release.RestoreApp(ctx, s, sqid, "tito"); err != nil { |
| 1550 | t.Fatal(err) |
| 1551 | } |
| 1552 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1553 | t.Fatal(err) |
| 1554 | } |
| 1555 | mustGet(t, reg, "acme.go.tito.io") |
| 1556 | } |
| 1557 | |
| 1558 | // Trash: a route whose app is in the trash is RETIRED, on every box that |
| 1559 | // serves it (#340). This is provision's job and not domains.Reconcile's, |
| 1560 | // even though domains removes the same routes for the same reason: the |
| 1561 | // domains pass runs only on the box holding the housekeeper lease, so on a |
| 1562 | // multi-box deployment every other box kept a trashed app's route forever |
| 1563 | // and went on serving it — publicly, from its own edge, with the app |
| 1564 | // deleted. Single-box deployments were correct by accident. |
| 1565 | // |
| 1566 | // The record stays in the bucket while an app is trashed, so the removal |
| 1567 | // phase's "record gone" trigger never fires for it. The trash marker is the |
| 1568 | // second trigger, asked under exactly the same three gates: Provisioned, |
| 1569 | // in scope, and nothing failed to read this pass. |
| 1570 | func TestReconcileRemovesTrashedAppsRoute(t *testing.T) { |
| 1571 | _, pub := releaseKeys(t) |
| 1572 | s, sqid := setup(t) |
| 1573 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 1574 | t.Fatal(err) |
| 1575 | } |
| 1576 | reg := regWith(t, registry.Route{ |
| 1577 | Host: "acme.go.tito.io", Slug: "acme", Kind: "instance", |
| 1578 | Socket: "/run/carlos/acme.go.tito.io.sock", DBPath: "/data/acme.go.tito.io.db", |
| 1579 | App: "tito", Account: sqid, Pool: "tito", Region: "ie", Channel: "stable", |
| 1580 | Hibernate: true, Provisioned: true, Seed: `["seed"]`, Status: "ready", |
| 1581 | }) |
| 1582 | if err := release.TrashApp(ctx, s, sqid, "tito", "paul@keymail.dev", time.Now()); err != nil { |
| 1583 | t.Fatal(err) |
| 1584 | } |
| 1585 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data", ReleasePub: pub} |
| 1586 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1587 | t.Fatal(err) |
| 1588 | } |
| 1589 | if rt, err := reg.Get("acme.go.tito.io"); err != nil || rt != nil { |
| 1590 | t.Fatalf("a trashed app's route survived: %+v %v", rt, err) |
| 1591 | } |
| 1592 | } |
| 1593 | |
| 1594 | // #340's own shape: the FAR box, the everywhere class, a static site. This |
| 1595 | // is the case that was live on flagship — three edges publishing a trashed |
| 1596 | // app's host in their edges files while the lease holder had already |
| 1597 | // dropped it, so even the lease holder served it again through the |
| 1598 | // deployment-wide host map. |
| 1599 | func TestReconcileRemovesTrashedEverywhereRouteOnAFarBox(t *testing.T) { |
| 1600 | _, pub := releaseKeys(t) |
| 1601 | s, sqid := setup(t) |
| 1602 | rec := titoRecord(sqid, "site") |
| 1603 | rec.Kind = "static" |
| 1604 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 1605 | t.Fatal(err) |
| 1606 | } |
| 1607 | reg := regWith(t) |
| 1608 | cfg := farBoxCfg() |
| 1609 | cfg.ReleasePub = pub |
| 1610 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1611 | t.Fatal(err) |
| 1612 | } |
| 1613 | mustGet(t, reg, "site.go.tito.io") |
| 1614 | |
| 1615 | if err := release.TrashApp(ctx, s, sqid, "tito", "paul@keymail.dev", time.Now()); err != nil { |
| 1616 | t.Fatal(err) |
| 1617 | } |
| 1618 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1619 | t.Fatal(err) |
| 1620 | } |
| 1621 | if rt, err := reg.Get("site.go.tito.io"); err != nil || rt != nil { |
| 1622 | t.Errorf("a far box kept serving a trashed app's static route: %+v %v", rt, err) |
| 1623 | } |
| 1624 | } |
| 1625 | |
| 1626 | // The two trash questions are NOT the same question. Declining to mint is |
| 1627 | // reversible and answers "true" to an unreadable marker on purpose; |
| 1628 | // removal is not reversible and must answer only to a marker positively |
| 1629 | // read and present. A pass that conflated them would let a corrupt object |
| 1630 | // — or a bucket writer with delete-but-not-sign — retire live routes. |
| 1631 | func TestReconcileKeepsTrashedAppsRouteWhenTheMarkerIsUnreadable(t *testing.T) { |
| 1632 | _, pub := releaseKeys(t) |
| 1633 | s, sqid := setup(t) |
| 1634 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 1635 | t.Fatal(err) |
| 1636 | } |
| 1637 | reg := regWith(t, registry.Route{ |
| 1638 | Host: "acme.go.tito.io", Slug: "acme", Kind: "instance", |
| 1639 | Socket: "/run/carlos/acme.go.tito.io.sock", DBPath: "/data/acme.go.tito.io.db", |
| 1640 | App: "tito", Account: sqid, Pool: "tito", Region: "ie", Channel: "stable", |
| 1641 | Hibernate: true, Provisioned: true, Seed: `["seed"]`, Status: "ready", |
| 1642 | }) |
| 1643 | if err := release.TrashApp(ctx, s, sqid, "tito", "paul@keymail.dev", time.Now()); err != nil { |
| 1644 | t.Fatal(err) |
| 1645 | } |
| 1646 | putReplace(t, s, release.AppTombstoneKey(sqid, "tito"), []byte("{not json")) |
| 1647 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data", ReleasePub: pub} |
| 1648 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1649 | t.Fatal(err) |
| 1650 | } |
| 1651 | mustGet(t, reg, "acme.go.tito.io") |
| 1652 | } |
| 1653 | |
| 1654 | // The production flag is why this removal lived in domains.Reconcile at |
| 1655 | // all: dropping every route of a production app is an outage, which is |
| 1656 | // exactly what the flag exists to prevent. Moving the trigger without the |
| 1657 | // guard would have been a regression, not a fix. Same refusal, same |
| 1658 | // wording, same pass-after-pass loudness as domains §1c. |
| 1659 | func TestReconcileNeverRemovesRoutesOfAProductionFlaggedTrashedApp(t *testing.T) { |
| 1660 | priv, pub := releaseKeys(t) |
| 1661 | s, sqid := setup(t) |
| 1662 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 1663 | t.Fatal(err) |
| 1664 | } |
| 1665 | reg := regWith(t, registry.Route{ |
| 1666 | Host: "acme.go.tito.io", Slug: "acme", Kind: "instance", |
| 1667 | Socket: "/run/carlos/acme.go.tito.io.sock", DBPath: "/data/acme.go.tito.io.db", |
| 1668 | App: "tito", Account: sqid, Pool: "tito", Region: "ie", Channel: "stable", |
| 1669 | Hibernate: true, Provisioned: true, Seed: `["seed"]`, Status: "ready", |
| 1670 | }) |
| 1671 | // The flag goes on FIRST: SetProduction refuses a trashed app. |
| 1672 | if err := release.SetProduction(ctx, s, priv, sqid, "tito", true, "paul@keymail.dev"); err != nil { |
| 1673 | t.Fatal(err) |
| 1674 | } |
| 1675 | if err := release.TrashApp(ctx, s, sqid, "tito", "eve@evil.example", time.Now()); err != nil { |
| 1676 | t.Fatal(err) |
| 1677 | } |
| 1678 | |
| 1679 | var logged bytes.Buffer |
| 1680 | log.SetOutput(&logged) |
| 1681 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 1682 | |
| 1683 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data", ReleasePub: pub} |
| 1684 | for pass := 1; pass <= 2; pass++ { |
| 1685 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1686 | t.Fatalf("pass %d: %v", pass, err) |
| 1687 | } |
| 1688 | mustGet(t, reg, "acme.go.tito.io") |
| 1689 | } |
| 1690 | if !strings.Contains(logged.String(), "trash marker on production app") { |
| 1691 | t.Errorf("no refusal logged; log was:\n%s", logged.String()) |
| 1692 | } |
| 1693 | } |
| 1694 | |
| 1695 | // flagUnreadableStore breaks exactly one read — the app's production flag — |
| 1696 | // the way an attacker with delete-but-not-sign, or a half-broken bucket, |
| 1697 | // would. Everything else reads normally, so the trash marker is perfectly |
| 1698 | // valid and only the flag is in doubt. |
| 1699 | type flagUnreadableStore struct { |
| 1700 | release.Store |
| 1701 | key string |
| 1702 | } |
| 1703 | |
| 1704 | func (s flagUnreadableStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 1705 | if key == s.key { |
| 1706 | return nil, "", errors.New("bucket refused the read") |
| 1707 | } |
| 1708 | return s.Store.Get(ctx, key) |
| 1709 | } |
| 1710 | |
| 1711 | // An UNREADABLE flag refuses in the same direction as an ON one, here as |
| 1712 | // everywhere else in this platform: the state an attacker can manufacture |
| 1713 | // is exactly the unreadable one. |
| 1714 | func TestReconcileDoesNotRemoveTrashedRoutesWhenTheProductionFlagIsUnreadable(t *testing.T) { |
| 1715 | _, pub := releaseKeys(t) |
| 1716 | dir, sqid := setup(t) |
| 1717 | if err := CreateRecord(ctx, dir, acmeRecord(sqid)); err != nil { |
| 1718 | t.Fatal(err) |
| 1719 | } |
| 1720 | reg := regWith(t, registry.Route{ |
| 1721 | Host: "acme.go.tito.io", Slug: "acme", Kind: "instance", |
| 1722 | Socket: "/run/carlos/acme.go.tito.io.sock", DBPath: "/data/acme.go.tito.io.db", |
| 1723 | App: "tito", Account: sqid, Pool: "tito", Region: "ie", Channel: "stable", |
| 1724 | Hibernate: true, Provisioned: true, Seed: `["seed"]`, Status: "ready", |
| 1725 | }) |
| 1726 | if err := release.TrashApp(ctx, dir, sqid, "tito", "paul@keymail.dev", time.Now()); err != nil { |
| 1727 | t.Fatal(err) |
| 1728 | } |
| 1729 | s := flagUnreadableStore{Store: dir, key: release.ProductionKey(sqid, "tito")} |
| 1730 | |
| 1731 | var logged bytes.Buffer |
| 1732 | log.SetOutput(&logged) |
| 1733 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 1734 | |
| 1735 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data", ReleasePub: pub} |
| 1736 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1737 | t.Fatal(err) |
| 1738 | } |
| 1739 | mustGet(t, reg, "acme.go.tito.io") |
| 1740 | if !strings.Contains(logged.String(), "cannot read the production flag") { |
| 1741 | t.Errorf("no refusal logged; log was:\n%s", logged.String()) |
| 1742 | } |
| 1743 | } |
| 1744 | |
| 1745 | // A KEYLESS box acts on the marker, deliberately — domains §1c's posture, |
| 1746 | // carried over verbatim so the two passes cannot disagree about the same |
| 1747 | // app. It cannot verify a flag, and the alternative — hold every trash |
| 1748 | // forever wherever there is no pubkey — would break hiding entirely on |
| 1749 | // legitimately keyless deployments, whose owners have no way to make the |
| 1750 | // guard pass. |
| 1751 | func TestReconcileRemovesTrashedAppsRouteOnAKeylessBox(t *testing.T) { |
| 1752 | s, sqid := setup(t) |
| 1753 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 1754 | t.Fatal(err) |
| 1755 | } |
| 1756 | reg := regWith(t, registry.Route{ |
| 1757 | Host: "acme.go.tito.io", Slug: "acme", Kind: "instance", |
| 1758 | Socket: "/run/carlos/acme.go.tito.io.sock", DBPath: "/data/acme.go.tito.io.db", |
| 1759 | App: "tito", Account: sqid, Pool: "tito", Region: "ie", Channel: "stable", |
| 1760 | Hibernate: true, Provisioned: true, Seed: `["seed"]`, Status: "ready", |
| 1761 | }) |
| 1762 | if err := release.TrashApp(ctx, s, sqid, "tito", "paul@keymail.dev", time.Now()); err != nil { |
| 1763 | t.Fatal(err) |
| 1764 | } |
| 1765 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} // no ReleasePub |
| 1766 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1767 | t.Fatal(err) |
| 1768 | } |
| 1769 | if rt, err := reg.Get("acme.go.tito.io"); err != nil || rt != nil { |
| 1770 | t.Fatalf("a keyless box kept a trashed app's route: %+v %v", rt, err) |
| 1771 | } |
| 1772 | } |
| 1773 | |
| 1774 | // Provisioned is the flag that grants removal authority, and the trash |
| 1775 | // trigger does not widen it: a hand-added route this pass never adopted is |
| 1776 | // not this pass's to retire, whatever the app's marker says. |
| 1777 | func TestReconcileNeverRemovesAHandAddedRouteOfATrashedApp(t *testing.T) { |
| 1778 | _, pub := releaseKeys(t) |
| 1779 | s, sqid := setup(t) |
| 1780 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 1781 | t.Fatal(err) |
| 1782 | } |
| 1783 | reg := regWith(t, registry.Route{ |
| 1784 | Host: "hand.go.tito.io", Slug: "hand", Kind: "instance", |
| 1785 | App: "tito", Account: sqid, Pool: "tito", Region: "ie", Channel: "stable", |
| 1786 | Status: "ready", // no Provisioned |
| 1787 | }) |
| 1788 | if err := release.TrashApp(ctx, s, sqid, "tito", "paul@keymail.dev", time.Now()); err != nil { |
| 1789 | t.Fatal(err) |
| 1790 | } |
| 1791 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data", ReleasePub: pub} |
| 1792 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1793 | t.Fatal(err) |
| 1794 | } |
| 1795 | mustGet(t, reg, "hand.go.tito.io") |
| 1796 | } |
| 1797 | |
| 1798 | // Trash: an unreadable or malformed marker also declines minting — not |
| 1799 | // minting is the reversible direction, and a writer who could plant garbage |
| 1800 | // here to stall minting can already retire the instance outright by |
| 1801 | // deleting its record. |
| 1802 | func TestReconcileDeclinesUnreadableTrashMarker(t *testing.T) { |
| 1803 | s, sqid := setup(t) |
| 1804 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 1805 | t.Fatal(err) |
| 1806 | } |
| 1807 | if err := release.TrashApp(ctx, s, sqid, "tito", "paul@keymail.dev", time.Now()); err != nil { |
| 1808 | t.Fatal(err) |
| 1809 | } |
| 1810 | putReplace(t, s, release.AppTombstoneKey(sqid, "tito"), []byte("{not json")) |
| 1811 | reg := regWith(t) |
| 1812 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 1813 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1814 | t.Fatal(err) |
| 1815 | } |
| 1816 | if rt, err := reg.Get("acme.go.tito.io"); err != nil || rt != nil { |
| 1817 | t.Fatalf("an unreadable trash marker still minted a route: %+v %v", rt, err) |
| 1818 | } |
| 1819 | } |
| 1820 | |
| 1821 | // Design §4.2 — the payoff. Backfilling the fleet is a pure bucket-side |
| 1822 | // write per route: describe what is already there, and the next serial bump |
| 1823 | // adopts it. No carlos add, no SSM session, no on-box verb. |
| 1824 | func TestReconcileAdoptsMatchingHandAddedRoute(t *testing.T) { |
| 1825 | s, sqid := setup(t) |
| 1826 | rec := acmeRecord(sqid) |
| 1827 | rec.Channel = "edge" |
| 1828 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 1829 | t.Fatal(err) |
| 1830 | } |
| 1831 | // The route as an operator hand-added it before records existed: the |
| 1832 | // same shape, a different channel, and no Provisioned flag. |
| 1833 | handAdded := registry.Route{ |
| 1834 | Host: "acme.go.tito.io", Slug: "acme", Kind: "instance", |
| 1835 | Socket: "/run/carlos/acme.go.tito.io.sock", DBPath: "/data/acme.go.tito.io.db", |
| 1836 | App: "tito", Account: sqid, Pool: "tito", Region: "ie", |
| 1837 | Channel: "stable", Hibernate: true, Status: "ready", |
| 1838 | } |
| 1839 | reg := regWith(t, handAdded) |
| 1840 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 1841 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1842 | t.Fatal(err) |
| 1843 | } |
| 1844 | got := *mustGet(t, reg, "acme.go.tito.io") |
| 1845 | if !got.Provisioned { |
| 1846 | t.Fatal("a matching hand-added route was not adopted") |
| 1847 | } |
| 1848 | if got.Channel != "edge" { |
| 1849 | t.Errorf("channel = %q, want edge — adoption should converge in the same pass", got.Channel) |
| 1850 | } |
| 1851 | } |
| 1852 | |
| 1853 | // The match test EXCLUDES the three columns the record is taking ownership |
| 1854 | // of. Requiring them to agree first would make adoption impossible in |
| 1855 | // exactly the case it exists for — a hand-added route on the wrong channel |
| 1856 | // is the whole reason to adopt it. |
| 1857 | func TestReconcileAdoptionIgnoresOwnedColumns(t *testing.T) { |
| 1858 | s, sqid := setup(t) |
| 1859 | rec := acmeRecord(sqid) |
| 1860 | rec.Channel = "edge" |
| 1861 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 1862 | t.Fatal(err) |
| 1863 | } |
| 1864 | handAdded := registry.Route{ |
| 1865 | Host: "acme.go.tito.io", Slug: "acme", Kind: "instance", |
| 1866 | Socket: "/run/carlos/acme.go.tito.io.sock", DBPath: "/data/acme.go.tito.io.db", |
| 1867 | App: "tito", Account: sqid, Pool: "tito", Region: "ie", |
| 1868 | Channel: "canary/rehearsal", Environment: "staging", Hibernate: false, Status: "ready", |
| 1869 | } |
| 1870 | reg := regWith(t, handAdded) |
| 1871 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 1872 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1873 | t.Fatal(err) |
| 1874 | } |
| 1875 | if !mustGet(t, reg, "acme.go.tito.io").Provisioned { |
| 1876 | t.Error("adoption refused over a Channel/Environment/Hibernate difference") |
| 1877 | } |
| 1878 | } |
| 1879 | |
| 1880 | // A shape that differs is left exactly as alone as it is today. This is the |
| 1881 | // guard that stands between "adopt on match" and reconcile's removal phase |
| 1882 | // acquiring authority over a route it does not describe. |
| 1883 | func TestReconcileRefusesToAdoptMismatchedRoute(t *testing.T) { |
| 1884 | s, sqid := setup(t) |
| 1885 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 1886 | t.Fatal(err) |
| 1887 | } |
| 1888 | mismatched := registry.Route{ |
| 1889 | Host: "acme.go.tito.io", Slug: "acme", Kind: "instance", |
| 1890 | Socket: "/run/carlos/acme.go.tito.io.sock", DBPath: "/data/acme.go.tito.io.db", |
| 1891 | App: "other", Account: sqid, Pool: "tito", Region: "ie", |
| 1892 | Channel: "stable", Hibernate: true, Status: "ready", |
| 1893 | } |
| 1894 | reg := regWith(t, mismatched) |
| 1895 | before := *mustGet(t, reg, "acme.go.tito.io") |
| 1896 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 1897 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1898 | t.Fatal(err) |
| 1899 | } |
| 1900 | after := *mustGet(t, reg, "acme.go.tito.io") |
| 1901 | if after != before { |
| 1902 | t.Errorf("a mismatched route was touched: before %+v after %+v", before, after) |
| 1903 | } |
| 1904 | if after.Provisioned { |
| 1905 | t.Error("a mismatched route was adopted — removal authority granted over a route the record does not describe") |
| 1906 | } |
| 1907 | } |
| 1908 | |
| 1909 | // A record has no field for Backing — registry.Route.Backing's own doc |
| 1910 | // comment calls it a box-scoped operator column, alongside capability |
| 1911 | // grants and tunnels — so a record can never describe a carlos-app@<host> |
| 1912 | // unit tenant at all. Even a row that otherwise matches in every column |
| 1913 | // must not be adopted: adoption would hand this pass's removal phase |
| 1914 | // authority over a row it cannot even state the shape of. |
| 1915 | func TestReconcileRefusesToAdoptBackedRoute(t *testing.T) { |
| 1916 | s, sqid := setup(t) |
| 1917 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 1918 | t.Fatal(err) |
| 1919 | } |
| 1920 | handAdded := registry.Route{ |
| 1921 | Host: "acme.go.tito.io", Slug: "acme", Kind: "instance", |
| 1922 | Socket: "/run/carlos/acme.go.tito.io.sock", DBPath: "/data/acme.go.tito.io.db", |
| 1923 | App: "tito", Account: sqid, Pool: "tito", Region: "ie", |
| 1924 | Channel: "stable", Hibernate: true, Status: "ready", |
| 1925 | Backing: registry.BackingUnit, |
| 1926 | } |
| 1927 | reg := regWith(t, handAdded) |
| 1928 | before := *mustGet(t, reg, "acme.go.tito.io") |
| 1929 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 1930 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1931 | t.Fatal(err) |
| 1932 | } |
| 1933 | after := *mustGet(t, reg, "acme.go.tito.io") |
| 1934 | if after != before { |
| 1935 | t.Errorf("a Backing-owned route was touched: before %+v after %+v", before, after) |
| 1936 | } |
| 1937 | if after.Provisioned { |
| 1938 | t.Error("a carlos-app@ unit tenant was adopted — a record cannot state its Backing") |
| 1939 | } |
| 1940 | } |
| 1941 | |
| 1942 | // Tunnel is routing-decisive and OUTRANKS the Socket/Addr a record can |
| 1943 | // state: internal/edge picks the tunnel proxy before it ever looks at |
| 1944 | // Socket (edge.go's route-selection switch), so a tunnel-backed route can |
| 1945 | // otherwise agree with a record on every other compared column while every |
| 1946 | // byte it actually serves goes down a channel the record cannot see or |
| 1947 | // reclaim (2026-08-10 review, Critical). |
| 1948 | func TestReconcileRefusesToAdoptTunnelBoundRoute(t *testing.T) { |
| 1949 | s, sqid := setup(t) |
| 1950 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 1951 | t.Fatal(err) |
| 1952 | } |
| 1953 | handAdded := registry.Route{ |
| 1954 | Host: "acme.go.tito.io", Slug: "acme", Kind: "instance", |
| 1955 | Socket: "/run/carlos/acme.go.tito.io.sock", DBPath: "/data/acme.go.tito.io.db", |
| 1956 | App: "tito", Account: sqid, Pool: "tito", Region: "ie", |
| 1957 | Channel: "stable", Hibernate: true, Status: "ready", |
| 1958 | Tunnel: "pi-fleet", |
| 1959 | } |
| 1960 | reg := regWith(t, handAdded) |
| 1961 | before := *mustGet(t, reg, "acme.go.tito.io") |
| 1962 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 1963 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1964 | t.Fatal(err) |
| 1965 | } |
| 1966 | after := *mustGet(t, reg, "acme.go.tito.io") |
| 1967 | if after != before { |
| 1968 | t.Errorf("a tunnel-bound route was touched: before %+v after %+v", before, after) |
| 1969 | } |
| 1970 | if after.Provisioned { |
| 1971 | t.Error("a tunnel-bound route was adopted — a record cannot state or reclaim its Tunnel binding") |
| 1972 | } |
| 1973 | } |
| 1974 | |
| 1975 | // Caps is the operator-granted, box-local capability-grant column, and |
| 1976 | // registry.Delete's purge of a route's grants is already a human-gated, |
| 1977 | // last-resort action elsewhere in this file (the shape-drift log). Adoption |
| 1978 | // must not hand that purge to an unattended bucket-record deletion |
| 1979 | // (2026-08-10 review, Critical). |
| 1980 | func TestReconcileRefusesToAdoptCapsBearingRoute(t *testing.T) { |
| 1981 | s, sqid := setup(t) |
| 1982 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 1983 | t.Fatal(err) |
| 1984 | } |
| 1985 | handAdded := registry.Route{ |
| 1986 | Host: "acme.go.tito.io", Slug: "acme", Kind: "instance", |
| 1987 | Socket: "/run/carlos/acme.go.tito.io.sock", DBPath: "/data/acme.go.tito.io.db", |
| 1988 | App: "tito", Account: sqid, Pool: "tito", Region: "ie", |
| 1989 | Channel: "stable", Hibernate: true, Status: "ready", |
| 1990 | Caps: "port:25", |
| 1991 | } |
| 1992 | reg := regWith(t, handAdded) |
| 1993 | before := *mustGet(t, reg, "acme.go.tito.io") |
| 1994 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 1995 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 1996 | t.Fatal(err) |
| 1997 | } |
| 1998 | after := *mustGet(t, reg, "acme.go.tito.io") |
| 1999 | if after != before { |
| 2000 | t.Errorf("a caps-bearing route was touched: before %+v after %+v", before, after) |
| 2001 | } |
| 2002 | if after.Provisioned { |
| 2003 | t.Error("a caps-bearing route was adopted — its grants would now be one record deletion from purged") |
| 2004 | } |
| 2005 | } |
| 2006 | |
| 2007 | // domains.mintedRoute builds platform routes with empty Slug/Pool/Region/ |
| 2008 | // DBPath, copying Kind/Socket/Addr from the source route — so a record |
| 2009 | // whose Slug/Pool/Region are also empty (an unconstrained-region/pool box) |
| 2010 | // can otherwise agree with a platform-minted row on every other compared |
| 2011 | // column. Adoption must not hand this pass's removal authority to a row |
| 2012 | // domains.Reconcile already owns and periodically re-Upserts/removes on its |
| 2013 | // own authority (2026-08-10 review, Important 1). |
| 2014 | func TestReconcileRefusesToAdoptPlatformRoute(t *testing.T) { |
| 2015 | s, sqid := setup(t) |
| 2016 | rec := Record{Schema: 1, Host: "acme.go.tito.io", App: "tito", Account: sqid, |
| 2017 | Channel: "stable", CreatedAt: time.Now()} |
| 2018 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 2019 | t.Fatal(err) |
| 2020 | } |
| 2021 | handAdded := registry.Route{ |
| 2022 | Host: "acme.go.tito.io", Kind: "instance", |
| 2023 | Socket: "/run/carlos/acme.go.tito.io.sock", DBPath: "/data/acme.go.tito.io.db", |
| 2024 | App: "tito", Account: sqid, Status: "ready", Platform: true, |
| 2025 | } |
| 2026 | reg := regWith(t, handAdded) |
| 2027 | before := *mustGet(t, reg, "acme.go.tito.io") |
| 2028 | cfg := ReconcileConfig{DataDir: "/data"} // unconstrained region, default pool only |
| 2029 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 2030 | t.Fatal(err) |
| 2031 | } |
| 2032 | after := *mustGet(t, reg, "acme.go.tito.io") |
| 2033 | if after != before { |
| 2034 | t.Errorf("a platform-owned route was touched: before %+v after %+v", before, after) |
| 2035 | } |
| 2036 | if after.Provisioned { |
| 2037 | t.Error("a platform-owned route was adopted — removal authority granted to a second pass") |
| 2038 | } |
| 2039 | } |
| 2040 | |
| 2041 | // Same guard, the other flag domains.Reconcile also uses for its own |
| 2042 | // removal authority (2026-08-10 review, Important 1). |
| 2043 | func TestReconcileRefusesToAdoptCanaryRoute(t *testing.T) { |
| 2044 | s, sqid := setup(t) |
| 2045 | rec := Record{Schema: 1, Host: "acme.go.tito.io", App: "tito", Account: sqid, |
| 2046 | Channel: "stable", CreatedAt: time.Now()} |
| 2047 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 2048 | t.Fatal(err) |
| 2049 | } |
| 2050 | handAdded := registry.Route{ |
| 2051 | Host: "acme.go.tito.io", Kind: "instance", |
| 2052 | Socket: "/run/carlos/acme.go.tito.io.sock", DBPath: "/data/acme.go.tito.io.db", |
| 2053 | App: "tito", Account: sqid, Status: "ready", Canary: true, |
| 2054 | } |
| 2055 | reg := regWith(t, handAdded) |
| 2056 | before := *mustGet(t, reg, "acme.go.tito.io") |
| 2057 | cfg := ReconcileConfig{DataDir: "/data"} |
| 2058 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 2059 | t.Fatal(err) |
| 2060 | } |
| 2061 | after := *mustGet(t, reg, "acme.go.tito.io") |
| 2062 | if after != before { |
| 2063 | t.Errorf("a canary-owned route was touched: before %+v after %+v", before, after) |
| 2064 | } |
| 2065 | if after.Provisioned { |
| 2066 | t.Error("a canary-owned route was adopted — removal authority granted to a second pass") |
| 2067 | } |
| 2068 | } |
| 2069 | |
| 2070 | // Nothing currently locked the derived Socket/DBPath comparison specifically |
| 2071 | // — the only prior mismatch test differed on App, so a regression that |
| 2072 | // dropped the DBPath comparison from shapeMismatch would leave the suite |
| 2073 | // green (2026-08-10 review, test gap called out alongside the Important |
| 2074 | // findings). |
| 2075 | func TestReconcileRefusesToAdoptOnDBPathMismatch(t *testing.T) { |
| 2076 | s, sqid := setup(t) |
| 2077 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 2078 | t.Fatal(err) |
| 2079 | } |
| 2080 | handAdded := registry.Route{ |
| 2081 | Host: "acme.go.tito.io", Slug: "acme", Kind: "instance", |
| 2082 | Socket: "/run/carlos/acme.go.tito.io.sock", DBPath: "/data/other.db", |
| 2083 | App: "tito", Account: sqid, Pool: "tito", Region: "ie", |
| 2084 | Channel: "stable", Hibernate: true, Status: "ready", |
| 2085 | } |
| 2086 | reg := regWith(t, handAdded) |
| 2087 | before := *mustGet(t, reg, "acme.go.tito.io") |
| 2088 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 2089 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 2090 | t.Fatal(err) |
| 2091 | } |
| 2092 | after := *mustGet(t, reg, "acme.go.tito.io") |
| 2093 | if after != before { |
| 2094 | t.Errorf("a DBPath-mismatched route was touched: before %+v after %+v", before, after) |
| 2095 | } |
| 2096 | if after.Provisioned { |
| 2097 | t.Error("a route was adopted despite a DBPath mismatch") |
| 2098 | } |
| 2099 | } |
| 2100 | |
| 2101 | // An adopted route is removable by the pass that adopted it, on the same |
| 2102 | // terms as one it minted: the record disappearing is the bucket's own |
| 2103 | // decision to retire the instance. |
| 2104 | func TestReconcileRemovesAdoptedRouteWhenRecordGoes(t *testing.T) { |
| 2105 | s, sqid := setup(t) |
| 2106 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 2107 | t.Fatal(err) |
| 2108 | } |
| 2109 | handAdded := registry.Route{ |
| 2110 | Host: "acme.go.tito.io", Slug: "acme", Kind: "instance", |
| 2111 | Socket: "/run/carlos/acme.go.tito.io.sock", DBPath: "/data/acme.go.tito.io.db", |
| 2112 | App: "tito", Account: sqid, Pool: "tito", Region: "ie", |
| 2113 | Channel: "stable", Hibernate: true, Status: "ready", |
| 2114 | } |
| 2115 | reg := regWith(t, handAdded) |
| 2116 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 2117 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 2118 | t.Fatal(err) |
| 2119 | } |
| 2120 | if err := DeleteRecord(ctx, s, sqid, "tito", "acme.go.tito.io"); err != nil { |
| 2121 | t.Fatal(err) |
| 2122 | } |
| 2123 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 2124 | t.Fatal(err) |
| 2125 | } |
| 2126 | rt, err := reg.Get("acme.go.tito.io") |
| 2127 | if err != nil { |
| 2128 | t.Fatal(err) |
| 2129 | } |
| 2130 | if rt != nil { |
| 2131 | t.Errorf("adopted route survived its record's deletion: %+v", rt) |
| 2132 | } |
| 2133 | } |
| 2134 | |
| 2135 | // A dry-run pass reports what it would do and writes nothing — the backfill |
| 2136 | // mitigation (design §4.3): a production route is adopted only after someone |
| 2137 | // has read what adopting it would mean, because adoption is what grants the |
| 2138 | // removal phase authority over that row. This exercises all four write |
| 2139 | // shapes in the same pass — mint (acme.go.tito.io has no existing route), |
| 2140 | // adopt (a hand-added route matching a record's shape but on the wrong |
| 2141 | // channel), converge (an already-provisioned route whose channel, |
| 2142 | // environment, hibernate and seed all disagree with its record), and remove |
| 2143 | // (a provisioned route whose record is gone) — because a test that only |
| 2144 | // covers minting would pass while a removal wrote through. |
| 2145 | func TestReconcileDryRunWritesNothing(t *testing.T) { |
| 2146 | s, sqid := setup(t) |
| 2147 | |
| 2148 | // Mint: no existing route for this host. |
| 2149 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 2150 | t.Fatal(err) |
| 2151 | } |
| 2152 | |
| 2153 | // Adopt: a hand-added route matching the record's shape, on a |
| 2154 | // different channel AND environment — adoption would converge both in |
| 2155 | // the same pass. Two differing columns, deliberately: a dry-run report |
| 2156 | // that only ever mentioned channel here would leave environment's |
| 2157 | // change invisible to the operator reading it (design §4.3 review). |
| 2158 | adoptRec := Record{Schema: 1, Host: "adopt.go.tito.io", Slug: "adopt", App: "tito", |
| 2159 | Account: sqid, Pool: "tito", Region: "ie", Channel: "edge", Environment: "staging", CreatedAt: time.Now()} |
| 2160 | if err := CreateRecord(ctx, s, adoptRec); err != nil { |
| 2161 | t.Fatal(err) |
| 2162 | } |
| 2163 | adoptable := registry.Route{ |
| 2164 | Host: "adopt.go.tito.io", Slug: "adopt", Kind: "instance", |
| 2165 | Socket: "/run/carlos/adopt.go.tito.io.sock", DBPath: "/data/adopt.go.tito.io.db", |
| 2166 | App: "tito", Account: sqid, Pool: "tito", Region: "ie", |
| 2167 | Channel: "stable", Hibernate: true, Status: "ready", |
| 2168 | } |
| 2169 | |
| 2170 | // Converge: already provisioned (this pass or an earlier one owns it), |
| 2171 | // but channel, environment, hibernate and seed all disagree. |
| 2172 | onHibernate := true |
| 2173 | ownedRec := Record{Schema: 1, Host: "owned.go.tito.io", Slug: "owned", App: "tito", |
| 2174 | Account: sqid, Pool: "tito", Region: "ie", Channel: "edge", Environment: "staging", |
| 2175 | Hibernate: &onHibernate, CreatedAt: time.Now()} |
| 2176 | if err := CreateRecord(ctx, s, ownedRec); err != nil { |
| 2177 | t.Fatal(err) |
| 2178 | } |
| 2179 | owned := registry.Route{ |
| 2180 | Host: "owned.go.tito.io", Slug: "owned", Kind: "instance", |
| 2181 | Socket: "/run/carlos/owned.go.tito.io.sock", DBPath: "/data/owned.go.tito.io.db", |
| 2182 | App: "tito", Account: sqid, Pool: "tito", Region: "ie", |
| 2183 | Channel: "stable", Provisioned: true, Status: "ready", |
| 2184 | } |
| 2185 | |
| 2186 | // Remove: provisioned, in scope, and its record is gone. |
| 2187 | retired := registry.Route{Host: "gone.go.tito.io", Kind: "instance", App: "tito", Account: sqid, |
| 2188 | Pool: "tito", Region: "ie", Status: "ready", Provisioned: true} |
| 2189 | |
| 2190 | reg := regWith(t, adoptable, owned, retired) |
| 2191 | before, err := reg.List() |
| 2192 | if err != nil { |
| 2193 | t.Fatal(err) |
| 2194 | } |
| 2195 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data", DryRun: true} |
| 2196 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 2197 | t.Fatal(err) |
| 2198 | } |
| 2199 | after, err := reg.List() |
| 2200 | if err != nil { |
| 2201 | t.Fatal(err) |
| 2202 | } |
| 2203 | if !reflect.DeepEqual(before, after) { |
| 2204 | t.Errorf("dry-run wrote to the registry:\nbefore %+v\nafter %+v", before, after) |
| 2205 | } |
| 2206 | if rt, err := reg.Get("acme.go.tito.io"); err != nil || rt != nil { |
| 2207 | t.Errorf("dry-run minted a route: %+v %v", rt, err) |
| 2208 | } |
| 2209 | |
| 2210 | // Positive control (reviewer-flagged): the assertion above — total |
| 2211 | // state unchanged — would also pass if this fixture simply never |
| 2212 | // exercised the mint/adopt/converge/remove branches at all, e.g. if a |
| 2213 | // future validation change caused ownedRec to be skipped and |
| 2214 | // protect-ed, silently dropping the converge branch from coverage while |
| 2215 | // the test still reported green. Re-running the identical fixture for |
| 2216 | // real proves the fixture DOES drive every branch: if state came out |
| 2217 | // unchanged here too, the dry-run assertion above would have been |
| 2218 | // vacuous, not a proof of anything. |
| 2219 | cfg.DryRun = false |
| 2220 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 2221 | t.Fatal(err) |
| 2222 | } |
| 2223 | live, err := reg.List() |
| 2224 | if err != nil { |
| 2225 | t.Fatal(err) |
| 2226 | } |
| 2227 | if reflect.DeepEqual(before, live) { |
| 2228 | t.Fatal("a real pass over the identical fixture left the registry unchanged — " + |
| 2229 | "the dry-run assertion above proves nothing if nothing was ever going to write") |
| 2230 | } |
| 2231 | } |
| 2232 | |
| 2233 | // The other side of that coin, and the one that matters more: a route no |
| 2234 | // record ever described is never removed, no matter how many passes run. |
| 2235 | // Adoption is the ONLY way this pass acquires removal authority, and this |
| 2236 | // test is what stands between a backfill and a retired production route. |
| 2237 | func TestReconcileNeverRemovesUnadoptedRoute(t *testing.T) { |
| 2238 | s, sqid := setup(t) |
| 2239 | // A record for some OTHER host, so the pass runs a full removal phase |
| 2240 | // with a non-empty record set rather than bailing early. |
| 2241 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 2242 | t.Fatal(err) |
| 2243 | } |
| 2244 | handAdded := registry.Route{ |
| 2245 | Host: "console.go.tito.io", Kind: "instance", App: "tito", Account: sqid, |
| 2246 | Pool: "tito", Region: "ie", Channel: "stable", Addr: "127.0.0.1:8090", |
| 2247 | Unit: "console.service", Status: "ready", |
| 2248 | } |
| 2249 | reg := regWith(t, handAdded) |
| 2250 | // Upsert stamps CreatedAt; read the row back so the final comparison |
| 2251 | // isn't comparing against a zero value the row never actually had. |
| 2252 | handAdded = *mustGet(t, reg, "console.go.tito.io") |
| 2253 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 2254 | for i := 0; i < 3; i++ { |
| 2255 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 2256 | t.Fatal(err) |
| 2257 | } |
| 2258 | } |
| 2259 | got, err := reg.Get("console.go.tito.io") |
| 2260 | if err != nil { |
| 2261 | t.Fatal(err) |
| 2262 | } |
| 2263 | if got == nil { |
| 2264 | t.Fatal("a hand-added route with no record was removed") |
| 2265 | } |
| 2266 | if got.Provisioned { |
| 2267 | t.Error("a hand-added route with no record was adopted") |
| 2268 | } |
| 2269 | if *got != handAdded { |
| 2270 | t.Errorf("a hand-added route with no record was modified: %+v", got) |
| 2271 | } |
| 2272 | } |
| 2273 | |
| 2274 | // TestReconcileAdoptsStaticStoryRoute tests the backfill golden path: |
| 2275 | // a declared static record adopts the story-shaped route that was |
| 2276 | // hand-added before records existed. This is issue #82 backfill path. |
| 2277 | func TestReconcileAdoptsStaticStoryRoute(t *testing.T) { |
| 2278 | s := dirStore(t) |
| 2279 | // Create a different account for story (id=3, to avoid collision with the tito account) |
| 2280 | storyAcctSqid := writeAccountManifest(t, s, 3, "Story") |
| 2281 | // Create an app manifest for story |
| 2282 | if err := PutManifest(ctx, s, storyAcctSqid, "story", Manifest{ |
| 2283 | Schema: 1, App: "story", Domain: "carlosframework.com", Seed: []string{}, |
| 2284 | }); err != nil { |
| 2285 | t.Fatal(err) |
| 2286 | } |
| 2287 | |
| 2288 | // Pre-Upsert the operator-added shape: a static route as it existed |
| 2289 | // before records were written to the bucket |
| 2290 | handAdded := registry.Route{ |
| 2291 | Host: "story.carlosframework.com", Kind: "static", |
| 2292 | App: "story", Account: storyAcctSqid, Channel: "canary/live", |
| 2293 | Status: "ready", |
| 2294 | } |
| 2295 | reg := regWith(t, handAdded) |
| 2296 | |
| 2297 | // Write the matching declared record |
| 2298 | rec := Record{ |
| 2299 | Schema: 1, Host: "story.carlosframework.com", App: "story", |
| 2300 | Account: storyAcctSqid, Channel: "canary/live", Kind: "static", |
| 2301 | CreatedAt: time.Now(), |
| 2302 | } |
| 2303 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 2304 | t.Fatal(err) |
| 2305 | } |
| 2306 | |
| 2307 | // Run Reconcile with an unconstrained region/pool (matches the static route) |
| 2308 | cfg := ReconcileConfig{DataDir: "/data"} |
| 2309 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 2310 | t.Fatal(err) |
| 2311 | } |
| 2312 | |
| 2313 | // Assert the route is now Provisioned |
| 2314 | got := *mustGet(t, reg, "story.carlosframework.com") |
| 2315 | if !got.Provisioned { |
| 2316 | t.Fatal("the matching static route was not adopted") |
| 2317 | } |
| 2318 | |
| 2319 | // Static routes should have no Socket or DBPath |
| 2320 | if got.Socket != "" || got.DBPath != "" { |
| 2321 | t.Errorf("a static route derived filesystem paths: socket %q db %q", got.Socket, got.DBPath) |
| 2322 | } |
| 2323 | |
| 2324 | // Static routes should not be hibernating |
| 2325 | if got.Hibernate { |
| 2326 | t.Error("a static route was armed for hibernation — it has no process to wake") |
| 2327 | } |
| 2328 | } |
| 2329 | |
| 2330 | // Companion negative: a static record does NOT adopt an instance route. |
| 2331 | // The kind must match for adoption to happen. |
| 2332 | func TestReconcileRefusesAdoptInstanceRouteForStaticRecord(t *testing.T) { |
| 2333 | s := dirStore(t) |
| 2334 | // Create a different account for story (id=3) |
| 2335 | storyAcctSqid := writeAccountManifest(t, s, 3, "Story") |
| 2336 | // Create an app manifest for story |
| 2337 | if err := PutManifest(ctx, s, storyAcctSqid, "story", Manifest{ |
| 2338 | Schema: 1, App: "story", Domain: "carlosframework.com", Seed: []string{}, |
| 2339 | }); err != nil { |
| 2340 | t.Fatal(err) |
| 2341 | } |
| 2342 | |
| 2343 | // Pre-Upsert a route with Kind: "instance" (not "static") |
| 2344 | handAdded := registry.Route{ |
| 2345 | Host: "story.carlosframework.com", Kind: "instance", |
| 2346 | Socket: "/run/carlos/story.carlosframework.com.sock", |
| 2347 | DBPath: "/data/story.carlosframework.com.db", |
| 2348 | App: "story", Account: storyAcctSqid, Channel: "canary/live", |
| 2349 | Status: "ready", |
| 2350 | } |
| 2351 | reg := regWith(t, handAdded) |
| 2352 | before := *mustGet(t, reg, "story.carlosframework.com") |
| 2353 | |
| 2354 | // Write the matching declared record WITH Kind: "static" |
| 2355 | rec := Record{ |
| 2356 | Schema: 1, Host: "story.carlosframework.com", App: "story", |
| 2357 | Account: storyAcctSqid, Channel: "canary/live", Kind: "static", |
| 2358 | CreatedAt: time.Now(), |
| 2359 | } |
| 2360 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 2361 | t.Fatal(err) |
| 2362 | } |
| 2363 | |
| 2364 | // Run Reconcile |
| 2365 | cfg := ReconcileConfig{DataDir: "/data"} |
| 2366 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 2367 | t.Fatal(err) |
| 2368 | } |
| 2369 | |
| 2370 | // Assert the route was NOT adopted (Provisioned still false) |
| 2371 | after := *mustGet(t, reg, "story.carlosframework.com") |
| 2372 | if after.Provisioned { |
| 2373 | t.Error("a kind-mismatched route was adopted — removal authority granted over a route the record does not describe") |
| 2374 | } |
| 2375 | |
| 2376 | // Verify the route was not modified at all |
| 2377 | if after != before { |
| 2378 | t.Errorf("a mismatched route was modified: before %+v after %+v", before, after) |
| 2379 | } |
| 2380 | } |
| 2381 | |
| 2382 | // ---- serving classes (multi-region ingress design, Part 3's table) --------- |
| 2383 | // |
| 2384 | // The same deployment seen from two PLATFORM boxes: the record's home region |
| 2385 | // ("ie") and another one ("us"), both serving the tito pool. Everything below |
| 2386 | // asks the same question of one record — "does the far box materialize it?" — |
| 2387 | // and the answer is the record's serving class. |
| 2388 | |
| 2389 | func homeBoxCfg() ReconcileConfig { |
| 2390 | return ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 2391 | } |
| 2392 | |
| 2393 | func farBoxCfg() ReconcileConfig { |
| 2394 | return ReconcileConfig{Region: "us", Pools: []string{"tito"}, DataDir: "/data"} |
| 2395 | } |
| 2396 | |
| 2397 | // titoRecord is the tito-app fixture every serving-class case below mutates: |
| 2398 | // homed in "ie", in the "tito" pool, on the stable channel. |
| 2399 | func titoRecord(sqid, host string) Record { |
| 2400 | return Record{Schema: 1, Host: host + ".go.tito.io", Slug: host, App: "tito", |
| 2401 | Account: sqid, Pool: "tito", Region: "ie", Channel: "stable", CreatedAt: time.Now()} |
| 2402 | } |
| 2403 | |
| 2404 | // A static (bake-served) record is materialized by EVERY platform box in the |
| 2405 | // deployment, not only its home: the edge resolves the app's channel pointer |
| 2406 | // out of the same bucket every box already reads, so no box needs the home box |
| 2407 | // to serve it. Spec Part 3, class "Everywhere". |
| 2408 | func TestReconcileStaticIsMaterializedOnEveryPlatformBox(t *testing.T) { |
| 2409 | s, sqid := setup(t) |
| 2410 | rec := titoRecord(sqid, "site") |
| 2411 | rec.Kind = "static" |
| 2412 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 2413 | t.Fatal(err) |
| 2414 | } |
| 2415 | reg := regWith(t) |
| 2416 | if err := Reconcile(ctx, s, reg, farBoxCfg()); err != nil { |
| 2417 | t.Fatal(err) |
| 2418 | } |
| 2419 | got := mustGet(t, reg, "site.go.tito.io") |
| 2420 | if !got.Provisioned || got.Kind != "static" { |
| 2421 | t.Fatalf("far box did not materialize the static record: %+v", *got) |
| 2422 | } |
| 2423 | // The row still carries the RECORD's region — the record is the desired |
| 2424 | // state and routeFor is its one renderer, so a far box's copy and the home |
| 2425 | // box's copy are the same route, byte for byte. |
| 2426 | if got.Region != "ie" { |
| 2427 | t.Errorf("Region = %q, want the record's own %q", got.Region, "ie") |
| 2428 | } |
| 2429 | if got.Socket != "" || got.DBPath != "" || got.Hibernate { |
| 2430 | t.Errorf("a static route grew a process shape on the far box: %+v", *got) |
| 2431 | } |
| 2432 | } |
| 2433 | |
| 2434 | // An addr route whose upstreams are all publicly dialable is materialized |
| 2435 | // everywhere too — every edge probes and dials the same public backends. Spec |
| 2436 | // Part 3, class "Everywhere, probed". |
| 2437 | func TestReconcilePublicAddrIsMaterializedOnEveryPlatformBox(t *testing.T) { |
| 2438 | s, sqid := setup(t) |
| 2439 | one := titoRecord(sqid, "pub") |
| 2440 | one.Kind = "service" |
| 2441 | one.Addr = "203.0.113.10:443" |
| 2442 | many := titoRecord(sqid, "pubs") |
| 2443 | many.Kind = "service" |
| 2444 | many.Addrs = []string{"203.0.113.11:443", "198.51.100.7:443"} |
| 2445 | for _, rec := range []Record{one, many} { |
| 2446 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 2447 | t.Fatal(err) |
| 2448 | } |
| 2449 | } |
| 2450 | reg := regWith(t) |
| 2451 | if err := Reconcile(ctx, s, reg, farBoxCfg()); err != nil { |
| 2452 | t.Fatal(err) |
| 2453 | } |
| 2454 | for _, host := range []string{"pub.go.tito.io", "pubs.go.tito.io"} { |
| 2455 | if got := mustGet(t, reg, host); !got.Provisioned { |
| 2456 | t.Errorf("%s: far box did not materialize a public-upstream record: %+v", host, *got) |
| 2457 | } |
| 2458 | } |
| 2459 | } |
| 2460 | |
| 2461 | // The home-boxed classes, unchanged: the process backing them exists on |
| 2462 | // exactly ONE box, so a far box must still refuse every one of them. This is |
| 2463 | // the refusal test the class change is only safe with. |
| 2464 | func TestReconcileHomeBoxedClassesStayHomeBoxed(t *testing.T) { |
| 2465 | s, sqid := setup(t) |
| 2466 | socketed := titoRecord(sqid, "exec") // no kind, no addr: the socket/exec shape |
| 2467 | loopback := titoRecord(sqid, "loop") |
| 2468 | loopback.Kind = "service" |
| 2469 | loopback.Addr = "127.0.0.1:8090" |
| 2470 | private := titoRecord(sqid, "priv") |
| 2471 | private.Kind = "service" |
| 2472 | private.Addrs = []string{"10.0.0.5:3000"} |
| 2473 | unitOwned := titoRecord(sqid, "unit") |
| 2474 | unitOwned.Addr = "203.0.113.20:8090" |
| 2475 | unitOwned.Unit = "console.service" |
| 2476 | named := titoRecord(sqid, "named") |
| 2477 | named.Kind = "service" |
| 2478 | named.Addr = "backend.example.com:443" |
| 2479 | for _, rec := range []Record{socketed, loopback, private, unitOwned, named} { |
| 2480 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 2481 | t.Fatal(err) |
| 2482 | } |
| 2483 | } |
| 2484 | reg := regWith(t) |
| 2485 | if err := Reconcile(ctx, s, reg, farBoxCfg()); err != nil { |
| 2486 | t.Fatal(err) |
| 2487 | } |
| 2488 | for _, host := range []string{"exec.go.tito.io", "loop.go.tito.io", "priv.go.tito.io", |
| 2489 | "unit.go.tito.io", "named.go.tito.io"} { |
| 2490 | if rt, err := reg.Get(host); err != nil || rt != nil { |
| 2491 | t.Errorf("%s: a home-boxed record materialized on a foreign box: %+v %v", host, rt, err) |
| 2492 | } |
| 2493 | } |
| 2494 | // ...and every one of them still materializes on its OWN box. |
| 2495 | if err := Reconcile(ctx, s, reg, homeBoxCfg()); err != nil { |
| 2496 | t.Fatal(err) |
| 2497 | } |
| 2498 | for _, host := range []string{"exec.go.tito.io", "loop.go.tito.io", "priv.go.tito.io", |
| 2499 | "unit.go.tito.io", "named.go.tito.io"} { |
| 2500 | mustGet(t, reg, host) |
| 2501 | } |
| 2502 | } |
| 2503 | |
| 2504 | // Pool is a TENANCY boundary, not a geography: widening the region half of the |
| 2505 | // gate must not widen the pool half. A static record in the tito pool stays |
| 2506 | // off a box that does not serve that pool, however far it travels. |
| 2507 | func TestReconcileEverywhereClassStillRespectsThePool(t *testing.T) { |
| 2508 | s, sqid := setup(t) |
| 2509 | rec := titoRecord(sqid, "site") |
| 2510 | rec.Kind = "static" |
| 2511 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 2512 | t.Fatal(err) |
| 2513 | } |
| 2514 | reg := regWith(t) |
| 2515 | // A far box serving only the shared "" pool (nil Pools), and one serving |
| 2516 | // somebody else's pool. |
| 2517 | for _, cfg := range []ReconcileConfig{ |
| 2518 | {Region: "us", DataDir: "/data"}, |
| 2519 | {Region: "us", Pools: []string{"eleven"}, DataDir: "/data"}, |
| 2520 | } { |
| 2521 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 2522 | t.Fatal(err) |
| 2523 | } |
| 2524 | if rt, err := reg.Get("site.go.tito.io"); err != nil || rt != nil { |
| 2525 | t.Fatalf("pools %v: a static record crossed a pool boundary: %+v %v", cfg.Pools, rt, err) |
| 2526 | } |
| 2527 | } |
| 2528 | } |
| 2529 | |
| 2530 | // Fleet placement is the other tenancy boundary, and it is refused from BOTH |
| 2531 | // sides: a customer fleet's static site is not served by the platform's boxes, |
| 2532 | // and a customer's fleet box does not pick up the platform's static records. |
| 2533 | // (registry.FleetPlacement is the grammar — "fleet/<name>" in the Region.) |
| 2534 | func TestReconcileFleetPlacementIsNeverEverywhere(t *testing.T) { |
| 2535 | s, sqid := setup(t) |
| 2536 | onFleet := titoRecord(sqid, "pisite") |
| 2537 | onFleet.Kind = "static" |
| 2538 | onFleet.Region = "fleet/pi" |
| 2539 | platform := titoRecord(sqid, "site") |
| 2540 | platform.Kind = "static" |
| 2541 | for _, rec := range []Record{onFleet, platform} { |
| 2542 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 2543 | t.Fatal(err) |
| 2544 | } |
| 2545 | } |
| 2546 | reg := regWith(t) |
| 2547 | |
| 2548 | // A platform box must not serve the fleet's site... |
| 2549 | if err := Reconcile(ctx, s, reg, farBoxCfg()); err != nil { |
| 2550 | t.Fatal(err) |
| 2551 | } |
| 2552 | if rt, err := reg.Get("pisite.go.tito.io"); err != nil || rt != nil { |
| 2553 | t.Errorf("a fleet-placed static record materialized on a platform box: %+v %v", rt, err) |
| 2554 | } |
| 2555 | // ...and the fleet's own box must not pick up the platform's. |
| 2556 | fleetReg := regWith(t) |
| 2557 | if err := Reconcile(ctx, s, fleetReg, ReconcileConfig{Region: "fleet/pi", Pools: []string{"tito"}, DataDir: "/data"}); err != nil { |
| 2558 | t.Fatal(err) |
| 2559 | } |
| 2560 | if rt, err := fleetReg.Get("site.go.tito.io"); err != nil || rt != nil { |
| 2561 | t.Errorf("a platform static record materialized on a customer fleet box: %+v %v", rt, err) |
| 2562 | } |
| 2563 | mustGet(t, fleetReg, "pisite.go.tito.io") // its own placement, unchanged |
| 2564 | } |
| 2565 | |
| 2566 | // The other half of adopting everywhere: retiring everywhere. A far box that |
| 2567 | // mints an everywhere-class route from a record must remove it when that |
| 2568 | // record goes, or the widened mint gate leaves an immortal route serving a |
| 2569 | // retired site on every box but its home. |
| 2570 | func TestReconcileRemovesEverywhereRouteWhenItsRecordGoes(t *testing.T) { |
| 2571 | s, sqid := setup(t) |
| 2572 | rec := titoRecord(sqid, "site") |
| 2573 | rec.Kind = "static" |
| 2574 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 2575 | t.Fatal(err) |
| 2576 | } |
| 2577 | reg := regWith(t) |
| 2578 | cfg := farBoxCfg() |
| 2579 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 2580 | t.Fatal(err) |
| 2581 | } |
| 2582 | mustGet(t, reg, "site.go.tito.io") |
| 2583 | |
| 2584 | if err := DeleteRecord(ctx, s, sqid, "tito", "site.go.tito.io"); err != nil { |
| 2585 | t.Fatal(err) |
| 2586 | } |
| 2587 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 2588 | t.Fatal(err) |
| 2589 | } |
| 2590 | if rt, err := reg.Get("site.go.tito.io"); err != nil || rt != nil { |
| 2591 | t.Errorf("a far box's everywhere-class route outlived its record: %+v %v", rt, err) |
| 2592 | } |
| 2593 | } |
| 2594 | |
| 2595 | // TestEverywhereClassAgreesAcrossRecordAndRow is the drift pin between the two |
| 2596 | // halves of the class rule: everywhereClass reads a RECORD (the mint gate) and |
| 2597 | // everywhereRoute reads a ROW (the removal gate, asked when the record is gone |
| 2598 | // and the row is all that is left). routeFor is the one renderer between them, |
| 2599 | // so the property holds by construction today — and it is exactly what a class |
| 2600 | // rule added to one function alone would break, silently, with every other test |
| 2601 | // in this file still green. That divergence is the immortality bug the pair was |
| 2602 | // written to close: a far box that mints an everywhere-class route and then |
| 2603 | // cannot see it as one serves a retired site forever. |
| 2604 | // |
| 2605 | // The fixture set is every shape the file already exercises, plus the two |
| 2606 | // directions a new rule would most plausibly be written into one side only (a |
| 2607 | // probe config, an explicitly non-hibernating addr record). |
| 2608 | func TestEverywhereClassAgreesAcrossRecordAndRow(t *testing.T) { |
| 2609 | off := false |
| 2610 | on := true |
| 2611 | base := func(host string) Record { |
| 2612 | return Record{Schema: 1, Host: host, Slug: "s", App: "tito", Account: "bac", |
| 2613 | Pool: "tito", Region: "ie", Channel: "stable"} |
| 2614 | } |
| 2615 | with := func(host string, mut func(*Record)) Record { |
| 2616 | rec := base(host) |
| 2617 | mut(&rec) |
| 2618 | return rec |
| 2619 | } |
| 2620 | fixtures := []struct { |
| 2621 | name string |
| 2622 | rec Record |
| 2623 | }{ |
| 2624 | {"socket instance (no kind, no addr)", base("exec.go.tito.io")}, |
| 2625 | {"static", with("site.go.tito.io", func(r *Record) { r.Kind = "static" })}, |
| 2626 | {"static, status-named", with("named-site.go.tito.io", func(r *Record) { r.Kind, r.StatusNamed = "static", &on })}, |
| 2627 | {"public single addr", with("pub.go.tito.io", func(r *Record) { r.Kind, r.Addr = "service", "203.0.113.10:443" })}, |
| 2628 | {"public addrs set", with("pubs.go.tito.io", func(r *Record) { |
| 2629 | r.Kind, r.Addrs = "service", []string{"203.0.113.11:443", "198.51.100.7:443"} |
| 2630 | })}, |
| 2631 | {"public addrs with an explicit probe", with("probed.go.tito.io", func(r *Record) { |
| 2632 | r.Kind, r.Addrs, r.Health = "service", []string{"203.0.113.12:443", "198.51.100.8:443"}, &Health{Path: "/healthz"} |
| 2633 | })}, |
| 2634 | {"public addr, explicitly never hibernating", with("awake.go.tito.io", func(r *Record) { |
| 2635 | r.Kind, r.Addr, r.Hibernate = "service", "203.0.113.13:443", &off |
| 2636 | })}, |
| 2637 | {"loopback addr", with("loop.go.tito.io", func(r *Record) { r.Kind, r.Addr = "service", "127.0.0.1:8090" })}, |
| 2638 | {"private addrs", with("priv.go.tito.io", func(r *Record) { r.Kind, r.Addrs = "service", []string{"10.0.0.5:3000"} })}, |
| 2639 | {"mixed public and private addrs", with("mixed.go.tito.io", func(r *Record) { |
| 2640 | r.Kind, r.Addrs = "service", []string{"203.0.113.14:443", "192.168.1.9:3000"} |
| 2641 | })}, |
| 2642 | {"name-based addr", with("named.go.tito.io", func(r *Record) { r.Kind, r.Addr = "service", "backend.example.com:443" })}, |
| 2643 | {"unit-owned public addr", with("unit.go.tito.io", func(r *Record) { |
| 2644 | r.Addr, r.Unit = "203.0.113.20:8090", "console.service" |
| 2645 | })}, |
| 2646 | {"environment-bound static", with("env-site.go.tito.io", func(r *Record) { r.Kind, r.Environment = "static", "staging" })}, |
| 2647 | {"fleet-placed static", with("pisite.go.tito.io", func(r *Record) { r.Kind, r.Region = "static", "fleet/pi" })}, |
| 2648 | } |
| 2649 | for _, f := range fixtures { |
| 2650 | t.Run(f.name, func(t *testing.T) { |
| 2651 | // Every fixture must be one the pass would actually materialize — |
| 2652 | // a shape ValidateShape refuses never reaches either gate, so a |
| 2653 | // property proved on one would prove nothing. |
| 2654 | if err := ValidateShape(f.rec); err != nil { |
| 2655 | t.Fatalf("fixture is not a shape this pass would ever see: %v", err) |
| 2656 | } |
| 2657 | fromRecord := everywhereClass(f.rec) |
| 2658 | fromRow := everywhereRoute(RouteFor(f.rec, "/data", `["seed"]`, false, nil)) |
| 2659 | if fromRecord != fromRow { |
| 2660 | t.Fatalf("class disagrees across the mint and removal gates: everywhereClass = %v, "+ |
| 2661 | "everywhereRoute(RouteFor(...)) = %v — a record this box mints must be a row it can retire", |
| 2662 | fromRecord, fromRow) |
| 2663 | } |
| 2664 | }) |
| 2665 | } |
| 2666 | } |
| 2667 | |
| 2668 | // TestRouteForCarriesAddrsAndHealth is Task 3's mint-side proof: a record |
| 2669 | // with an explicit multi-upstream Addrs set mints a route whose Addrs is the |
| 2670 | // comma-joined string and whose Health carries a fully-materialized default |
| 2671 | // probe (design §Part II). A legacy single-Addr record mints a |
| 2672 | // byte-identical route to before this field existed — routeFor only ever |
| 2673 | // reads the record's EXPLICIT Addrs, never AddrsOrDefault's Addr-folding. |
| 2674 | func TestRouteForCarriesAddrsAndHealth(t *testing.T) { |
| 2675 | rec := Record{Host: "m.example.com", App: "web", Account: "bab", Channel: "stable", |
| 2676 | Kind: "service", Addrs: []string{"10.0.0.1:3000", "10.0.0.2:3000"}} |
| 2677 | rt := RouteFor(rec, "/data", "seed", false, nil) |
| 2678 | if rt.Addrs != "10.0.0.1:3000,10.0.0.2:3000" { |
| 2679 | t.Fatalf("Addrs = %q", rt.Addrs) |
| 2680 | } |
| 2681 | if rt.Health == "" { |
| 2682 | t.Fatal("multi-addr record must mint default health JSON") |
| 2683 | } |
| 2684 | var h Health |
| 2685 | if err := json.Unmarshal([]byte(rt.Health), &h); err != nil || h.Path != "/" || h.FailAfter != 3 { |
| 2686 | t.Fatalf("health JSON = %q (%v)", rt.Health, err) |
| 2687 | } |
| 2688 | // Legacy single-addr record: byte-identical route to before this change. |
| 2689 | old := Record{Host: "s.example.com", App: "web", Account: "bab", Channel: "stable", Kind: "service", Addr: "10.0.0.9:3000"} |
| 2690 | ort := RouteFor(old, "/data", "seed", false, nil) |
| 2691 | if ort.Addrs != "" || ort.Health != "" || ort.Addr != "10.0.0.9:3000" { |
| 2692 | t.Fatalf("legacy record's route changed shape: %+v", ort) |
| 2693 | } |
| 2694 | } |
| 2695 | |
| 2696 | // TestRouteForMaterializesPartialHealthDefaults is the controller amendment |
| 2697 | // from Task 1's review: a record with EXPLICIT-but-partial probe config |
| 2698 | // (only Path set here) must still mint a route whose Health JSON carries the |
| 2699 | // platform's fully-materialized defaults for every field the record left at |
| 2700 | // zero. An un-defaulted interval_ms of 0 reaching the edge would mean a |
| 2701 | // tight-loop probe against a customer upstream. |
| 2702 | func TestRouteForMaterializesPartialHealthDefaults(t *testing.T) { |
| 2703 | rec := Record{Host: "m.example.com", App: "web", Account: "bab", Channel: "stable", |
| 2704 | Kind: "service", Addrs: []string{"10.0.0.1:3000", "10.0.0.2:3000"}, |
| 2705 | Health: &Health{Path: "/healthz"}} |
| 2706 | rt := RouteFor(rec, "/data", "seed", false, nil) |
| 2707 | var h Health |
| 2708 | if err := json.Unmarshal([]byte(rt.Health), &h); err != nil { |
| 2709 | t.Fatalf("health JSON = %q (%v)", rt.Health, err) |
| 2710 | } |
| 2711 | if h.Path != "/healthz" { |
| 2712 | t.Fatalf("explicit Path lost: %+v", h) |
| 2713 | } |
| 2714 | if h.IntervalMS != 2000 { |
| 2715 | t.Fatalf("partial probe config minted an un-defaulted interval: %+v", h) |
| 2716 | } |
| 2717 | if h.FailAfter != 3 { |
| 2718 | t.Fatalf("partial probe config minted an un-defaulted fail_after: %+v", h) |
| 2719 | } |
| 2720 | } |
| 2721 | |
| 2722 | // TestShapeMismatchComparesAddrsAndHealth is Task 3's adopt-side proof: |
| 2723 | // shapeMismatch must refuse to adopt a hand-added route whose Addrs or |
| 2724 | // Health disagree with what the record describes, the same guard it already |
| 2725 | // applies to the legacy Addr column. |
| 2726 | func TestShapeMismatchComparesAddrsAndHealth(t *testing.T) { |
| 2727 | a := registry.Route{Kind: "service", Addrs: "x:1,y:2"} |
| 2728 | b := registry.Route{Kind: "service", Addrs: "x:1,z:3"} |
| 2729 | if shapeMismatch(a, b) == "" { |
| 2730 | t.Fatal("differing addrs adopted silently") |
| 2731 | } |
| 2732 | c := registry.Route{Kind: "service", Addrs: "x:1,y:2", Health: `{"path":"/up"}`} |
| 2733 | if shapeMismatch(a, c) == "" { |
| 2734 | t.Fatal("differing health adopted silently") |
| 2735 | } |
| 2736 | if shapeMismatch(a, registry.Route{Kind: "service", Addrs: "x:1,y:2"}) != "" { |
| 2737 | t.Fatal("equal shapes must match") |
| 2738 | } |
| 2739 | } |
| 2740 | |
| 2741 | // --- inbound placement (design §8, box half) -------------------------------- |
| 2742 | |
| 2743 | // TestReconcileNoInboundBoxRefusesRoutedInstances pins the instance pass's |
| 2744 | // half of design §8: a box that has declared CARLOS_INBOUND=none must never |
| 2745 | // mint a routed instance row, no matter what an account's own record asks |
| 2746 | // for — a box that cannot serve inbound traffic must never be handed |
| 2747 | // something to serve. This is independent of ReconcileSidecars' placement |
| 2748 | // gate (Task 9): that gate matches a RECORD's placement request against the |
| 2749 | // box; this one refuses ALL ordinary instance minting outright, because an |
| 2750 | // ordinary instance record carries no placement vocabulary at all. |
| 2751 | func TestReconcileNoInboundBoxRefusesRoutedInstances(t *testing.T) { |
| 2752 | s, sqid := setup(t) |
| 2753 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 2754 | t.Fatal(err) |
| 2755 | } |
| 2756 | reg := regWith(t) |
| 2757 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data", Inbound: InboundNone} |
| 2758 | |
| 2759 | var logged bytes.Buffer |
| 2760 | log.SetOutput(&logged) |
| 2761 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 2762 | |
| 2763 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 2764 | t.Fatal(err) |
| 2765 | } |
| 2766 | if rt, err := reg.Get("acme.go.tito.io"); err != nil || rt != nil { |
| 2767 | t.Fatalf("a no-inbound box minted a routed instance: %+v %v", rt, err) |
| 2768 | } |
| 2769 | if !strings.Contains(logged.String(), "acme.go.tito.io") || !strings.Contains(logged.String(), "no-inbound") { |
| 2770 | t.Fatalf("refusal must be logged loudly (host + reason), got: %s", logged.String()) |
| 2771 | } |
| 2772 | } |
| 2773 | |
| 2774 | // TestNormalizeInboundRestrictiveVocabulary pins the case/whitespace-folded |
| 2775 | // "none" spellings (review fix, Task 10 finding): "none", "NONE" and |
| 2776 | // " None " must all normalize to the one restrictive value the mint/ |
| 2777 | // placement gates compare against with exact equality, regardless of how an |
| 2778 | // operator typed it. |
| 2779 | func TestNormalizeInboundRestrictiveVocabulary(t *testing.T) { |
| 2780 | for _, raw := range []string{"none", "NONE", " None "} { |
| 2781 | if got := NormalizeInbound(raw); got != InboundNone { |
| 2782 | t.Errorf("NormalizeInbound(%q) = %q, want %q", raw, got, InboundNone) |
| 2783 | } |
| 2784 | } |
| 2785 | } |
| 2786 | |
| 2787 | // TestNormalizeInboundInertVocabulary pins the case/whitespace-folded |
| 2788 | // "any"/unset spellings: "", "any" and "ANY" must all normalize to a value |
| 2789 | // that is NOT InboundNone, and a box configured with any of them must mint a |
| 2790 | // routed instance exactly as it always has — the closed vocabulary must |
| 2791 | // never make a deliberate "any" spelling behave like an unrecognised one. |
| 2792 | func TestNormalizeInboundInertVocabulary(t *testing.T) { |
| 2793 | for _, raw := range []string{"", "any", "ANY"} { |
| 2794 | got := NormalizeInbound(raw) |
| 2795 | if got == InboundNone { |
| 2796 | t.Fatalf("NormalizeInbound(%q) = %q, must not be restrictive", raw, got) |
| 2797 | } |
| 2798 | |
| 2799 | s, sqid := setup(t) |
| 2800 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 2801 | t.Fatal(err) |
| 2802 | } |
| 2803 | reg := regWith(t) |
| 2804 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data", Inbound: got} |
| 2805 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 2806 | t.Fatal(err) |
| 2807 | } |
| 2808 | mustGet(t, reg, "acme.go.tito.io") |
| 2809 | } |
| 2810 | } |
| 2811 | |
| 2812 | // TestNormalizeInboundUnrecognisedIsRestrictiveAndLogged pins the review |
| 2813 | // fix's ruling directly (Task 10 finding): the only way to produce an |
| 2814 | // unrecognised CARLOS_INBOUND value is to have typed one, and an operator |
| 2815 | // only sets this variable at all when they mean to restrict — so an |
| 2816 | // unrecognised value must normalize to the restrictive reading AND be |
| 2817 | // logged loudly, naming the offending value, so a fat-fingered box is |
| 2818 | // caught on the next log tail rather than by a misplaced workload. |
| 2819 | func TestNormalizeInboundUnrecognisedIsRestrictiveAndLogged(t *testing.T) { |
| 2820 | var logged bytes.Buffer |
| 2821 | log.SetOutput(&logged) |
| 2822 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 2823 | |
| 2824 | got := NormalizeInbound("off") |
| 2825 | if got != InboundNone { |
| 2826 | t.Fatalf("NormalizeInbound(%q) = %q, want restrictive %q", "off", got, InboundNone) |
| 2827 | } |
| 2828 | if !strings.Contains(logged.String(), "off") { |
| 2829 | t.Fatalf("unrecognised value must be logged by name, got: %s", logged.String()) |
| 2830 | } |
| 2831 | } |
| 2832 | |
| 2833 | // TestReconcileUnsetInboundMintsAsBefore pins the inert-default half: a box |
| 2834 | // whose CARLOS_INBOUND is unset (the zero value read by provisionTickConfig) |
| 2835 | // must mint exactly as it always has — the new guard must never fire for |
| 2836 | // "any"/"" boxes, which is every box that exists today. |
| 2837 | func TestReconcileUnsetInboundMintsAsBefore(t *testing.T) { |
| 2838 | s, sqid := setup(t) |
| 2839 | if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| 2840 | t.Fatal(err) |
| 2841 | } |
| 2842 | reg := regWith(t) |
| 2843 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| 2844 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 2845 | t.Fatal(err) |
| 2846 | } |
| 2847 | mustGet(t, reg, "acme.go.tito.io") |
| 2848 | } |
| 2849 | |
| 2850 | // TestRouteForCarriesSteering: the mint renders the record's steering opt-in |
| 2851 | // onto the route, byte for byte — no derivation, unlike Hibernate/StatusNamed. |
| 2852 | func TestRouteForCarriesSteering(t *testing.T) { |
| 2853 | rec := Record{Host: "m.example.com", App: "web", Account: "bab", Channel: "stable", Steering: "latency"} |
| 2854 | if got := RouteFor(rec, "/data", "seed", false, nil).Steering; got != "latency" { |
| 2855 | t.Fatalf("Steering = %q, want latency", got) |
| 2856 | } |
| 2857 | unsteered := Record{Host: "s.example.com", App: "web", Account: "bab", Channel: "stable"} |
| 2858 | if got := RouteFor(unsteered, "/data", "seed", false, nil).Steering; got != "" { |
| 2859 | t.Fatalf("Steering = %q, want empty for a record that never mentioned it", got) |
| 2860 | } |
| 2861 | } |
| 2862 | |
| 2863 | // TestShapeMismatchIgnoresSteering is the re-ruled adopt-side proof (plan |
| 2864 | // amendment b485ec5, review finding 2): Steering is RECORD-OWNED, the same |
| 2865 | // category Channel/Environment/Hibernate are — a hand-added route whose |
| 2866 | // steering disagrees with what the record describes must still be ADOPTED, |
| 2867 | // not refused, because comparing it here would permanently block adoption |
| 2868 | // for exactly the case adoption exists to serve (an operator's hand-added |
| 2869 | // route predating the console's steering PATCH door). convergeOwned stamps |
| 2870 | // the record's Steering onto the row unconditionally once adoption succeeds |
| 2871 | // — this test's first assertion is what proves that stamp isn't blocked. |
| 2872 | func TestShapeMismatchIgnoresSteering(t *testing.T) { |
| 2873 | a := registry.Route{Kind: "instance", Steering: "latency"} |
| 2874 | b := registry.Route{Kind: "instance", Steering: ""} |
| 2875 | if shapeMismatch(a, b) != "" { |
| 2876 | t.Fatalf("differing steering blocked adoption: %q, want a match — steering is record-owned", shapeMismatch(a, b)) |
| 2877 | } |
| 2878 | if shapeMismatch(a, registry.Route{Kind: "instance", Steering: "latency"}) != "" { |
| 2879 | t.Fatal("equal steering must match") |
| 2880 | } |
| 2881 | // A steering-only difference must not mask a REAL mismatch on another |
| 2882 | // column — shapeMismatch still has to walk past it to the next case. |
| 2883 | c := registry.Route{Kind: "service", Steering: "latency"} |
| 2884 | if reason := shapeMismatch(a, c); reason == "" { |
| 2885 | t.Fatal("a genuine kind mismatch was hidden behind the ignored steering column") |
| 2886 | } |
| 2887 | } |
| 2888 | |
| 2889 | // --- #162: replica records, the home's placement set, and the two boxes' -- |
| 2890 | // --- division of labour --------------------------------------------------- |
| 2891 | |
| 2892 | // TestReplicaRecordMintsOnlyOnItsPlacement: a replica record names ONE |
| 2893 | // placement — its Region — and only the box carrying that placement mints |
| 2894 | // it. Everywhere else the record is out of scope exactly like any other |
| 2895 | // region's record: a replica minted on the wrong region would be a process |
| 2896 | // holding live order data nobody authorized into that region (the residence |
| 2897 | // rule), and a mint on the HOME box would stand up a second process of the |
| 2898 | // same account on a box that already holds the authoritative copy. |
| 2899 | func TestReplicaRecordMintsOnlyOnItsPlacement(t *testing.T) { |
| 2900 | s, sqid := setup(t) |
| 2901 | home := acmeRecord(sqid) |
| 2902 | if err := CreateRecord(ctx, s, home); err != nil { |
| 2903 | t.Fatal(err) |
| 2904 | } |
| 2905 | replica := Record{Schema: 1, Host: "acme-uk.go.tito.io", Slug: "acme-uk", App: "tito", |
| 2906 | Account: sqid, Pool: "tito", Region: "uk", Channel: "stable", |
| 2907 | Role: "replica", CreatedAt: time.Now()} |
| 2908 | if err := CreateRecord(ctx, s, replica); err != nil { |
| 2909 | t.Fatal(err) |
| 2910 | } |
| 2911 | |
| 2912 | // The uk box mints the replica. |
| 2913 | reg := regWith(t) |
| 2914 | if err := Reconcile(ctx, s, reg, ReconcileConfig{Region: "uk", Pools: []string{"tito"}, DataDir: "/data"}); err != nil { |
| 2915 | t.Fatal(err) |
| 2916 | } |
| 2917 | got, err := reg.Get("acme-uk.go.tito.io") |
| 2918 | if err != nil { |
| 2919 | t.Fatal(err) |
| 2920 | } |
| 2921 | if got == nil { |
| 2922 | t.Fatal("the uk box did not mint the replica record homed in uk") |
| 2923 | } |
| 2924 | |
| 2925 | // The ie box (the home's region) mints the home and nothing else. |
| 2926 | regIE := regWith(t) |
| 2927 | if err := Reconcile(ctx, s, regIE, ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"}); err != nil { |
| 2928 | t.Fatal(err) |
| 2929 | } |
| 2930 | if rt, _ := regIE.Get("acme-uk.go.tito.io"); rt != nil { |
| 2931 | t.Fatal("the ie box minted a record whose replica placement is uk — out of scope") |
| 2932 | } |
| 2933 | if rt, _ := regIE.Get("acme.go.tito.io"); rt == nil { |
| 2934 | t.Fatal("the ie box did not mint the home record") |
| 2935 | } |
| 2936 | } |
| 2937 | |
| 2938 | // TestHomeRecordConvergesReplicaSet: the home record's placement set is |
| 2939 | // authoritative in BOTH directions — the one deliberate widening the owned |
| 2940 | // arm permits — because a set that shrank must actually shrink on the row, |
| 2941 | // or the edges file goes on publishing a replica placement the account no |
| 2942 | // longer has. That is a residence lie, not a cosmetic one. |
| 2943 | func TestHomeRecordConvergesReplicaSet(t *testing.T) { |
| 2944 | s, sqid := setup(t) |
| 2945 | home := acmeRecord(sqid) |
| 2946 | home.Replicas = []string{"uk", "ie"} |
| 2947 | if err := CreateRecord(ctx, s, home); err != nil { |
| 2948 | t.Fatal(err) |
| 2949 | } |
| 2950 | reg := regWith(t) |
| 2951 | if err := Reconcile(ctx, s, reg, ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"}); err != nil { |
| 2952 | t.Fatal(err) |
| 2953 | } |
| 2954 | |
| 2955 | if rt, _ := reg.Get("acme.go.tito.io"); rt == nil { |
| 2956 | t.Fatal("home not minted") |
| 2957 | } else if rt.Replicas != "ie,uk" { |
| 2958 | t.Fatalf("minted row replicas = %q, want the canonical sorted set %q", rt.Replicas, "ie,uk") |
| 2959 | } |
| 2960 | |
| 2961 | // The set shrinks in the bucket; the row follows on the next pass. |
| 2962 | home.Replicas = []string{"uk"} |
| 2963 | if _, err := updateRecord(ctx, s, sqid, "tito", "acme.go.tito.io", func(r *Record) error { r.Replicas = []string{"uk"}; return nil }); err != nil { |
| 2964 | t.Fatal(err) |
| 2965 | } |
| 2966 | if err := Reconcile(ctx, s, reg, ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"}); err != nil { |
| 2967 | t.Fatal(err) |
| 2968 | } |
| 2969 | if rt, _ := reg.Get("acme.go.tito.io"); rt == nil { |
| 2970 | t.Fatal("home vanished") |
| 2971 | } else if rt.Replicas != "uk" { |
| 2972 | t.Fatalf("row replicas = %q after the record's set shrank, want %q — a stale placement keeps a retired region serving the account", rt.Replicas, "uk") |
| 2973 | } |
| 2974 | |
| 2975 | // And to none: the mirror deleted its last replica. |
| 2976 | if _, err := updateRecord(ctx, s, sqid, "tito", "acme.go.tito.io", func(r *Record) error { r.Replicas = nil; return nil }); err != nil { |
| 2977 | t.Fatal(err) |
| 2978 | } |
| 2979 | if err := Reconcile(ctx, s, reg, ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"}); err != nil { |
| 2980 | t.Fatal(err) |
| 2981 | } |
| 2982 | if rt, _ := reg.Get("acme.go.tito.io"); rt == nil { |
| 2983 | t.Fatal("home vanished") |
| 2984 | } else if rt.Replicas != "" { |
| 2985 | t.Fatalf("row replicas = %q after the record cleared its set, want empty", rt.Replicas) |
| 2986 | } |
| 2987 | } |
| 2988 | |
| 2989 | // TestReplicaMarkMintsAndConverges: the row's replica mark is Record.Role's |
| 2990 | // shadow, and the mark decides how the edges file publishes the row — so it |
| 2991 | // converges from the record in BOTH directions. A re-mint carrying no role |
| 2992 | // must DEMOTE a row that carried one: a stale mark keeps steering treating |
| 2993 | // the row's box as a replica of an account that no longer has one there. |
| 2994 | func TestReplicaMarkMintsAndConverges(t *testing.T) { |
| 2995 | s, sqid := setup(t) |
| 2996 | replica := Record{Schema: 1, Host: "acme-uk.go.tito.io", Slug: "acme-uk", App: "tito", |
| 2997 | Account: sqid, Pool: "tito", Region: "uk", Channel: "stable", |
| 2998 | Role: "replica", CreatedAt: time.Now()} |
| 2999 | if err := CreateRecord(ctx, s, replica); err != nil { |
| 3000 | t.Fatal(err) |
| 3001 | } |
| 3002 | reg := regWith(t) |
| 3003 | if err := Reconcile(ctx, s, reg, ReconcileConfig{Region: "uk", Pools: []string{"tito"}, DataDir: "/data"}); err != nil { |
| 3004 | t.Fatal(err) |
| 3005 | } |
| 3006 | if rt, _ := reg.Get("acme-uk.go.tito.io"); rt == nil { |
| 3007 | t.Fatal("replica not minted") |
| 3008 | } else if !rt.Replica { |
| 3009 | t.Fatal("minted row carries no replica mark") |
| 3010 | } |
| 3011 | |
| 3012 | // The role leaves the record; the mark follows. |
| 3013 | if _, err := updateRecord(ctx, s, sqid, "tito", "acme-uk.go.tito.io", func(r *Record) error { r.Role = ""; return nil }); err != nil { |
| 3014 | t.Fatal(err) |
| 3015 | } |
| 3016 | if err := Reconcile(ctx, s, reg, ReconcileConfig{Region: "uk", Pools: []string{"tito"}, DataDir: "/data"}); err != nil { |
| 3017 | t.Fatal(err) |
| 3018 | } |
| 3019 | rt, err := reg.Get("acme-uk.go.tito.io") |
| 3020 | if err != nil || rt == nil { |
| 3021 | t.Fatalf("get: %v", err) |
| 3022 | } |
| 3023 | if rt.Replica { |
| 3024 | t.Fatal("row keeps a replica mark its record no longer carries — the edges file would go on telling the fleet this box replicas an account it homes") |
| 3025 | } |
| 3026 | } |
| 3027 | |
| 3028 | // TestReplicaNeverEverywhereClass pins the residence rule's second door |
| 3029 | // (Fable review, finding 3): a replica is a process on ONE box — the one its |
| 3030 | // placement names — and everywhereClass's widening is the mint gate for |
| 3031 | // every non-fleet box. The class check refuses a replica FIRST (before any |
| 3032 | // kind or addr test), and everywhereRoute agrees, because the two must |
| 3033 | // never disagree about a row whose record is gone. |
| 3034 | func TestReplicaNeverEverywhereClass(t *testing.T) { |
| 3035 | // Static would pass the kind arm on its own — the role check must be |
| 3036 | // first and absolute. |
| 3037 | if everywhereClass(Record{Role: "replica", Kind: "static"}) { |
| 3038 | t.Error("everywhereClass(replica static) = true — a replica must never widen past its placement box") |
| 3039 | } |
| 3040 | if everywhereClass(Record{Role: "replica", Kind: "instance"}) { |
| 3041 | t.Error("everywhereClass(replica instance) = true") |
| 3042 | } |
| 3043 | // A public-addr replica would pass the dial test on its own; the shape |
| 3044 | // door refuses it, and the class check must not depend on that door |
| 3045 | // holding. |
| 3046 | if err := ValidateShape(Record{Role: "replica", Addr: "203.0.113.5:8090"}); err == nil { |
| 3047 | t.Error("ValidateShape(replica with a public addr) accepted — the everywhere-class second door") |
| 3048 | } |
| 3049 | if everywhereRoute(registry.Route{Replica: true, Kind: "static"}) { |
| 3050 | t.Error("everywhereRoute(replica row) = true — the removal-phase half of the same question") |
| 3051 | } |
| 3052 | } |
| 3053 | |
| 3054 | // A fresh row on a box whose layout intent is nested is minted at the |
| 3055 | // nested paths (design §8.6 step 5, plan 5), never flat and then moved. A |
| 3056 | // flat or absent intent mints the flat defaults as before; an unflagged |
| 3057 | // (externally supervised) socket route stays flat whatever the intent; an |
| 3058 | // intent that cannot be read defers the mint rather than guessing. |
| 3059 | func TestReconcileMintsNestedUnderANestedIntent(t *testing.T) { |
| 3060 | s, sqid := setup(t) |
| 3061 | off := false |
| 3062 | recs := []Record{ |
| 3063 | acmeRecord(sqid), |
| 3064 | {Schema: 1, Host: "router.go.tito.io", Slug: "router", App: "tito", Account: sqid, |
| 3065 | Pool: "tito", Region: "ie", Channel: "stable", Hibernate: &off, CreatedAt: time.Now()}, |
| 3066 | } |
| 3067 | for _, rec := range recs { |
| 3068 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 3069 | t.Fatal(err) |
| 3070 | } |
| 3071 | } |
| 3072 | reg := regWith(t) |
| 3073 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data", |
| 3074 | Layout: func(context.Context, release.Store) (string, error) { return release.LayoutModeNested, nil }} |
| 3075 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 3076 | t.Fatal(err) |
| 3077 | } |
| 3078 | got := *mustGet(t, reg, "acme.go.tito.io") |
| 3079 | if got.DBPath != NestedDBPath("/data", "acme.go.tito.io") || got.Socket != NestedSocket("acme.go.tito.io") || !got.Hibernate { |
| 3080 | t.Errorf("minted under a nested intent = db %q socket %q hibernate %v, want the nested paths", got.DBPath, got.Socket, got.Hibernate) |
| 3081 | } |
| 3082 | ext := *mustGet(t, reg, "router.go.tito.io") |
| 3083 | if ext.DBPath != DefaultDBPath("/data", "router.go.tito.io") || ext.Socket != DefaultSocket("router.go.tito.io") { |
| 3084 | t.Errorf("an unflagged socket route minted nested: db %q socket %q", ext.DBPath, ext.Socket) |
| 3085 | } |
| 3086 | // The next pass sees the nested row as the record's own, not drift, |
| 3087 | // and converges nothing on it. |
| 3088 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 3089 | t.Fatal(err) |
| 3090 | } |
| 3091 | if again := *mustGet(t, reg, "acme.go.tito.io"); again.DBPath != got.DBPath || again.Socket != got.Socket { |
| 3092 | t.Errorf("second pass moved the row: %+v", again) |
| 3093 | } |
| 3094 | |
| 3095 | // Flat intent, and no intent: the flat defaults. |
| 3096 | for name, layout := range map[string]func(context.Context, release.Store) (string, error){ |
| 3097 | "flat": func(context.Context, release.Store) (string, error) { return release.LayoutModeFlat, nil }, |
| 3098 | "none": func(context.Context, release.Store) (string, error) { return release.LayoutModeNone, nil }, |
| 3099 | "nil": nil, |
| 3100 | } { |
| 3101 | reg := regWith(t) |
| 3102 | if err := Reconcile(ctx, s, reg, ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data", Layout: layout}); err != nil { |
| 3103 | t.Fatal(err) |
| 3104 | } |
| 3105 | if got := *mustGet(t, reg, "acme.go.tito.io"); got.DBPath != "/data/acme.go.tito.io.db" { |
| 3106 | t.Errorf("%s intent: db %q, want the flat default", name, got.DBPath) |
| 3107 | } |
| 3108 | } |
| 3109 | |
| 3110 | // Unreadable: nothing is minted this pass, and the pass itself is not |
| 3111 | // an error (the records are retried next pass). |
| 3112 | reg = regWith(t) |
| 3113 | calls := 0 |
| 3114 | cfg.Layout = func(context.Context, release.Store) (string, error) { calls++; return "", errors.New("store down") } |
| 3115 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 3116 | t.Fatal(err) |
| 3117 | } |
| 3118 | if rt, err := reg.Get("acme.go.tito.io"); err != nil || rt != nil { |
| 3119 | t.Errorf("a row was minted while the intent was unreadable: %+v %v", rt, err) |
| 3120 | } |
| 3121 | if calls != 1 { |
| 3122 | t.Errorf("the intent was read %d times in one pass, want once", calls) |
| 3123 | } |
| 3124 | } |
| 3125 | |
| 3126 | // A hand-added row at the nested paths (`carlos add` on a nested box) is |
| 3127 | // the record's route and is adopted, exactly as a flat hand-added one is. |
| 3128 | func TestReconcileAdoptsAHandAddedNestedRoute(t *testing.T) { |
| 3129 | s, sqid := setup(t) |
| 3130 | rec := acmeRecord(sqid) |
| 3131 | rec.Channel = "edge" |
| 3132 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 3133 | t.Fatal(err) |
| 3134 | } |
| 3135 | handAdded := registry.Route{ |
| 3136 | Host: "acme.go.tito.io", Slug: "acme", Kind: "instance", |
| 3137 | Socket: NestedSocket("acme.go.tito.io"), DBPath: NestedDBPath("/data", "acme.go.tito.io"), |
| 3138 | App: "tito", Account: sqid, Pool: "tito", Region: "ie", |
| 3139 | Channel: "stable", Hibernate: true, Status: "ready", |
| 3140 | } |
| 3141 | reg := regWith(t, handAdded) |
| 3142 | if err := Reconcile(ctx, s, reg, ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"}); err != nil { |
| 3143 | t.Fatal(err) |
| 3144 | } |
| 3145 | got := *mustGet(t, reg, "acme.go.tito.io") |
| 3146 | if !got.Provisioned || got.Channel != "edge" { |
| 3147 | t.Fatalf("a nested hand-added route was not adopted: %+v", got) |
| 3148 | } |
| 3149 | if got.DBPath != handAdded.DBPath || got.Socket != handAdded.Socket { |
| 3150 | t.Errorf("adoption rewrote the nested paths: %+v", got) |
| 3151 | } |
| 3152 | } |
| 3153 | |
| 3154 | // ---- move holds (moves spec §5.2, §9.6, §19 K9) ---- |
| 3155 | |
| 3156 | // movingRecord is acme with a Phase 1 move from ie to uk after the flip: the |
| 3157 | // record's region is already the target's. |
| 3158 | func movingRecord(sqid string) Record { |
| 3159 | rec := acmeRecord(sqid) |
| 3160 | rec.Region = "uk" |
| 3161 | rec.Move = &MoveState{ID: "mv_acme", From: "ie", To: "uk", State: MoveLanding, LandingPhase: LandChecking, |
| 3162 | RequestedAt: time.Now(), UpdatedAt: time.Now()} |
| 3163 | return rec |
| 3164 | } |
| 3165 | |
| 3166 | // holdLog records the hold writes a pass reports through HoldWritten. |
| 3167 | type holdLog struct{ hosts []string } |
| 3168 | |
| 3169 | func (h *holdLog) written(host string) { h.hosts = append(h.hosts, host) } |
| 3170 | |
| 3171 | // Review Focus 5: on the target the landing row is minted held and never |
| 3172 | // ready, at flat paths whatever the box's layout intent says (G11); on the |
| 3173 | // source, a participant's moving row survives the record naming another |
| 3174 | // region, and without the participant answer it is removed as today. |
| 3175 | func TestReconcileHoldsParticipantRows(t *testing.T) { |
| 3176 | s, sqid := setup(t) |
| 3177 | rec := movingRecord(sqid) |
| 3178 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 3179 | t.Fatal(err) |
| 3180 | } |
| 3181 | hold := func(r Record) (string, string, bool) { |
| 3182 | if r.Host == rec.Host && r.Move != nil { |
| 3183 | return "landing", r.Move.ID, true |
| 3184 | } |
| 3185 | return "", "", false |
| 3186 | } |
| 3187 | |
| 3188 | // The uk box: minted held, twice over, never ready; flat paths. |
| 3189 | uk := regWith(t) |
| 3190 | var wrote holdLog |
| 3191 | ukCfg := ReconcileConfig{Region: "uk", Pools: []string{"tito"}, DataDir: "/data", Hold: hold, HoldWritten: wrote.written, |
| 3192 | Layout: func(context.Context, release.Store) (string, error) { return release.LayoutModeNested, nil }} |
| 3193 | for pass := 1; pass <= 2; pass++ { |
| 3194 | if err := Reconcile(ctx, s, uk, ukCfg); err != nil { |
| 3195 | t.Fatal(err) |
| 3196 | } |
| 3197 | got := mustGet(t, uk, rec.Host) |
| 3198 | if got.Status != "landing" || got.HoldOp != "mv_acme" { |
| 3199 | t.Fatalf("pass %d: the uk row is %q held by %q, want landing held by mv_acme", pass, got.Status, got.HoldOp) |
| 3200 | } |
| 3201 | if got.DBPath != DefaultDBPath("/data", rec.Host) || got.Socket != DefaultSocket(rec.Host) { |
| 3202 | t.Fatalf("pass %d: the held row is not flat: db %q socket %q", pass, got.DBPath, got.Socket) |
| 3203 | } |
| 3204 | } |
| 3205 | if len(wrote.hosts) != 1 || wrote.hosts[0] != rec.Host { |
| 3206 | t.Fatalf("hold writes reported = %v, want the one mint", wrote.hosts) |
| 3207 | } |
| 3208 | |
| 3209 | // The ie box: the source's moving row, held by the move. |
| 3210 | ieRow := RouteFor(acmeRecord(sqid), "/data", `["seed"]`, false, nil) |
| 3211 | ieRow.Status, ieRow.HoldOp = "moving", "mv_acme" |
| 3212 | ie := regWith(t, ieRow) |
| 3213 | keep := true |
| 3214 | ieCfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data", |
| 3215 | KeepParticipant: func(host string) bool { return keep && host == rec.Host }} |
| 3216 | if err := Reconcile(ctx, s, ie, ieCfg); err != nil { |
| 3217 | t.Fatal(err) |
| 3218 | } |
| 3219 | if got := mustGet(t, ie, rec.Host); got.Status != "moving" || got.HoldOp != "mv_acme" { |
| 3220 | t.Fatalf("the participant's row changed: %+v", got) |
| 3221 | } |
| 3222 | keep = false |
| 3223 | if err := Reconcile(ctx, s, ie, ieCfg); err != nil { |
| 3224 | t.Fatal(err) |
| 3225 | } |
| 3226 | if rt, err := ie.Get(rec.Host); err != nil || rt != nil { |
| 3227 | t.Fatalf("without the participant answer the row was not removed: %+v %v", rt, err) |
| 3228 | } |
| 3229 | } |
| 3230 | |
| 3231 | // A held mint goes through the claim hook as a ready mint does (claim Plan |
| 3232 | // 6b): with claims on, the row is minted held under the owner the hook |
| 3233 | // grants, flat, and a later pass over it keeps it held; a held mint the box |
| 3234 | // refuses (no inbound, the config gate) never reaches the hook. The hold and |
| 3235 | // the claim both have to allow the mint. |
| 3236 | func TestReconcileHeldMintGoesThroughTheClaimHook(t *testing.T) { |
| 3237 | hold := func(host string) func(Record) (string, string, bool) { |
| 3238 | return func(r Record) (string, string, bool) { |
| 3239 | if r.Host == host { |
| 3240 | return "landing", "mv_acme", true |
| 3241 | } |
| 3242 | return "", "", false |
| 3243 | } |
| 3244 | } |
| 3245 | t.Run("minted held under the hook's owner", func(t *testing.T) { |
| 3246 | s, reg, host, cfg := instanceClaimFixture(t) |
| 3247 | fx := newClaimEffects(t) |
| 3248 | var wrote holdLog |
| 3249 | cfg.ClaimInstall, cfg.Hold, cfg.HoldWritten = fx.hook(t), hold(host), wrote.written |
| 3250 | cfg.Layout = func(context.Context, release.Store) (string, error) { return release.LayoutModeNested, nil } |
| 3251 | for pass := 1; pass <= 2; pass++ { |
| 3252 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 3253 | t.Fatal(err) |
| 3254 | } |
| 3255 | got := mustGet(t, reg, host) |
| 3256 | if got.Status != "landing" || got.HoldOp != "mv_acme" { |
| 3257 | t.Fatalf("pass %d: the row is %q held by %q, want landing held by mv_acme", pass, got.Status, got.HoldOp) |
| 3258 | } |
| 3259 | if got.ClaimOwnership != fx.owner { |
| 3260 | t.Fatalf("pass %d: the held row's ownership is %+v, want the hook's %+v", pass, got.ClaimOwnership, fx.owner) |
| 3261 | } |
| 3262 | if got.DBPath != DefaultDBPath(cfg.DataDir, host) || got.Socket != DefaultSocket(host) { |
| 3263 | t.Fatalf("pass %d: the held row is not flat: db %q socket %q", pass, got.DBPath, got.Socket) |
| 3264 | } |
| 3265 | } |
| 3266 | fx.reached(t, 2, "the held mint, then the kept row") |
| 3267 | if len(wrote.hosts) != 1 || wrote.hosts[0] != host { |
| 3268 | t.Fatalf("hold writes reported = %v, want the one mint", wrote.hosts) |
| 3269 | } |
| 3270 | }) |
| 3271 | t.Run("a refused held mint never reaches the hook", func(t *testing.T) { |
| 3272 | refuse := errors.New("no sealed config") |
| 3273 | cases := []struct { |
| 3274 | name string |
| 3275 | apply func(*ReconcileConfig) |
| 3276 | }{ |
| 3277 | {"inbound none", func(cfg *ReconcileConfig) { cfg.Inbound = InboundNone }}, |
| 3278 | {"config gate", func(cfg *ReconcileConfig) { |
| 3279 | cfg.ConfigGate = func(context.Context, registry.Route) error { return refuse } |
| 3280 | }}, |
| 3281 | } |
| 3282 | for _, tc := range cases { |
| 3283 | t.Run(tc.name, func(t *testing.T) { |
| 3284 | s, reg, host, cfg := instanceClaimFixture(t) |
| 3285 | fx := newClaimEffects(t) |
| 3286 | cfg.ClaimInstall, cfg.Hold = fx.hook(t), hold(host) |
| 3287 | tc.apply(&cfg) |
| 3288 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 3289 | t.Fatal(err) |
| 3290 | } |
| 3291 | fx.untouched(t, tc.name) |
| 3292 | if got, err := reg.Get(host); err != nil || got != nil { |
| 3293 | t.Fatalf("a refused held mint wrote a route: %+v %v", got, err) |
| 3294 | } |
| 3295 | }) |
| 3296 | } |
| 3297 | }) |
| 3298 | } |
| 3299 | |
| 3300 | // G10: Upsert cannot hold an existing row, so Hold on an unheld row of this |
| 3301 | // pass's own is SetHold, and reported; a row already held by the same op is |
| 3302 | // left as is, unreported; a row held by another op is left alone; a row this |
| 3303 | // pass does not own is not taken into a move. |
| 3304 | func TestReconcileHoldsAnExistingRow(t *testing.T) { |
| 3305 | s, sqid := setup(t) |
| 3306 | rec := movingRecord(sqid) |
| 3307 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 3308 | t.Fatal(err) |
| 3309 | } |
| 3310 | owned := RouteFor(rec, "/data", `["seed"]`, false, nil) |
| 3311 | cases := []struct { |
| 3312 | name string |
| 3313 | row registry.Route |
| 3314 | wantStatus, wantOp string |
| 3315 | reported bool |
| 3316 | }{ |
| 3317 | {"unheld, owned", owned, "landing", "mv_acme", true}, |
| 3318 | {"held by this op", withHold(owned, "landing", "mv_acme"), "landing", "mv_acme", false}, |
| 3319 | {"held by another op", withHold(owned, "restoring", "rb_other"), "restoring", "rb_other", false}, |
| 3320 | {"not this pass's row", registry.Route{Host: rec.Host, Kind: "instance", App: "other", Status: "ready", Addr: "127.0.0.1:9"}, "ready", "", false}, |
| 3321 | // A recovery hold is written by the operation itself, which opens its |
| 3322 | // own row while the record still names the recovery (§7.4 reverting |
| 3323 | // probed, §7.8 opened): an unheld row there is one it opened. |
| 3324 | {"unheld, a recovery hold", owned, "ready", "", false}, |
| 3325 | } |
| 3326 | for _, c := range cases { |
| 3327 | t.Run(c.name, func(t *testing.T) { |
| 3328 | reg := regWith(t, c.row) |
| 3329 | var wrote holdLog |
| 3330 | status := "landing" |
| 3331 | if strings.Contains(c.name, "recovery") { |
| 3332 | status = "restoring" |
| 3333 | } |
| 3334 | cfg := ReconcileConfig{Region: "uk", Pools: []string{"tito"}, DataDir: "/data", HoldWritten: wrote.written, |
| 3335 | Hold: func(r Record) (string, string, bool) { return status, r.Move.ID, r.Move.State == MoveLanding }} |
| 3336 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 3337 | t.Fatal(err) |
| 3338 | } |
| 3339 | got := mustGet(t, reg, rec.Host) |
| 3340 | if got.Status != c.wantStatus || got.HoldOp != c.wantOp { |
| 3341 | t.Fatalf("row is %q held by %q, want %q held by %q", got.Status, got.HoldOp, c.wantStatus, c.wantOp) |
| 3342 | } |
| 3343 | if reported := len(wrote.hosts) > 0; reported != c.reported { |
| 3344 | t.Fatalf("hold write reported = %v, want %v", wrote.hosts, c.reported) |
| 3345 | } |
| 3346 | }) |
| 3347 | } |
| 3348 | } |
| 3349 | |
| 3350 | func withHold(rt registry.Route, status, op string) registry.Route { |
| 3351 | rt.Status, rt.HoldOp = status, op |
| 3352 | return rt |
| 3353 | } |
| 3354 | |
| 3355 | // A dry run reports a held mint and writes nothing. |
| 3356 | func TestReconcileDryRunWritesNoHold(t *testing.T) { |
| 3357 | s, sqid := setup(t) |
| 3358 | rec := movingRecord(sqid) |
| 3359 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 3360 | t.Fatal(err) |
| 3361 | } |
| 3362 | reg := regWith(t) |
| 3363 | var wrote holdLog |
| 3364 | cfg := ReconcileConfig{Region: "uk", Pools: []string{"tito"}, DataDir: "/data", DryRun: true, HoldWritten: wrote.written, |
| 3365 | Hold: func(Record) (string, string, bool) { return "landing", "mv_acme", true }} |
| 3366 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 3367 | t.Fatal(err) |
| 3368 | } |
| 3369 | if rt, _ := reg.Get(rec.Host); rt != nil || len(wrote.hosts) != 0 { |
| 3370 | t.Fatalf("a dry run wrote: %+v %v", rt, wrote.hosts) |
| 3371 | } |
| 3372 | } |
| 3373 | |
| 3374 | // changingStore answers the record's key with first, then with every later |
| 3375 | // read with later: the record moved on between the pass's listing and its |
| 3376 | // hold write. |
| 3377 | type changingStore struct { |
| 3378 | release.Store |
| 3379 | key string |
| 3380 | reads int |
| 3381 | first, later []byte |
| 3382 | } |
| 3383 | |
| 3384 | func (c *changingStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 3385 | if key == c.key { |
| 3386 | c.reads++ |
| 3387 | body := c.later |
| 3388 | if c.reads == 1 { |
| 3389 | body = c.first |
| 3390 | } |
| 3391 | return body, fmt.Sprintf("etag-%d", c.reads), nil |
| 3392 | } |
| 3393 | return c.Store.Get(ctx, key) |
| 3394 | } |
| 3395 | |
| 3396 | // A hold write on an existing row re-reads the record first: the target |
| 3397 | // opens its row right after its commit, and the pass's listing (read before |
| 3398 | // the routes) can still say landing. A record that has moved on is not held |
| 3399 | // again. |
| 3400 | func TestReconcileHoldRereadsTheRecordBeforeHoldingARow(t *testing.T) { |
| 3401 | s, sqid := setup(t) |
| 3402 | rec := movingRecord(sqid) |
| 3403 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 3404 | t.Fatal(err) |
| 3405 | } |
| 3406 | committed := rec |
| 3407 | m := *rec.Move |
| 3408 | m.State = MoveCommitted |
| 3409 | committed.Move = &m |
| 3410 | cs := &changingStore{Store: s, key: RecordKey(rec.Account, rec.App, rec.Host), first: mustJSON(t, rec), later: mustJSON(t, committed)} |
| 3411 | reg := regWith(t, RouteFor(rec, "/data", `["seed"]`, false, nil)) // the target's row, opened |
| 3412 | var wrote holdLog |
| 3413 | cfg := ReconcileConfig{Region: "uk", Pools: []string{"tito"}, DataDir: "/data", HoldWritten: wrote.written, |
| 3414 | Hold: func(r Record) (string, string, bool) { return "landing", r.Move.ID, r.Move.State == MoveLanding }} |
| 3415 | if err := Reconcile(ctx, cs, reg, cfg); err != nil { |
| 3416 | t.Fatal(err) |
| 3417 | } |
| 3418 | if cs.reads < 2 { |
| 3419 | t.Fatalf("the record was read %d time(s); the hold write did not re-read it", cs.reads) |
| 3420 | } |
| 3421 | if got := mustGet(t, reg, rec.Host); got.Status != "ready" || got.HoldOp != "" || len(wrote.hosts) != 0 { |
| 3422 | t.Fatalf("an opened row was held again from a stale listing: %+v %v", got, wrote.hosts) |
| 3423 | } |
| 3424 | } |
| 3425 | |
| 3426 | // Fix round 1 (task-6-review, Important 2): the hold write on an existing |
| 3427 | // row is SetHoldIf, conditional on the row still being unheld in the same |
| 3428 | // statement. Another op that takes the row after the pass's listing (here |
| 3429 | // while the pass is paused just before its write) keeps it: the stale write |
| 3430 | // does not land and nothing is reported. |
| 3431 | func TestReconcileHoldWriteIsRefusedWhenAnotherOpTookTheRow(t *testing.T) { |
| 3432 | s, sqid := setup(t) |
| 3433 | rec := movingRecord(sqid) |
| 3434 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 3435 | t.Fatal(err) |
| 3436 | } |
| 3437 | reg := regWith(t, RouteFor(rec, "/data", `["seed"]`, false, nil)) // unheld at the listing |
| 3438 | var wrote holdLog |
| 3439 | cfg := ReconcileConfig{Region: "uk", Pools: []string{"tito"}, DataDir: "/data", HoldWritten: wrote.written, |
| 3440 | Hold: func(r Record) (string, string, bool) { return "landing", r.Move.ID, r.Move.State == MoveLanding }} |
| 3441 | paused := 0 |
| 3442 | cfg.beforeHoldWrite = func(host string) { |
| 3443 | paused++ |
| 3444 | // A rollback's hold, from a writer outside this pass's lock (root, |
| 3445 | // or another process): the row is another op's by the time the |
| 3446 | // pass writes. |
| 3447 | if err := reg.SetHold(host, "restoring", "rb_other"); err != nil { |
| 3448 | t.Error(err) |
| 3449 | } |
| 3450 | } |
| 3451 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 3452 | t.Fatal(err) |
| 3453 | } |
| 3454 | if paused != 1 { |
| 3455 | t.Fatalf("the pass paused %d time(s) before a hold write, want once", paused) |
| 3456 | } |
| 3457 | if got := mustGet(t, reg, rec.Host); got.Status != "restoring" || got.HoldOp != "rb_other" { |
| 3458 | t.Fatalf("the stale hold write landed over another op's: %+v", got) |
| 3459 | } |
| 3460 | if len(wrote.hosts) != 0 { |
| 3461 | t.Fatalf("a refused hold write was reported: %v", wrote.hosts) |
| 3462 | } |
| 3463 | } |
| 3464 | |
| 3465 | // Fix round 1 (task-6-review, Important 2): the hold decision, from the |
| 3466 | // record re-read through the write, runs under the host's hold lock, the |
| 3467 | // one the landing's open and finalisation take (WithHostHoldLock). An open |
| 3468 | // that arrives inside that window waits for the write, then finds the hold |
| 3469 | // under its own op and releases it; it can never run between the re-read |
| 3470 | // and the write, so the pass cannot re-hold a row the move has opened. A |
| 3471 | // later pass sees the committed record and leaves the opened row alone. |
| 3472 | func TestReconcileHoldWriteIsSerialisedWithTheMoveOpening(t *testing.T) { |
| 3473 | s, sqid := setup(t) |
| 3474 | rec := movingRecord(sqid) |
| 3475 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 3476 | t.Fatal(err) |
| 3477 | } |
| 3478 | reg := regWith(t, RouteFor(rec, "/data", `["seed"]`, false, nil)) |
| 3479 | var wrote holdLog |
| 3480 | cfg := ReconcileConfig{Region: "uk", Pools: []string{"tito"}, DataDir: "/data", HoldWritten: wrote.written, |
| 3481 | Hold: func(r Record) (string, string, bool) { return "landing", r.Move.ID, r.Move.State == MoveLanding }} |
| 3482 | opened := make(chan struct{}) |
| 3483 | cfg.beforeHoldWrite = func(host string) { |
| 3484 | // The target's open, racing the write: the commit (record CAS) and |
| 3485 | // the conditional release, under the same lock. |
| 3486 | go func() { |
| 3487 | defer close(opened) |
| 3488 | WithHostHoldLock(host, func() { |
| 3489 | fresh, etag, err := GetRecordETag(ctx, s, rec.Account, rec.App, rec.Host) |
| 3490 | if err != nil { |
| 3491 | t.Error(err) |
| 3492 | return |
| 3493 | } |
| 3494 | m := *fresh.Move |
| 3495 | m.State = MoveCommitted |
| 3496 | fresh.Move = &m |
| 3497 | if _, err := s.Put(ctx, RecordKey(rec.Account, rec.App, rec.Host), mustJSON(t, fresh), etag); err != nil { |
| 3498 | t.Error(err) |
| 3499 | } |
| 3500 | if ok, err := reg.ReleaseHold(host, rec.Move.ID); err != nil || !ok { |
| 3501 | t.Errorf("the open found no hold under its own op to release: %v %v", ok, err) |
| 3502 | } |
| 3503 | }) |
| 3504 | }() |
| 3505 | select { |
| 3506 | case <-opened: |
| 3507 | t.Error("the open ran inside the pass's hold decision") |
| 3508 | case <-time.After(100 * time.Millisecond): |
| 3509 | } |
| 3510 | } |
| 3511 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 3512 | t.Fatal(err) |
| 3513 | } |
| 3514 | select { |
| 3515 | case <-opened: |
| 3516 | case <-time.After(10 * time.Second): |
| 3517 | t.Fatal("the open never got the lock after the pass") |
| 3518 | } |
| 3519 | if got := mustGet(t, reg, rec.Host); got.Status != "ready" || got.HoldOp != "" { |
| 3520 | t.Fatalf("after the open the row is %+v, want ready and unheld", got) |
| 3521 | } |
| 3522 | if len(wrote.hosts) != 1 { |
| 3523 | t.Fatalf("hold writes reported = %v, want the pass's one write before the open", wrote.hosts) |
| 3524 | } |
| 3525 | cfg.beforeHoldWrite = func(string) { t.Error("a later pass tried to hold the opened row") } |
| 3526 | if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| 3527 | t.Fatal(err) |
| 3528 | } |
| 3529 | if got := mustGet(t, reg, rec.Host); got.Status != "ready" || got.HoldOp != "" || len(wrote.hosts) != 1 { |
| 3530 | t.Fatalf("a later pass re-held the opened row: %+v %v", got, wrote.hosts) |
| 3531 | } |
| 3532 | } |
| 3533 | |
| 3534 | // WithHostHoldLock is exclusive per host and independent across hosts. |
| 3535 | func TestWithHostHoldLockIsExclusivePerHost(t *testing.T) { |
| 3536 | entered, release := make(chan struct{}), make(chan struct{}) |
| 3537 | go WithHostHoldLock("a.go.tito.io", func() { close(entered); <-release }) |
| 3538 | <-entered |
| 3539 | WithHostHoldLock("b.go.tito.io", func() {}) // another host's lock is free |
| 3540 | second := make(chan struct{}) |
| 3541 | go func() { WithHostHoldLock("a.go.tito.io", func() {}); close(second) }() |
| 3542 | select { |
| 3543 | case <-second: |
| 3544 | t.Fatal("a second holder entered the host's lock") |
| 3545 | case <-time.After(50 * time.Millisecond): |
| 3546 | } |
| 3547 | close(release) |
| 3548 | select { |
| 3549 | case <-second: |
| 3550 | case <-time.After(5 * time.Second): |
| 3551 | t.Fatal("the lock was not handed on") |
| 3552 | } |
| 3553 | } |
| 3554 | |
| 3555 | // Reconcile's own reads carry the runs to the box (run spec §5.2): every |
| 3556 | // in-scope record with a run, including one whose row this very pass |
| 3557 | // mints, and none out of scope. |
| 3558 | func TestReconcileHandsRunsToRunSeen(t *testing.T) { |
| 3559 | s, sqid := setup(t) |
| 3560 | shopRecord := func(host, slug string) Record { |
| 3561 | rec := acmeRecord(sqid) |
| 3562 | rec.Host, rec.Slug, rec.App, rec.Pool = host, slug, "shop", "shared" |
| 3563 | return rec |
| 3564 | } |
| 3565 | withRun := shopRecord("acme.example.com", "acme") |
| 3566 | withRun.ActiveRun = &RunState{ID: "run_abcdefghijklmnopqrstuvwxyz", State: RunQueued, Deadline: time.Now().Add(time.Minute)} |
| 3567 | plain := shopRecord("plain.example.com", "plain") |
| 3568 | away := shopRecord("away.example.com", "away") |
| 3569 | away.Region = "uk" |
| 3570 | away.ActiveRun = withRun.ActiveRun |
| 3571 | for _, rec := range []Record{withRun, plain, away} { |
| 3572 | if err := CreateRecord(ctx, s, rec); err != nil { |
| 3573 | t.Fatal(err) |
| 3574 | } |
| 3575 | } |
| 3576 | var seen []string |
| 3577 | cfg := ReconcileConfig{Region: "ie", Pools: []string{"shared"}, DataDir: "/data", RunSeen: func(rec Record) { |
| 3578 | seen = append(seen, rec.Host+":"+rec.ActiveRun.ID) |
| 3579 | }} |
| 3580 | if err := Reconcile(ctx, s, regWith(t), cfg); err != nil { |
| 3581 | t.Fatal(err) |
| 3582 | } |
| 3583 | if len(seen) != 1 || seen[0] != "acme.example.com:run_abcdefghijklmnopqrstuvwxyz" { |
| 3584 | t.Fatalf("RunSeen saw %q", seen) |
| 3585 | } |
| 3586 | } |
| 3587 | |
| 3588 | // strictShopRecord is a record of app "shop" under sqid, on a host under |
| 3589 | // example.com, for the strict-listing tests below. |
| 3590 | func strictShopRecord(sqid, slug string) Record { |
| 3591 | return Record{Schema: 1, Host: slug + ".shop.example.com", Slug: slug, App: "shop", |
| 3592 | Account: sqid, Region: "ie", Channel: "stable", CreatedAt: time.Now()} |
| 3593 | } |
| 3594 | |
| 3595 | func TestListAllRecordsStrictNamesUnreadableHosts(t *testing.T) { |
| 3596 | s, sqid := shopSetup(t) |
| 3597 | for _, slug := range []string{"good", "garbled", "liar"} { |
| 3598 | if err := CreateRecord(ctx, s, strictShopRecord(sqid, slug)); err != nil { |
| 3599 | t.Fatal(err) |
| 3600 | } |
| 3601 | } |
| 3602 | putReplace(t, s, RecordKey(sqid, "shop", "garbled.shop.example.com"), []byte("{not json")) |
| 3603 | // A body that decodes but names another host than its path does. |
| 3604 | liar := strictShopRecord(sqid, "other") |
| 3605 | putReplace(t, s, RecordKey(sqid, "shop", "liar.shop.example.com"), mustJSON(t, liar)) |
| 3606 | |
| 3607 | recs, unreadable, err := ListAllRecordsStrict(ctx, s) |
| 3608 | if err != nil { |
| 3609 | t.Fatal(err) |
| 3610 | } |
| 3611 | if len(recs) != 1 || recs[0].Host != "good.shop.example.com" { |
| 3612 | t.Fatalf("recs %+v, want only good.shop.example.com", recs) |
| 3613 | } |
| 3614 | if len(unreadable) != 2 || !unreadable["garbled.shop.example.com"] || !unreadable["liar.shop.example.com"] { |
| 3615 | t.Fatalf("unreadable %v, want garbled and liar keyed by their path hosts", unreadable) |
| 3616 | } |
| 3617 | |
| 3618 | // The wrappers behave as they always did: the same records, no |
| 3619 | // unreadable host, no error. |
| 3620 | plain, err := ListAllRecords(ctx, s) |
| 3621 | if err != nil || len(plain) != 1 || plain[0].Host != "good.shop.example.com" { |
| 3622 | t.Fatalf("ListAllRecords = %+v, %v", plain, err) |
| 3623 | } |
| 3624 | tol, complete, err := ListAllRecordsTolerant(ctx, s) |
| 3625 | if err != nil || !complete || len(tol) != 1 || tol[0].Host != "good.shop.example.com" { |
| 3626 | t.Fatalf("ListAllRecordsTolerant = %+v, complete %v, %v", tol, complete, err) |
| 3627 | } |
| 3628 | } |
| 3629 | |
| 3630 | // The tolerant reader fills the same set. No wrapper exposes it, so this |
| 3631 | // reads readAllRecords directly: a record that reads but does not decode is |
| 3632 | // skipped, named, and does not make the set incomplete (it was read). |
| 3633 | func TestReadAllRecordsTolerantNamesUnreadableHosts(t *testing.T) { |
| 3634 | s, sqid := shopSetup(t) |
| 3635 | for _, slug := range []string{"good", "garbled"} { |
| 3636 | if err := CreateRecord(ctx, s, strictShopRecord(sqid, slug)); err != nil { |
| 3637 | t.Fatal(err) |
| 3638 | } |
| 3639 | } |
| 3640 | putReplace(t, s, RecordKey(sqid, "shop", "garbled.shop.example.com"), []byte("{not json")) |
| 3641 | recs, complete, unreadable, err := readAllRecords(ctx, s, false) |
| 3642 | if err != nil || !complete || len(recs) != 1 || recs[0].Host != "good.shop.example.com" { |
| 3643 | t.Fatalf("recs %+v complete %v err %v", recs, complete, err) |
| 3644 | } |
| 3645 | if len(unreadable) != 1 || !unreadable["garbled.shop.example.com"] { |
| 3646 | t.Fatalf("unreadable %v, want garbled.shop.example.com", unreadable) |
| 3647 | } |
| 3648 | } |
| 3649 | |
| 3650 | func TestListAllRecordsStrictCleanSetHasNoUnreadableHosts(t *testing.T) { |
| 3651 | s, sqid := shopSetup(t) |
| 3652 | if err := CreateRecord(ctx, s, strictShopRecord(sqid, "one")); err != nil { |
| 3653 | t.Fatal(err) |
| 3654 | } |
| 3655 | recs, unreadable, err := ListAllRecordsStrict(ctx, s) |
| 3656 | if err != nil || len(recs) != 1 { |
| 3657 | t.Fatalf("recs %+v err %v", recs, err) |
| 3658 | } |
| 3659 | if unreadable == nil || len(unreadable) != 0 { |
| 3660 | t.Fatalf("unreadable = %#v, want an empty non-nil map", unreadable) |
| 3661 | } |
| 3662 | } |
| 3663 | |
| 3664 | // A read failure is still an error, with no records and no map: the strict |
| 3665 | // listing adds to ListAllRecords' answer and never weakens it. |
| 3666 | func TestListAllRecordsStrictKeepsFailingClosed(t *testing.T) { |
| 3667 | inner, sqid := shopSetup(t) |
| 3668 | if err := CreateRecord(ctx, inner, strictShopRecord(sqid, "one")); err != nil { |
| 3669 | t.Fatal(err) |
| 3670 | } |
| 3671 | s := &bucketShaped{Store: inner, failList: RecordsPrefix(sqid, "shop")} |
| 3672 | recs, unreadable, err := ListAllRecordsStrict(ctx, s) |
| 3673 | if err == nil || recs != nil || unreadable != nil { |
| 3674 | t.Fatalf("a failed app listing = %+v, %v, %v; want an error and nothing else", recs, unreadable, err) |
| 3675 | } |
| 3676 | fs := &failStore{Store: inner} |
| 3677 | fs.failReadsFor(RecordKey(sqid, "shop", "one.shop.example.com")) |
| 3678 | recs, unreadable, err = ListAllRecordsStrict(ctx, fs) |
| 3679 | if err == nil || recs != nil || unreadable != nil { |
| 3680 | t.Fatalf("a record that will not Get = %+v, %v, %v; want an error and nothing else", recs, unreadable, err) |
| 3681 | } |
| 3682 | } |
| 3683 | |
amadan