| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "fmt" |
| 8 | "net/http" |
| 9 | "net/http/httptest" |
| 10 | "strings" |
| 11 | "testing" |
| 12 | "time" |
| 13 | |
| 14 | "github.com/carlosframework/platform/internal/boxchoice" |
| 15 | "github.com/carlosframework/platform/internal/hostclaim" |
| 16 | "github.com/carlosframework/platform/internal/hostops" |
| 17 | "github.com/carlosframework/platform/internal/objstore" |
| 18 | "github.com/carlosframework/platform/internal/provision" |
| 19 | "github.com/carlosframework/platform/internal/status" |
| 20 | ) |
| 21 | |
| 22 | func TestNoCapacityAlertsRepeatAfterAnHour(t *testing.T) { |
| 23 | var a noCapacityAlerts |
| 24 | t0 := time.Date(2026, 10, 16, 12, 0, 0, 0, time.UTC) |
| 25 | if !a.due("uk", t0) || a.due("uk", t0.Add(59*time.Minute)) || !a.due("ie", t0) || !a.due("uk", t0.Add(time.Hour)) { |
| 26 | t.Fatal("once per region an hour") |
| 27 | } |
| 28 | } |
| 29 | |
| 30 | func TestClaimRefusalAnswersNoRoom(t *testing.T) { |
| 31 | ref := claimResponseRefusal(fmt.Errorf("begin: %w", &noCapacityError{Region: "uk"})) |
| 32 | if ref.Status != http.StatusServiceUnavailable || ref.Msg != noCapacityRefusal("uk") { |
| 33 | t.Fatalf("refusal %+v", ref) |
| 34 | } |
| 35 | var err error = &noCapacityError{Region: "uk"} |
| 36 | if !errors.Is(err, hostops.ErrNoEligibleBox) { |
| 37 | t.Fatal("a no-room refusal is not ErrNoEligibleBox") |
| 38 | } |
| 39 | } |
| 40 | |
| 41 | // pagesTo replaces the alert sender for the test and returns what it sent. |
| 42 | func pagesTo(t *testing.T) chan string { |
| 43 | t.Helper() |
| 44 | pages := make(chan string, 16) |
| 45 | old := pageNoCapacity |
| 46 | pageNoCapacity = func(_ context.Context, level, subject, _ string) error { |
| 47 | pages <- level + " " + subject |
| 48 | return nil |
| 49 | } |
| 50 | t.Cleanup(func() { pageNoCapacity = old }) |
| 51 | return pages |
| 52 | } |
| 53 | |
| 54 | func expectPages(t *testing.T, pages chan string, want ...string) { |
| 55 | t.Helper() |
| 56 | for _, w := range want { |
| 57 | select { |
| 58 | case got := <-pages: |
| 59 | if got != w { |
| 60 | t.Fatalf("page %q, want %q", got, w) |
| 61 | } |
| 62 | case <-time.After(5 * time.Second): |
| 63 | t.Fatalf("no page %q", w) |
| 64 | } |
| 65 | } |
| 66 | select { |
| 67 | case got := <-pages: |
| 68 | t.Fatalf("an extra page: %q", got) |
| 69 | case <-time.After(100 * time.Millisecond): |
| 70 | } |
| 71 | } |
| 72 | |
| 73 | func noRoomSubject(region string) string { |
| 74 | subject, _ := noCapacityAlert(region) |
| 75 | return "warning " + subject |
| 76 | } |
| 77 | |
| 78 | // haClaimCase is a host-claim console with region HA on: its create doors |
| 79 | // choose through the region HA chooser over the edges files the test |
| 80 | // publishes (publishHABox). home names the console's home region ("" for |
| 81 | // none): a first route is placed in the home. |
| 82 | func haClaimCase(t *testing.T, home string) *claimCase { |
| 83 | t.Helper() |
| 84 | cx := newClaimCaseWith(t, func(c *Config) { c.HomeRegion = home }) |
| 85 | cx.srv.cfg.RegionHA = true |
| 86 | a := cx.srv.cfg.ClaimAuthority |
| 87 | a.Executors = cx.srv.executorChooser(a.Directory) |
| 88 | return cx |
| 89 | } |
| 90 | |
| 91 | // publishHABox publishes (or republishes) a roster box's edges file and |
| 92 | // status snapshot: placeable in ie unless mut changes it, and stale when |
| 93 | // stale is set. |
| 94 | func publishHABox(t *testing.T, cx *claimCase, box string, stale bool, mut func(map[string]any)) { |
| 95 | t.Helper() |
| 96 | ctx := context.Background() |
| 97 | m := map[string]any{ |
| 98 | "v": 1, "box": box, "box_id": box, "eip": "203.0.113.7", "region": "eu-west-1", "placement": "ie", |
| 99 | "hibernation_configured": true, "serves_shared_pool": true, "accepts_inbound": true, |
| 100 | "secrets_key_fp": cx.srv.expectedSecretsFP(), "capabilities": []string{"region-ha-1"}, |
| 101 | "headroom": map[string]any{"mem_free_mib": 4000, "mem_total_mib": 8000, "disk_free_mib": 50000, "disk_total_mib": 100000, "awake": 0}, |
| 102 | "generation": time.Now().UTC().Format(time.RFC3339), "hosts": map[string]any{}, |
| 103 | } |
| 104 | if mut != nil { |
| 105 | mut(m) |
| 106 | } |
| 107 | body, err := json.Marshal(m) |
| 108 | if err != nil { |
| 109 | t.Fatal(err) |
| 110 | } |
| 111 | key := "control/edges/" + box + ".json" |
| 112 | _, tag, err := cx.rel.Get(ctx, key) |
| 113 | if err != nil { |
| 114 | tag = "" |
| 115 | } |
| 116 | if _, err := cx.rel.Put(ctx, key, body, tag); err != nil { |
| 117 | t.Fatal(err) |
| 118 | } |
| 119 | at := time.Now() |
| 120 | if stale { |
| 121 | at = at.Add(-time.Hour) |
| 122 | } |
| 123 | if err := status.WriteBox(ctx, cx.rel, status.Box{Label: box, GeneratedAt: at}); err != nil { |
| 124 | t.Fatal(err) |
| 125 | } |
| 126 | } |
| 127 | |
| 128 | // registrations counts account/app's registered operations. |
| 129 | func registrations(t *testing.T, cx *claimCase, account, app string) int { |
| 130 | t.Helper() |
| 131 | a, err := hostops.ReadCanonicalAdmission(context.Background(), cx.rel, account, app) |
| 132 | if err != nil { |
| 133 | t.Fatal(err) |
| 134 | } |
| 135 | return len(a.Value.Registrations) |
| 136 | } |
| 137 | |
| 138 | // refusedBeforeRegistering checks what H27 promises of a refused create: |
| 139 | // nothing registered, no claim, and nothing recovery writes afterwards. |
| 140 | func refusedBeforeRegistering(t *testing.T, cx *claimCase, host string, registered int) { |
| 141 | t.Helper() |
| 142 | ctx := context.Background() |
| 143 | if n := registrations(t, cx, "bab", "titogo"); n != registered { |
| 144 | t.Fatalf("registrations %d after the refusal, want %d: the door registered before choosing", n, registered) |
| 145 | } |
| 146 | if obs, err := hostclaim.Read(ctx, cx.rel, host); err != nil || !obs.Absent { |
| 147 | t.Fatalf("claim after the refusal: %+v %v", obs, err) |
| 148 | } |
| 149 | if err := cx.srv.claimRecoveryPass(ctx); err != nil { |
| 150 | t.Fatalf("recovery: %v", err) |
| 151 | } |
| 152 | if obs, err := hostclaim.Read(ctx, cx.rel, host); err != nil || !obs.Absent { |
| 153 | t.Fatalf("claim after recovery: %+v %v", obs, err) |
| 154 | } |
| 155 | if n := registrations(t, cx, "bab", "titogo"); n != registered { |
| 156 | t.Fatalf("registrations %d after recovery, want %d", n, registered) |
| 157 | } |
| 158 | } |
| 159 | |
| 160 | // wroteNothing checks that nothing reached the bucket or a backend since |
| 161 | // mark: no lease, no registration, no record and no store write. |
| 162 | func wroteNothing(t *testing.T, cx *claimCase, mark int) { |
| 163 | t.Helper() |
| 164 | if events := cx.rec.since(mark); len(events) != 0 { |
| 165 | t.Fatalf("a refused request wrote %v", events) |
| 166 | } |
| 167 | } |
| 168 | |
| 169 | // Region HA spec 2 §7 (H27): every create door that chooses a box refuses |
| 170 | // with a 503 when none is eligible, before it registers anything, and the |
| 171 | // operator hears of it once per region an hour. Every box here is stale. |
| 172 | func TestCreateDoorsRefuseWithNoRoomBeforeRegistering(t *testing.T) { |
| 173 | t.Run("instance create", func(t *testing.T) { |
| 174 | pages := pagesTo(t) |
| 175 | cx := haClaimCase(t, "") |
| 176 | cx.instanceWorld(t) |
| 177 | publishHABox(t, cx, "box-ie-tito", true, nil) |
| 178 | before := registrations(t, cx, "bab", "titogo") |
| 179 | mark := cx.rec.mark() |
| 180 | for range 3 { // a retry loop: refused every time, paged once |
| 181 | r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", `{"host":"acme.go.tito.io","region":"ie"}`) |
| 182 | if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, noCapacityRefusal("ie")) { |
| 183 | t.Fatalf("create = %d %s", r.Status, r.Body) |
| 184 | } |
| 185 | } |
| 186 | wroteNothing(t, cx, mark) |
| 187 | refusedBeforeRegistering(t, cx, "acme.go.tito.io", before) |
| 188 | expectPages(t, pages, noRoomSubject("ie")) |
| 189 | }) |
| 190 | t.Run("standalone sidecar create", func(t *testing.T) { |
| 191 | pages := pagesTo(t) |
| 192 | cx := haClaimCase(t, "") |
| 193 | cx.sidecarWorld(t) |
| 194 | before := registrations(t, cx, "bab", "titogo") |
| 195 | mark := cx.rec.mark() |
| 196 | r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/sidecars", `{"name":"bot","region":"ie"}`) |
| 197 | if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, noCapacityRefusal("ie")) { |
| 198 | t.Fatalf("sidecar create = %d %s", r.Status, r.Body) |
| 199 | } |
| 200 | wroteNothing(t, cx, mark) |
| 201 | refusedBeforeRegistering(t, cx, provision.SidecarHost("bot", "titogo", "bab"), before) |
| 202 | expectPages(t, pages, noRoomSubject("ie")) |
| 203 | }) |
| 204 | t.Run("domain attach, a first route", func(t *testing.T) { |
| 205 | pages := pagesTo(t) |
| 206 | cx := haClaimCase(t, "ie") |
| 207 | shipVersion(t, cx.consoleFixture, "titogo", "v1") |
| 208 | seedChannel(t, cx.consoleFixture, "titogo", "edge", "v1") |
| 209 | before := registrations(t, cx, "bab", "titogo") |
| 210 | mark := cx.rec.mark() |
| 211 | r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/domains", `{"host":"first.titogo-claim.example","kind":"static"}`) |
| 212 | if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, noCapacityRefusal("ie")) { |
| 213 | t.Fatalf("attach = %d %s", r.Status, r.Body) |
| 214 | } |
| 215 | wroteNothing(t, cx, mark) |
| 216 | refusedBeforeRegistering(t, cx, "first.titogo-claim.example", before) |
| 217 | expectPages(t, pages, noRoomSubject("ie")) |
| 218 | }) |
| 219 | t.Run("browser domain attach, a first route", func(t *testing.T) { |
| 220 | pages := pagesTo(t) |
| 221 | cx := haClaimCase(t, "ie") |
| 222 | shipVersion(t, cx.consoleFixture, "titogo", "v1") |
| 223 | seedChannel(t, cx.consoleFixture, "titogo", "edge", "v1") |
| 224 | before := registrations(t, cx, "bab", "titogo") |
| 225 | mark := cx.rec.mark() |
| 226 | r := cx.formReply(t, "/a/bab/apps/titogo/domains/attach", "host", "first.titogo-claim.example", "kind", "static") |
| 227 | if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, noCapacityRefusal("ie")) { |
| 228 | t.Fatalf("browser attach = %d %s", r.Status, r.Body) |
| 229 | } |
| 230 | wroteNothing(t, cx, mark) |
| 231 | refusedBeforeRegistering(t, cx, "first.titogo-claim.example", before) |
| 232 | expectPages(t, pages, noRoomSubject("ie")) |
| 233 | }) |
| 234 | } |
| 235 | |
| 236 | // appDerivedWorld is titogo with no pool and no region allow-list, whose |
| 237 | // app region is ie: a create that names no region takes the app's. |
| 238 | func appDerivedWorld(t *testing.T, cx *claimCase) { |
| 239 | t.Helper() |
| 240 | m := titoManifest() |
| 241 | m.Regions = nil |
| 242 | m.Pool = "" |
| 243 | enableInstances(t, cx.consoleFixture, "titogo", m) |
| 244 | cx.srv.appRegionOf = func(int64, string) (string, string, error) { return "ie", "app", nil } |
| 245 | } |
| 246 | |
| 247 | // Region HA spec 2 §7 (H27, Astra 2a round 1 finding 3): with region HA |
| 248 | // on, an app-derived region is judged by the chooser's eligible set. Every |
| 249 | // box stale is the no-room refusal, alert included, never the region |
| 250 | // refusal's 400. |
| 251 | func TestAppRegionWithEveryBoxStaleIsNoRoom(t *testing.T) { |
| 252 | pages := pagesTo(t) |
| 253 | cx := haClaimCase(t, "") |
| 254 | appDerivedWorld(t, cx) |
| 255 | publishHABox(t, cx, "box-ie-default", true, nil) |
| 256 | before := registrations(t, cx, "bab", "titogo") |
| 257 | r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", `{"host":"acme.go.tito.io"}`) |
| 258 | if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, noCapacityRefusal("ie")) { |
| 259 | t.Fatalf("create = %d %s", r.Status, r.Body) |
| 260 | } |
| 261 | refusedBeforeRegistering(t, cx, "acme.go.tito.io", before) |
| 262 | expectPages(t, pages, noRoomSubject("ie")) |
| 263 | } |
| 264 | |
| 265 | // ... and a misconfigured sibling (a box in ie holding another secrets |
| 266 | // key, outside the roster) no longer refuses a region whose eligible set |
| 267 | // holds another box: the create goes ahead on that box. With region HA off |
| 268 | // the sibling still refuses, as on main. |
| 269 | func TestAppRegionWithAMisconfiguredSiblingStillCreates(t *testing.T) { |
| 270 | cx := haClaimCase(t, "") |
| 271 | appDerivedWorld(t, cx) |
| 272 | publishHABox(t, cx, "box-ie-default", false, nil) |
| 273 | publishHABox(t, cx, "box-ie-stray", false, func(m map[string]any) { m["secrets_key_fp"] = "sha256:another-key" }) |
| 274 | off := cx.srv.placementChooser("") |
| 275 | cx.srv.cfg.RegionHA = false |
| 276 | r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", `{"host":"other.go.tito.io"}`) |
| 277 | if r.Status != http.StatusBadRequest { |
| 278 | t.Fatalf("region HA off: create = %d %s, want main's region refusal", r.Status, r.Body) |
| 279 | } |
| 280 | cx.srv.cfg.RegionHA = true |
| 281 | if off == nil { |
| 282 | t.Fatal("no chooser with region HA on") |
| 283 | } |
| 284 | r = cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", `{"host":"acme.go.tito.io"}`) |
| 285 | if r.Status != http.StatusCreated && r.Status != http.StatusAccepted { |
| 286 | t.Fatalf("create = %d %s", r.Status, r.Body) |
| 287 | } |
| 288 | obs, err := hostclaim.Read(context.Background(), cx.rel, "acme.go.tito.io") |
| 289 | if err != nil || obs.Absent { |
| 290 | t.Fatalf("claim: %+v %v", obs, err) |
| 291 | } |
| 292 | if exec, chosen := obs.Claim.Op.Exec, obs.Claim.Op.Intent["selected_executor"]; exec != "box-ie-default" && chosen != "box-ie-default" { |
| 293 | t.Fatalf("executor %q / %q, want box-ie-default", exec, chosen) |
| 294 | } |
| 295 | } |
| 296 | |
| 297 | // putSeedStore gives titogo's canary environment a store a seed made, and |
| 298 | // returns the store record's bytes. |
| 299 | func putSeedStore(t *testing.T, cx *claimCase) (key string, body []byte) { |
| 300 | t.Helper() |
| 301 | ctx := context.Background() |
| 302 | at := time.Now().UTC() |
| 303 | if _, err := objstore.PutIn(ctx, cx.rel, "bab", "titogo", "canary", &objstore.Record{ |
| 304 | Declared: &objstore.Declared{EnvPrefix: "S3", At: at, By: "bob@keymail.dev", Seed: newSeedID(t)}, |
| 305 | Grant: &objstore.GrantBlock{Backend: "mem", Bucket: "carlos-bab-titogo-canary", Source: objstore.GrantSourceAccountApproval, At: at, By: "operator@keymail.dev"}, |
| 306 | }, ""); err != nil { |
| 307 | t.Fatal(err) |
| 308 | } |
| 309 | key = objstore.KeyIn("bab", "titogo", "canary") |
| 310 | body, _, err := cx.rel.Get(ctx, key) |
| 311 | if err != nil { |
| 312 | t.Fatal(err) |
| 313 | } |
| 314 | return key, body |
| 315 | } |
| 316 | |
| 317 | // Astra 2a round 2 finding 2: the instance create door chooses before its |
| 318 | // first write. A create naming an environment whose store a seed made would |
| 319 | // take that store (bindEnvironmentCreate). Refused for want of room, it |
| 320 | // leaves the store's bytes as they were and writes nothing at all: when |
| 321 | // every box is stale (the region check refuses), and when the region's last |
| 322 | // placeable box goes between the region check and the choice (the early |
| 323 | // choice refuses, before the binding). With room, the same create takes the |
| 324 | // store, so the binding is on the path. |
| 325 | func TestRefusedCreateLeavesTheEnvironmentsStoreAlone(t *testing.T) { |
| 326 | const body = `{"host":"acme.go.tito.io","region":"ie","environment":"canary"}` |
| 327 | refused := func(t *testing.T, cx *claimCase, pages chan string) { |
| 328 | t.Helper() |
| 329 | key, stored := putSeedStore(t, cx) |
| 330 | before := registrations(t, cx, "bab", "titogo") |
| 331 | mark := cx.rec.mark() |
| 332 | r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", body) |
| 333 | if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, noCapacityRefusal("ie")) { |
| 334 | t.Fatalf("create = %d %s", r.Status, r.Body) |
| 335 | } |
| 336 | wroteNothing(t, cx, mark) |
| 337 | if now, _, err := cx.rel.Get(context.Background(), key); err != nil || string(now) != string(stored) { |
| 338 | t.Fatalf("the store after the refusal:\n%s\nwant\n%s (%v)", now, stored, err) |
| 339 | } |
| 340 | refusedBeforeRegistering(t, cx, "acme.go.tito.io", before) |
| 341 | expectPages(t, pages, noRoomSubject("ie")) |
| 342 | } |
| 343 | t.Run("every box stale", func(t *testing.T) { |
| 344 | pages := pagesTo(t) |
| 345 | cx := haClaimCase(t, "") |
| 346 | cx.instanceWorld(t) |
| 347 | publishHABox(t, cx, "box-ie-tito", true, nil) |
| 348 | refused(t, cx, pages) |
| 349 | |
| 350 | publishHABox(t, cx, "box-ie-tito", false, nil) |
| 351 | r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", body) |
| 352 | if r.Status != http.StatusCreated && r.Status != http.StatusAccepted { |
| 353 | t.Fatalf("create with room = %d %s", r.Status, r.Body) |
| 354 | } |
| 355 | store, _, err := objstore.GetIn(context.Background(), cx.rel, "bab", "titogo", "canary") |
| 356 | if err != nil || store == nil || store.Declared == nil || store.Declared.Seed != "" { |
| 357 | t.Fatalf("the store after a create with room: %+v %v, want the seed's mark cleared", store, err) |
| 358 | } |
| 359 | }) |
| 360 | t.Run("the last box goes before the choice", func(t *testing.T) { |
| 361 | pages := pagesTo(t) |
| 362 | cx := haClaimCase(t, "") |
| 363 | cx.instanceWorld(t) |
| 364 | publishHABox(t, cx, "box-ie-tito", false, nil) |
| 365 | x := cx.srv.cfg.ClaimAuthority.Executors |
| 366 | edges, reads := x.Edges, 0 |
| 367 | x.Edges = func(ctx context.Context) (boxchoice.Snapshot, error) { |
| 368 | if reads++; reads > 1 { |
| 369 | return boxchoice.Snapshot{Complete: true}, nil |
| 370 | } |
| 371 | return edges(ctx) |
| 372 | } |
| 373 | refused(t, cx, pages) |
| 374 | }) |
| 375 | } |
| 376 | |
| 377 | // Astra 2a round 2 finding 1, at the create door: with region HA on, a |
| 378 | // region whose boxes publish no public address (or no edges file) is a |
| 379 | // region with no room, not the armed-placement gate's 400. With region HA |
| 380 | // off the gate answers, as on main. |
| 381 | func TestCreateRegionWithNoReachableBoxIsNoRoom(t *testing.T) { |
| 382 | const body = `{"host":"acme.go.tito.io","region":"ie"}` |
| 383 | pages := pagesTo(t) |
| 384 | cx := haClaimCase(t, "") |
| 385 | cx.instanceWorld(t) |
| 386 | publishHABox(t, cx, "box-ie-tito", false, func(m map[string]any) { m["eip"] = "" }) |
| 387 | cx.srv.cfg.RegionHA = false |
| 388 | if r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", body); r.Status != http.StatusBadRequest { |
| 389 | t.Fatalf("region HA off: create = %d %s, want main's 400", r.Status, r.Body) |
| 390 | } |
| 391 | cx.srv.cfg.RegionHA = true |
| 392 | before := registrations(t, cx, "bab", "titogo") |
| 393 | mark := cx.rec.mark() |
| 394 | r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", body) |
| 395 | if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, noCapacityRefusal("ie")) { |
| 396 | t.Fatalf("create = %d %s", r.Status, r.Body) |
| 397 | } |
| 398 | wroteNothing(t, cx, mark) |
| 399 | refusedBeforeRegistering(t, cx, "acme.go.tito.io", before) |
| 400 | expectPages(t, pages, noRoomSubject("ie")) |
| 401 | } |
| 402 | |
| 403 | // Census ruling region-ha-2a rev 2, item 3: the early choice takes no lease |
| 404 | // and writes nothing, and its executor is the one the authority chooses on |
| 405 | // its own (the create's choice before the early choice existed). A choice |
| 406 | // the context already carries is kept: nothing is chosen again. |
| 407 | func TestChooseFirstWritesNothingAndChoosesOnce(t *testing.T) { |
| 408 | cx := haClaimCase(t, "") |
| 409 | appDerivedWorld(t, cx) |
| 410 | publishHABox(t, cx, "box-ie-default", false, nil) |
| 411 | ctx := context.Background() |
| 412 | host := "acme.go.tito.io" |
| 413 | build := func(now time.Time, acquisition string) (hostops.Record, error) { |
| 414 | rec := provision.Record{Schema: 1, Host: host, App: "titogo", Account: "bab", Channel: "stable", Kind: "static", CreatedBy: "bob@keymail.dev", Acquisition: acquisition} |
| 415 | return hostops.PrepareInstance(rec, now, cx.srv.cfg.claimPlacementRegion("ie")) |
| 416 | } |
| 417 | probe, err := build(time.Now().UTC(), "") |
| 418 | if err != nil { |
| 419 | t.Fatal(err) |
| 420 | } |
| 421 | before, err := hostclaim.Read(ctx, cx.rel, host) |
| 422 | if err != nil { |
| 423 | t.Fatal(err) |
| 424 | } |
| 425 | want, chosen, err := cx.srv.cfg.ClaimAuthority.SelectExecutor(ctx, before, probe) |
| 426 | if err != nil || !chosen || want.Box != "box-ie-default" { |
| 427 | t.Fatalf("the authority's own choice: %+v %v %v", want, chosen, err) |
| 428 | } |
| 429 | |
| 430 | mark := cx.rec.mark() |
| 431 | got, err := cx.srv.chooseCreateFirst(ctx, host, build) |
| 432 | if err != nil { |
| 433 | t.Fatal(err) |
| 434 | } |
| 435 | wroteNothing(t, cx, mark) |
| 436 | if sel, ok := hostops.ChosenExecutor(got, host); !ok || sel != want { |
| 437 | t.Fatalf("early choice %+v (%v), want the authority's %+v", sel, ok, want) |
| 438 | } |
| 439 | |
| 440 | again, err := cx.srv.chooseExecutorFirst(got, host, before, func(time.Time, string) (hostops.Record, error) { |
| 441 | t.Fatal("chose again") |
| 442 | return hostops.Record{}, nil |
| 443 | }) |
| 444 | if err != nil || again != got { |
| 445 | t.Fatalf("a carried choice was not kept: %v", err) |
| 446 | } |
| 447 | if sel, chosen, err := cx.srv.cfg.ClaimAuthority.SelectExecutor(got, before, probe); err != nil || !chosen || sel != want { |
| 448 | t.Fatalf("the create's selection %+v %v %v, want the door's %+v", sel, chosen, err, want) |
| 449 | } |
| 450 | } |
| 451 | |
| 452 | // Census ruling region-ha-2a item 5 (amadan #236): the early choice |
| 453 | // carries the claim observation it was decided on, and beginClaimCreate |
| 454 | // keeps it only if its own read of the claim is that observation. A claim |
| 455 | // write landing between the choice and begin refuses the create with |
| 456 | // main's retryable busy answer: the old choice is never used, nothing is |
| 457 | // chosen again, and nothing is registered or written. |
| 458 | func TestAStaleEarlyChoiceIsNeverUsed(t *testing.T) { |
| 459 | cx := haClaimCase(t, "") |
| 460 | cx.instanceWorld(t) |
| 461 | publishHABox(t, cx, "box-ie-tito", false, nil) |
| 462 | ctx := context.Background() |
| 463 | const host = "acme.go.tito.io" |
| 464 | acct, err := cx.st.AccountBySqid("bab") |
| 465 | if err != nil || acct == nil { |
| 466 | t.Fatalf("account: %+v %v", acct, err) |
| 467 | } |
| 468 | build := func(now time.Time, acquisition string) (hostops.Record, error) { |
| 469 | rec := provision.Record{Schema: 1, Host: host, App: "titogo", Account: "bab", Channel: "stable", Kind: "static", Pool: "tito", Region: "ie", |
| 470 | CreatedBy: "bob@keymail.dev", Acquisition: acquisition} |
| 471 | return hostops.PrepareInstance(rec, now, cx.srv.cfg.claimPlacementRegion("ie")) |
| 472 | } |
| 473 | chosen, err := cx.srv.chooseCreateFirst(ctx, host, build) |
| 474 | if _, ok := hostops.ChosenExecutor(chosen, host); err != nil || !ok { |
| 475 | t.Fatalf("early choice: %v (chosen %v)", err, ok) |
| 476 | } |
| 477 | |
| 478 | // A claim write lands between the choice and begin. |
| 479 | created := time.Now().UTC().Truncate(time.Second) |
| 480 | inc, err := hostclaim.Incarnation("bab", "titogo", host, created) |
| 481 | if err != nil { |
| 482 | t.Fatal(err) |
| 483 | } |
| 484 | writeServingClaim(t, cx.rel, host, inc, created, "box-ie-tito", "ie") |
| 485 | written, _, err := cx.rel.Get(ctx, "control/hosts/"+host+".json") |
| 486 | if err != nil { |
| 487 | t.Fatal(err) |
| 488 | } |
| 489 | before := registrations(t, cx, "bab", "titogo") |
| 490 | mark := cx.rec.mark() |
| 491 | // The build runs only to choose again or, past RegisterStart, to write |
| 492 | // the record: either is the old choice's create going on. |
| 493 | req, err := cx.srv.beginClaimCreate(chosen, acct, "titogo", host, "bob@keymail.dev", func(time.Time, string) (hostops.Record, error) { |
| 494 | t.Fatal("begin went on after the claim changed") |
| 495 | return hostops.Record{}, nil |
| 496 | }) |
| 497 | if req != nil { |
| 498 | req.Close() |
| 499 | } |
| 500 | if !errors.Is(err, errChoiceStale) || !errors.Is(err, hostclaim.ErrConflict) { |
| 501 | t.Fatalf("begin on a changed claim = %v, want the stale-choice refusal", err) |
| 502 | } |
| 503 | if ref := claimResponseRefusal(err); ref.Status != http.StatusConflict || ref.Msg != claimRetireBusyMsg { |
| 504 | t.Fatalf("refusal %+v, want main's retryable busy answer", ref) |
| 505 | } |
| 506 | wroteNothing(t, cx, mark) |
| 507 | if n := registrations(t, cx, "bab", "titogo"); n != before { |
| 508 | t.Fatalf("registrations %d, want %d", n, before) |
| 509 | } |
| 510 | if now, _, err := cx.rel.Get(ctx, "control/hosts/"+host+".json"); err != nil || string(now) != string(written) { |
| 511 | t.Fatalf("the claim changed under the refusal: %v", err) |
| 512 | } |
| 513 | } |
| 514 | |
| 515 | // Region HA 2a, Task 6's carry (controller ruling): PlaceableBoxes and |
| 516 | // Capacity choose nothing without a namespace, so every caller uses |
| 517 | // placementChooser, scoped as ProductionAuthority.SelectExecutor scopes the |
| 518 | // create. With a placeable box in each pool, the scoped chooser counts it, |
| 519 | // in the shared pool and in a named one; the unscoped chooser the authority |
| 520 | // holds counts nothing, which is why no door asks it. |
| 521 | func TestPlacementChooserIsScoped(t *testing.T) { |
| 522 | ctx := context.Background() |
| 523 | cx := haClaimCase(t, "") |
| 524 | publishHABox(t, cx, "box-ie-default", false, nil) |
| 525 | publishHABox(t, cx, "box-ie-tito", false, nil) |
| 526 | ie := cx.srv.cfg.claimPlacementRegion("ie") |
| 527 | for pool, want := range map[string]string{"": "box-ie-default", "tito": "box-ie-tito"} { |
| 528 | x := cx.srv.placementChooser(pool) |
| 529 | if x == nil { |
| 530 | t.Fatalf("pool %q: no chooser with region HA on", pool) |
| 531 | } |
| 532 | if x.Namespace == "" || x.Namespace != cx.srv.cfg.ClaimAuthority.Namespace { |
| 533 | t.Fatalf("pool %q: chooser namespace %q, want the authority's %q", pool, x.Namespace, cx.srv.cfg.ClaimAuthority.Namespace) |
| 534 | } |
| 535 | n, _, err := x.Capacity(ctx, ie) |
| 536 | if err != nil || n == 0 { |
| 537 | t.Fatalf("pool %q: capacity %d %v, want a placeable box", pool, n, err) |
| 538 | } |
| 539 | boxes, err := x.PlaceableBoxes(ctx, ie) |
| 540 | if err != nil || len(boxes) != 1 || boxes[0] != want { |
| 541 | t.Fatalf("pool %q: placeable %v %v, want [%s]", pool, boxes, err, want) |
| 542 | } |
| 543 | if err := cx.srv.roomIn(ctx, "ie", pool); err != nil { |
| 544 | t.Fatalf("pool %q: roomIn with a placeable box: %v", pool, err) |
| 545 | } |
| 546 | } |
| 547 | if n, _, err := cx.srv.cfg.ClaimAuthority.Executors.Capacity(ctx, ie); err != nil || n != 0 { |
| 548 | t.Fatalf("the unscoped chooser counted %d (%v): the scope is not what makes the count", n, err) |
| 549 | } |
| 550 | cx.srv.cfg.RegionHA = false |
| 551 | if cx.srv.placementChooser("") != nil || cx.srv.roomIn(ctx, "ie", "") != nil { |
| 552 | t.Fatal("region HA off: a chooser or a refusal") |
| 553 | } |
| 554 | } |
| 555 | |
| 556 | // Region HA 2a Task 11 fix round 1 (ruling: an incomplete placement read |
| 557 | // is not "no room"). A malformed edges file beside a placeable box makes |
| 558 | // the read incomplete: every choosing door answers main's temporarily |
| 559 | // unavailable 503 (on the app-derived path C28's could-not-confirm), never |
| 560 | // RHA-C17, pages nobody, and writes nothing. |
| 561 | func TestAnIncompletePlacementReadIsNotNoRoom(t *testing.T) { |
| 562 | malformed := func(t *testing.T, cx *claimCase) { |
| 563 | t.Helper() |
| 564 | if _, err := cx.rel.Put(context.Background(), "control/edges/box-x.json", []byte("{"), ""); err != nil { |
| 565 | t.Fatal(err) |
| 566 | } |
| 567 | } |
| 568 | unavailable := func(t *testing.T, r claimReply, msg string) { |
| 569 | t.Helper() |
| 570 | if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, msg) || strings.Contains(r.Body, noCapacityRefusal("ie")) { |
| 571 | t.Fatalf("answer %d %s, want 503 %q and not the no-room refusal", r.Status, r.Body, msg) |
| 572 | } |
| 573 | } |
| 574 | t.Run("instance create, an explicit region", func(t *testing.T) { |
| 575 | pages := pagesTo(t) |
| 576 | cx := haClaimCase(t, "") |
| 577 | cx.instanceWorld(t) |
| 578 | publishHABox(t, cx, "box-ie-tito", false, nil) |
| 579 | malformed(t, cx) |
| 580 | before := registrations(t, cx, "bab", "titogo") |
| 581 | mark := cx.rec.mark() |
| 582 | status, hdr, body := rawReq(t, http.MethodPost, cx.ts.URL+"/api/cli/apps/bab/titogo/instances", cx.bearer, `{"host":"acme.go.tito.io","region":"ie"}`) |
| 583 | unavailable(t, claimReply{Status: status, Body: body}, claimResponseUnavailable) |
| 584 | if got := hdr.Get("Retry-After"); got != "2" { |
| 585 | t.Fatalf("Retry-After %q, want main's 2", got) |
| 586 | } |
| 587 | wroteNothing(t, cx, mark) |
| 588 | refusedBeforeRegistering(t, cx, "acme.go.tito.io", before) |
| 589 | expectPages(t, pages) |
| 590 | }) |
| 591 | t.Run("standalone sidecar create", func(t *testing.T) { |
| 592 | pages := pagesTo(t) |
| 593 | cx := haClaimCase(t, "") |
| 594 | cx.sidecarWorld(t) |
| 595 | publishHABox(t, cx, "box-ie-tito", false, nil) |
| 596 | malformed(t, cx) |
| 597 | before := registrations(t, cx, "bab", "titogo") |
| 598 | mark := cx.rec.mark() |
| 599 | unavailable(t, cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/sidecars", `{"name":"bot","region":"ie"}`), claimResponseUnavailable) |
| 600 | wroteNothing(t, cx, mark) |
| 601 | refusedBeforeRegistering(t, cx, provision.SidecarHost("bot", "titogo", "bab"), before) |
| 602 | expectPages(t, pages) |
| 603 | }) |
| 604 | t.Run("domain attach, a first route", func(t *testing.T) { |
| 605 | pages := pagesTo(t) |
| 606 | cx := haClaimCase(t, "ie") |
| 607 | shipVersion(t, cx.consoleFixture, "titogo", "v1") |
| 608 | seedChannel(t, cx.consoleFixture, "titogo", "edge", "v1") |
| 609 | publishHABox(t, cx, "box-ie-default", false, nil) |
| 610 | malformed(t, cx) |
| 611 | before := registrations(t, cx, "bab", "titogo") |
| 612 | mark := cx.rec.mark() |
| 613 | unavailable(t, cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/domains", `{"host":"first.titogo-claim.example","kind":"static"}`), claimResponseUnavailable) |
| 614 | wroteNothing(t, cx, mark) |
| 615 | refusedBeforeRegistering(t, cx, "first.titogo-claim.example", before) |
| 616 | expectPages(t, pages) |
| 617 | }) |
| 618 | t.Run("instance create, the app's region", func(t *testing.T) { |
| 619 | pages := pagesTo(t) |
| 620 | cx := haClaimCase(t, "") |
| 621 | appDerivedWorld(t, cx) |
| 622 | publishHABox(t, cx, "box-ie-default", false, nil) |
| 623 | malformed(t, cx) |
| 624 | before := registrations(t, cx, "bab", "titogo") |
| 625 | unavailable(t, cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", `{"host":"acme.go.tito.io"}`), msgAppRegionUnconfirmed("ie")) |
| 626 | refusedBeforeRegistering(t, cx, "acme.go.tito.io", before) |
| 627 | expectPages(t, pages) |
| 628 | }) |
| 629 | } |
| 630 | |
| 631 | // Region HA 2a Task 11 fix round 1 (ruling): the no-capacity alert is |
| 632 | // recorded as sent only once a send succeeds, so a failed send is retried |
| 633 | // on the next refusal rather than silenced for an hour; a sent alert |
| 634 | // still silences the region for the hour. |
| 635 | func TestNoCapacityAlertIsRecordedOnlyWhenSent(t *testing.T) { |
| 636 | var a noCapacityAlerts |
| 637 | t0 := time.Date(2026, 10, 16, 12, 0, 0, 0, time.UTC) |
| 638 | if !a.due("uk", t0) { |
| 639 | t.Fatal("the first alert is not due") |
| 640 | } |
| 641 | if a.due("uk", t0.Add(time.Second)) { |
| 642 | t.Fatal("a second alert while the first is being sent") |
| 643 | } |
| 644 | a.failed("uk", t0) |
| 645 | if !a.due("uk", t0.Add(2*time.Second)) { |
| 646 | t.Fatal("a failed send silenced the region") |
| 647 | } |
| 648 | a.failed("uk", t0) // an older send's failure does not free a newer one |
| 649 | if a.due("uk", t0.Add(3*time.Second)) { |
| 650 | t.Fatal("a stale failure freed the region") |
| 651 | } |
| 652 | |
| 653 | sent := make(chan string, 16) |
| 654 | calls := 0 |
| 655 | old := pageNoCapacity |
| 656 | pageNoCapacity = func(_ context.Context, level, subject, _ string) error { |
| 657 | calls++ |
| 658 | sent <- subject |
| 659 | if calls == 1 { |
| 660 | return errors.New("the mail relay is down") |
| 661 | } |
| 662 | return nil |
| 663 | } |
| 664 | t.Cleanup(func() { pageNoCapacity = old }) |
| 665 | s := &Server{} |
| 666 | ctx := context.Background() |
| 667 | want, _ := noCapacityAlert("ie") |
| 668 | if err := s.noRoom(ctx, "ie"); !errors.Is(err, hostops.ErrNoEligibleBox) { |
| 669 | t.Fatalf("noRoom = %v", err) |
| 670 | } |
| 671 | if got := <-sent; got != want { |
| 672 | t.Fatalf("page %q", got) |
| 673 | } |
| 674 | // The failed send is given back once it returns: a later refusal pages. |
| 675 | deadline := time.Now().Add(5 * time.Second) |
| 676 | for second := false; !second; { |
| 677 | if time.Now().After(deadline) { |
| 678 | t.Fatal("a failed alert was never sent again") |
| 679 | } |
| 680 | _ = s.noRoom(ctx, "ie") |
| 681 | select { |
| 682 | case got := <-sent: |
| 683 | if got != want { |
| 684 | t.Fatalf("page %q", got) |
| 685 | } |
| 686 | second = true |
| 687 | case <-time.After(20 * time.Millisecond): |
| 688 | } |
| 689 | } |
| 690 | // That one was sent: the region is quiet for the hour. |
| 691 | for range 3 { |
| 692 | _ = s.noRoom(ctx, "ie") |
| 693 | } |
| 694 | select { |
| 695 | case got := <-sent: |
| 696 | t.Fatalf("an extra page after a sent alert: %q", got) |
| 697 | case <-time.After(200 * time.Millisecond): |
| 698 | } |
| 699 | } |
| 700 | |
| 701 | // Region HA 2a Task 11 fix round 1 (ruling): the no-room 503 carries |
| 702 | // Retry-After: 60, matching RHA-C17's "try again in a few minutes", on |
| 703 | // every choosing door; every other claim refusal keeps its own. |
| 704 | func TestNoRoomCarriesRetryAfterAMinute(t *testing.T) { |
| 705 | pagesTo(t) |
| 706 | for _, d := range []struct { |
| 707 | name, home, path, body string |
| 708 | world func(*testing.T, *claimCase) |
| 709 | }{ |
| 710 | {"instance create", "", "/api/cli/apps/bab/titogo/instances", `{"host":"acme.go.tito.io","region":"ie"}`, func(t *testing.T, cx *claimCase) { cx.instanceWorld(t) }}, |
| 711 | {"standalone sidecar create", "", "/api/cli/apps/bab/titogo/sidecars", `{"name":"bot","region":"ie"}`, func(t *testing.T, cx *claimCase) { cx.sidecarWorld(t) }}, |
| 712 | {"domain attach, a first route", "ie", "/api/cli/apps/bab/titogo/domains", `{"host":"first.titogo-claim.example","kind":"static"}`, func(t *testing.T, cx *claimCase) { |
| 713 | shipVersion(t, cx.consoleFixture, "titogo", "v1") |
| 714 | seedChannel(t, cx.consoleFixture, "titogo", "edge", "v1") |
| 715 | }}, |
| 716 | } { |
| 717 | cx := haClaimCase(t, d.home) |
| 718 | d.world(t, cx) |
| 719 | publishHABox(t, cx, "box-ie-tito", true, nil) |
| 720 | publishHABox(t, cx, "box-ie-default", true, nil) |
| 721 | status, hdr, body := rawReq(t, http.MethodPost, cx.ts.URL+d.path, cx.bearer, d.body) |
| 722 | if status != http.StatusServiceUnavailable || !strings.Contains(body, noCapacityRefusal("ie")) || hdr.Get("Retry-After") != "60" { |
| 723 | t.Fatalf("%s: %d Retry-After %q %s, want the no-room 503 with Retry-After 60", d.name, status, hdr.Get("Retry-After"), body) |
| 724 | } |
| 725 | } |
| 726 | cx := haClaimCase(t, "") |
| 727 | // The browser's attach door renders the same refusal. |
| 728 | req := httptest.NewRequest(http.MethodPost, "/a/bab/apps/titogo/domains/attach", nil) |
| 729 | rec := httptest.NewRecorder() |
| 730 | cx.srv.renderDoorRefusal(rec, req, claimResponseRefusal(&noCapacityError{Region: "ie"})) |
| 731 | if rec.Code != http.StatusServiceUnavailable || rec.Header().Get("Retry-After") != "60" { |
| 732 | t.Fatalf("browser refusal %d Retry-After %q", rec.Code, rec.Header().Get("Retry-After")) |
| 733 | } |
| 734 | // Other refusals are as they were. |
| 735 | rec = httptest.NewRecorder() |
| 736 | writeClaimResponseError(rec, hostclaim.ErrConflict) |
| 737 | if rec.Header().Get("Retry-After") != "2" { |
| 738 | t.Fatalf("busy refusal Retry-After %q, want 2", rec.Header().Get("Retry-After")) |
| 739 | } |
| 740 | rec = httptest.NewRecorder() |
| 741 | writeClaimResponseError(rec, hostops.ErrEvidenceIncomplete) |
| 742 | if rec.Code != http.StatusServiceUnavailable || rec.Header().Get("Retry-After") != "2" { |
| 743 | t.Fatalf("unavailable refusal %d Retry-After %q, want 503 and 2", rec.Code, rec.Header().Get("Retry-After")) |
| 744 | } |
| 745 | rec = httptest.NewRecorder() |
| 746 | cx.srv.renderDoorRefusal(rec, req, &doorRefusal{http.StatusServiceUnavailable, claimResponseUnavailable}) |
| 747 | if rec.Header().Get("Retry-After") != "" { |
| 748 | t.Fatalf("browser unavailable refusal Retry-After %q, want none (main's)", rec.Header().Get("Retry-After")) |
| 749 | } |
| 750 | } |
| 751 | |
amadan