| 1 | package hostops |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/ed25519" |
| 6 | "crypto/rand" |
| 7 | "crypto/sha256" |
| 8 | "encoding/hex" |
| 9 | "encoding/json" |
| 10 | "errors" |
| 11 | "strings" |
| 12 | "testing" |
| 13 | "time" |
| 14 | |
| 15 | "github.com/carlosframework/platform/internal/boxchoice" |
| 16 | "github.com/carlosframework/platform/internal/hostactors" |
| 17 | "github.com/carlosframework/platform/internal/hostclaim" |
| 18 | "github.com/carlosframework/platform/internal/park" |
| 19 | "github.com/carlosframework/platform/internal/release" |
| 20 | "github.com/carlosframework/platform/internal/termfence" |
| 21 | ) |
| 22 | |
| 23 | func chooserFixture(t *testing.T, boxes ...string) (ExecutorChooser, release.Store, *boxchoice.Snapshot, map[string]boxchoice.Load) { |
| 24 | t.Helper() |
| 25 | return chooserFixtureWith(t, nil, boxes...) |
| 26 | } |
| 27 | |
| 28 | // chooserFixtureWith is chooserFixture with edit applied to the roster |
| 29 | // before it is signed (another namespace or pool for some box). |
| 30 | func chooserFixtureWith(t *testing.T, edit func(*hostactors.Roster), boxes ...string) (ExecutorChooser, release.Store, *boxchoice.Snapshot, map[string]boxchoice.Load) { |
| 31 | t.Helper() |
| 32 | s := release.NewDirStore(t.TempDir()) |
| 33 | pub, key, err := ed25519.GenerateKey(rand.Reader) |
| 34 | if err != nil { |
| 35 | t.Fatal(err) |
| 36 | } |
| 37 | roster := hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 1, |
| 38 | Regions: map[string]hostactors.Region{"ie": {Account: "123", AWSRegion: "eu-west-1"}}, |
| 39 | Boxes: map[string]hostactors.Box{}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}} |
| 40 | snap := &boxchoice.Snapshot{Complete: true} |
| 41 | yes := true |
| 42 | for i, ref := range boxes { |
| 43 | roster.Boxes[ref] = hostactors.Box{Enrollment: strings.Repeat(string("abcdef"[i]), 32), Namespace: "platform", BoxID: ref, Label: ref, Region: "ie", Pool: "default", |
| 44 | Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-" + ref}, EvidenceSHA256: strings.Repeat("c", 64), State: "active"} |
| 45 | snap.Boxes = append(snap.Boxes, boxchoice.Box{BoxID: ref, EIP: "203.0.113.1", Region: "eu-west-1", Placement: "ie", Fresh: true, |
| 46 | HibernationConfigured: &yes, ServesSharedPool: &yes, AcceptsInbound: &yes, SecretsKeyFP: "fp", Capabilities: []string{boxchoice.Capability}, |
| 47 | // Equal, roomy headroom from a file a minute old: spec 2's |
| 48 | // policy (§7) admits every box, and the choice falls through |
| 49 | // to the hash unless a test changes it. |
| 50 | Headroom: &boxchoice.Headroom{MemFreeMiB: 4000, MemTotalMiB: 8000, DiskFreeMiB: 50000, DiskTotalMiB: 100000}, |
| 51 | Generation: "2026-10-07T08:59:00Z"}) |
| 52 | } |
| 53 | if edit != nil { |
| 54 | edit(&roster) |
| 55 | } |
| 56 | wire, err := release.SignRootEnvelope(release.RootSigner(key), roster) |
| 57 | if err != nil { |
| 58 | t.Fatal(err) |
| 59 | } |
| 60 | if _, err := s.Put(context.Background(), hostactors.Key, wire, ""); err != nil { |
| 61 | t.Fatal(err) |
| 62 | } |
| 63 | loads := map[string]boxchoice.Load{} |
| 64 | x := ExecutorChooser{Directory: hostactors.Directory{Store: s, PublicKey: pub, Bucket: "bucket"}, SecretsKeyFP: "fp", Namespace: "platform", |
| 65 | Edges: func(context.Context) (boxchoice.Snapshot, error) { return *snap, nil }, |
| 66 | Loads: func(context.Context) (map[string]boxchoice.Load, error) { return loads, nil }, |
| 67 | Now: func() time.Time { return time.Date(2026, 10, 7, 9, 0, 0, 0, time.UTC) }} |
| 68 | return x, s, snap, loads |
| 69 | } |
| 70 | |
| 71 | func chooserRecord() Record { |
| 72 | return Record{Holder: hostclaim.NewHolder{Family: "instance", Account: "bab", App: "jam", Host: "shop.bab.oncarlos.com", Region: "ie"}, |
| 73 | Body: []byte(`{"host":"shop.bab.oncarlos.com","account":"bab","app":"jam","region":"ie"}`)} |
| 74 | } |
| 75 | |
| 76 | func TestExecutorChooserSpreadsThenPrefersMemoryThenHashes(t *testing.T) { |
| 77 | ctx := context.Background() |
| 78 | x, _, snap, loads := chooserFixture(t, "box-a", "box-b") |
| 79 | absent := hostclaim.Observed{Host: "shop.bab.oncarlos.com", Absent: true} |
| 80 | first, err := x.Choose(ctx, absent, chooserRecord()) |
| 81 | if err != nil { |
| 82 | t.Fatal(err) |
| 83 | } |
| 84 | other := map[string]string{"box-a": "box-b", "box-b": "box-a"}[first.Box] |
| 85 | if first.Enrollment != map[string]string{"box-a": strings.Repeat("a", 32), "box-b": strings.Repeat("b", 32)}[first.Box] { |
| 86 | t.Fatalf("enrollment not the roster's: %+v", first) |
| 87 | } |
| 88 | // The hash winner already runs this app: spread prefers the other box. |
| 89 | loads[first.Box] = boxchoice.Load{Hosts: 1, Apps: map[string]int{"bab/jam": 1}} |
| 90 | if got, _ := x.Choose(ctx, absent, chooserRecord()); got.Box != other { |
| 91 | t.Fatalf("soft spread: got %s, want %s", got.Box, other) |
| 92 | } |
| 93 | // Both run the app: the most free memory wins (spec 2 §7), whatever |
| 94 | // the host counts say. |
| 95 | loads[other] = boxchoice.Load{Hosts: 5, Apps: map[string]int{"bab/jam": 1}} |
| 96 | for i := range snap.Boxes { |
| 97 | if snap.Boxes[i].BoxID == other { |
| 98 | snap.Boxes[i].Headroom.MemFreeMiB = 6000 |
| 99 | } |
| 100 | } |
| 101 | if got, _ := x.Choose(ctx, absent, chooserRecord()); got.Box != other { |
| 102 | t.Fatalf("headroom: got %s, want %s", got.Box, other) |
| 103 | } |
| 104 | } |
| 105 | |
| 106 | // withRole sets one box's roster role before the roster is signed. |
| 107 | func withRole(ref, role string) func(*hostactors.Roster) { |
| 108 | return func(r *hostactors.Roster) { |
| 109 | b := r.Boxes[ref] |
| 110 | b.Role = role |
| 111 | r.Boxes[ref] = b |
| 112 | } |
| 113 | } |
| 114 | |
| 115 | // Region HA spec 2 §7: the chooser, and so creates and relocation |
| 116 | // targets, applies the roster's roles and the policy's thresholds after |
| 117 | // Eligible. Permits answers the same set, and Capacity counts it. |
| 118 | func TestExecutorChooserAppliesRolesAndHeadroom(t *testing.T) { |
| 119 | ctx := context.Background() |
| 120 | absent := hostclaim.Observed{Host: "shop.bab.oncarlos.com", Absent: true} |
| 121 | cases := []struct { |
| 122 | name string |
| 123 | roster func(*hostactors.Roster) |
| 124 | edges func(*boxchoice.Box) |
| 125 | }{ |
| 126 | {"legacy role", withRole("box-a", hostactors.RoleLegacy), nil}, |
| 127 | {"platform role", withRole("box-a", hostactors.RolePlatform), nil}, |
| 128 | {"memory under 15%", nil, func(b *boxchoice.Box) { b.Headroom.MemFreeMiB = 1199 }}, |
| 129 | {"disk under 10%", nil, func(b *boxchoice.Box) { b.Headroom.DiskFreeMiB = 9999 }}, |
| 130 | {"edges file 16 minutes old", nil, func(b *boxchoice.Box) { b.Generation = "2026-10-07T08:44:00Z" }}, |
| 131 | {"no headroom", nil, func(b *boxchoice.Box) { b.Headroom = nil }}, |
| 132 | } |
| 133 | for _, tc := range cases { |
| 134 | t.Run(tc.name, func(t *testing.T) { |
| 135 | x, _, snap, _ := chooserFixtureWith(t, tc.roster, "box-a", "box-b") |
| 136 | for i := range snap.Boxes { |
| 137 | if snap.Boxes[i].BoxID == "box-a" && tc.edges != nil { |
| 138 | tc.edges(&snap.Boxes[i]) |
| 139 | } |
| 140 | } |
| 141 | for range 4 { |
| 142 | got, err := x.Choose(ctx, absent, chooserRecord()) |
| 143 | if err != nil || got.Box != "box-b" { |
| 144 | t.Fatalf("chose %+v %v, want box-b", got, err) |
| 145 | } |
| 146 | } |
| 147 | if err := x.Permits(ctx, absent, chooserRecord(), "box-a"); !errors.Is(err, ErrNoEligibleBox) { |
| 148 | t.Fatalf("Permits(box-a) = %v, want ErrNoEligibleBox", err) |
| 149 | } |
| 150 | if err := x.Permits(ctx, absent, chooserRecord(), "box-b"); err != nil { |
| 151 | t.Fatalf("Permits(box-b) = %v", err) |
| 152 | } |
| 153 | if n, aws, err := x.Capacity(ctx, "ie"); err != nil || n != 1 || aws != "eu-west-1" { |
| 154 | t.Fatalf("Capacity = %d in %q, %v; want 1 in eu-west-1", n, aws, err) |
| 155 | } |
| 156 | if boxes, err := x.PlaceableBoxes(ctx, "ie"); err != nil || len(boxes) != 1 || boxes[0] != "box-b" { |
| 157 | t.Fatalf("PlaceableBoxes = %v, %v; want [box-b]", boxes, err) |
| 158 | } |
| 159 | }) |
| 160 | } |
| 161 | x, _, _, _ := chooserFixture(t, "box-a", "box-b") |
| 162 | if n, aws, err := x.Capacity(ctx, "ie"); err != nil || n != 2 || aws != "eu-west-1" { |
| 163 | t.Fatalf("Capacity with both placeable = %d in %q, %v; want 2 in eu-west-1", n, aws, err) |
| 164 | } |
| 165 | if _, _, err := x.Capacity(ctx, "us"); !errors.Is(err, ErrContract) { |
| 166 | t.Fatalf("Capacity of a region the roster lacks = %v, want ErrContract", err) |
| 167 | } |
| 168 | } |
| 169 | |
| 170 | // Round 18 finding 2: with equal loads the winner is the greatest |
| 171 | // SHA-256 of host + "\x00" + box id (claim P5-D2), computed here |
| 172 | // independently of boxchoice. The two hostnames are chosen so that their |
| 173 | // winners differ: a chooser that hashed anything but the record's host |
| 174 | // would get at least one of them wrong. |
| 175 | func TestExecutorChooserHashesTheRecordsHostOnEqualLoads(t *testing.T) { |
| 176 | ctx := context.Background() |
| 177 | winner := func(host string) string { |
| 178 | a := sha256.Sum256([]byte(host + "\x00box-a")) |
| 179 | b := sha256.Sum256([]byte(host + "\x00box-b")) |
| 180 | if hex.EncodeToString(a[:]) > hex.EncodeToString(b[:]) { |
| 181 | return "box-a" |
| 182 | } |
| 183 | return "box-b" |
| 184 | } |
| 185 | hosts := []string{"shop.bab.oncarlos.com", "cafe.bab.oncarlos.com"} |
| 186 | if winner(hosts[0]) == winner(hosts[1]) { |
| 187 | t.Fatal("the fixture's hostnames must choose opposite boxes") |
| 188 | } |
| 189 | for _, host := range hosts { |
| 190 | x, _, _, _ := chooserFixture(t, "box-a", "box-b") |
| 191 | r := chooserRecord() |
| 192 | r.Holder.Host = host |
| 193 | got, err := x.Choose(ctx, hostclaim.Observed{Host: host, Absent: true}, r) |
| 194 | if err != nil || got.Box != winner(host) { |
| 195 | t.Fatalf("%s: chose %+v %v, want %s", host, got, err, winner(host)) |
| 196 | } |
| 197 | } |
| 198 | } |
| 199 | |
| 200 | func TestExecutorChooserNeverNamesAFencedLostOrUnknownBox(t *testing.T) { |
| 201 | ctx := context.Background() |
| 202 | absent := hostclaim.Observed{Host: "shop.bab.oncarlos.com", Absent: true} |
| 203 | t.Run("lost", func(t *testing.T) { |
| 204 | x, _, _, _ := chooserFixture(t, "box-a") |
| 205 | obs := hostclaim.Observed{Host: "shop.bab.oncarlos.com", Claim: &hostclaim.Claim{Lost: []string{"box-a"}}} |
| 206 | if _, err := x.Choose(ctx, obs, chooserRecord()); !errors.Is(err, ErrNoEligibleBox) { |
| 207 | t.Fatalf("chose a lost box: %v", err) |
| 208 | } |
| 209 | }) |
| 210 | t.Run("owing a local retirement", func(t *testing.T) { |
| 211 | // Spec 1 §2: a box whose old installation of the host is not yet |
| 212 | // retired (park-local) is never chosen; the clean one is. |
| 213 | x, _, _, _ := chooserFixture(t, "box-a", "box-b") |
| 214 | obs := hostclaim.Observed{Host: "shop.bab.oncarlos.com", Claim: &hostclaim.Claim{Obligations: []hostclaim.Obligation{{ID: "old", Op: "old-op", Kind: "park-local", Box: "box-a", Retirement: "r", Owner: "box:box-a"}}}} |
| 215 | for range 4 { |
| 216 | if got, err := x.Choose(ctx, obs, chooserRecord()); err != nil || got.Box != "box-b" { |
| 217 | t.Fatalf("got %+v %v, want the box with no chore", got, err) |
| 218 | } |
| 219 | } |
| 220 | if err := x.Permits(ctx, obs, chooserRecord(), "box-a"); !errors.Is(err, ErrNoEligibleBox) { |
| 221 | t.Fatalf("permitted a box owing park-local: %v", err) |
| 222 | } |
| 223 | }) |
| 224 | t.Run("fenced", func(t *testing.T) { |
| 225 | x, s, _, _ := chooserFixture(t, "box-a", "box-w") |
| 226 | f := termfence.Record{V: 1, Box: "box-a", Enrollment: strings.Repeat("a", 32), State: "terminated", |
| 227 | Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-box-a"}, |
| 228 | ObservedAt: "2026-10-07T08:00:00Z", Via: termfence.ViaDescribe, RequestID: "req", |
| 229 | WrittenBy: termfence.WrittenBy{Box: "box-w", Enrollment: strings.Repeat("b", 32)}} |
| 230 | raw, _ := json.Marshal(f) |
| 231 | if _, err := s.Put(ctx, "control/box-fences/"+strings.Repeat("a", 32)+".json", raw, ""); err != nil { |
| 232 | t.Fatal(err) |
| 233 | } |
| 234 | got, err := x.Choose(ctx, absent, chooserRecord()) |
| 235 | if err != nil || got.Box != "box-w" { |
| 236 | t.Fatalf("got %+v %v, want the unfenced box", got, err) |
| 237 | } |
| 238 | }) |
| 239 | t.Run("an edges file for a box the roster does not hold", func(t *testing.T) { |
| 240 | x, _, snap, _ := chooserFixture(t) |
| 241 | yes := true |
| 242 | snap.Boxes = append(snap.Boxes, boxchoice.Box{BoxID: "stray", EIP: "203.0.113.9", Region: "eu-west-1", Placement: "ie", Fresh: true, |
| 243 | HibernationConfigured: &yes, ServesSharedPool: &yes, AcceptsInbound: &yes, SecretsKeyFP: "fp", Capabilities: []string{boxchoice.Capability}}) |
| 244 | if _, err := x.Choose(ctx, absent, chooserRecord()); !errors.Is(err, ErrNoEligibleBox) { |
| 245 | t.Fatalf("chose an unenrolled box: %v", err) |
| 246 | } |
| 247 | }) |
| 248 | t.Run("an incomplete snapshot", func(t *testing.T) { |
| 249 | // Region HA 2a Task 11 ruling: a partial read is not "no room". |
| 250 | x, _, snap, _ := chooserFixture(t, "box-a") |
| 251 | snap.Complete = false |
| 252 | if _, err := x.Choose(ctx, absent, chooserRecord()); !errors.Is(err, ErrEvidenceIncomplete) || errors.Is(err, ErrNoEligibleBox) { |
| 253 | t.Fatalf("chose from a partial read: %v, want ErrEvidenceIncomplete", err) |
| 254 | } |
| 255 | }) |
| 256 | t.Run("an unreadable fence", func(t *testing.T) { |
| 257 | x, s, _, _ := chooserFixture(t, "box-a") |
| 258 | if _, err := s.Put(ctx, "control/box-fences/"+strings.Repeat("a", 32)+".json", []byte("{"), ""); err != nil { |
| 259 | t.Fatal(err) |
| 260 | } |
| 261 | if _, err := x.Choose(ctx, absent, chooserRecord()); err == nil || errors.Is(err, ErrNoEligibleBox) { |
| 262 | t.Fatalf("an unreadable fence must refuse, not skip: %v", err) |
| 263 | } |
| 264 | }) |
| 265 | } |
| 266 | |
| 267 | func TestExecutorChooserPermitsOnlyTheEligibleSet(t *testing.T) { |
| 268 | ctx := context.Background() |
| 269 | absent := hostclaim.Observed{Host: "shop.bab.oncarlos.com", Absent: true} |
| 270 | for _, tc := range []struct { |
| 271 | name string |
| 272 | change func(*boxchoice.Box) |
| 273 | }{ |
| 274 | {"a stale edges file", func(b *boxchoice.Box) { b.Fresh = false }}, |
| 275 | {"no region-ha-1 capability", func(b *boxchoice.Box) { b.Capabilities = nil }}, |
| 276 | {"another secrets key", func(b *boxchoice.Box) { b.SecretsKeyFP = "other" }}, |
| 277 | {"a draining box", func(b *boxchoice.Box) { b.Draining = true }}, |
| 278 | } { |
| 279 | t.Run(tc.name, func(t *testing.T) { |
| 280 | x, _, snap, _ := chooserFixture(t, "box-a", "box-b") |
| 281 | if err := x.Permits(ctx, absent, chooserRecord(), "box-a"); err != nil { |
| 282 | t.Fatal(err) |
| 283 | } |
| 284 | tc.change(&snap.Boxes[0]) |
| 285 | if err := x.Permits(ctx, absent, chooserRecord(), "box-a"); !errors.Is(err, ErrNoEligibleBox) { |
| 286 | t.Fatalf("permitted: %v", err) |
| 287 | } |
| 288 | if err := x.Permits(ctx, absent, chooserRecord(), "box-b"); err != nil { |
| 289 | t.Fatalf("the other box: %v", err) |
| 290 | } |
| 291 | }) |
| 292 | } |
| 293 | t.Run("an unknown box", func(t *testing.T) { |
| 294 | x, _, _, _ := chooserFixture(t, "box-a") |
| 295 | if err := x.Permits(ctx, absent, chooserRecord(), "box-z"); !errors.Is(err, ErrNoEligibleBox) { |
| 296 | t.Fatalf("permitted: %v", err) |
| 297 | } |
| 298 | }) |
| 299 | } |
| 300 | |
| 301 | func TestClaimLoadsCountsExecutorsPerBoxAndApp(t *testing.T) { |
| 302 | ctx := context.Background() |
| 303 | s := release.NewDirStore(t.TempDir()) |
| 304 | created := time.Date(2026, 10, 4, 1, 0, 0, 0, time.UTC) |
| 305 | for i, tc := range []struct{ host, app, box string }{ |
| 306 | {"a.bab.oncarlos.com", "jam", "box-a"}, {"b.bab.oncarlos.com", "jam", "box-a"}, {"c.bab.oncarlos.com", "tea", "box-b"}, |
| 307 | } { |
| 308 | nh := hostclaim.NewHolder{Family: "instance", Key: "apps/bab/" + tc.app + "/instances/" + tc.host + ".json", Account: "bab", App: tc.app, Host: tc.host, Region: "ie", RecordRegion: "ie", CreatedAt: created} |
| 309 | body := []byte(`{"host":"` + tc.host + `","account":"bab","app":"` + tc.app + `","region":"ie","created_at":"2026-10-04T01:00:00Z"}`) |
| 310 | o, _ := hostclaim.Read(ctx, s, tc.host) |
| 311 | o, err := hostclaim.Update(ctx, s, o, hostclaim.BeginCreateOn(nh, body, "c", "console", created, hostclaim.ExecutorSelection{Box: tc.box, Enrollment: "e"})) |
| 312 | if err != nil { |
| 313 | t.Fatal(err) |
| 314 | } |
| 315 | if i < 2 { // the third is still creating: it has no executor yet |
| 316 | if _, err := hostclaim.Update(ctx, s, o, hostclaim.CompleteCreate("c", tc.box)); err != nil { |
| 317 | t.Fatal(err) |
| 318 | } |
| 319 | } |
| 320 | } |
| 321 | if _, err := s.Put(ctx, "control/hosts/broken.bab.oncarlos.com.json", []byte("{"), ""); err != nil { |
| 322 | t.Fatal(err) |
| 323 | } |
| 324 | loads, err := ClaimLoads(ctx, s) |
| 325 | if err != nil { |
| 326 | t.Fatal(err) |
| 327 | } |
| 328 | if loads["box-a"].Hosts != 2 || loads["box-a"].Apps["bab/jam"] != 2 || loads["box-b"].Hosts != 0 { |
| 329 | t.Fatalf("loads: %+v", loads) |
| 330 | } |
| 331 | } |
| 332 | |
| 333 | func TestProductionAuthoritySelectsOnlyWithAChooser(t *testing.T) { |
| 334 | // No chooser (CARLOS_REGION_HA off): no selection and no error, so the |
| 335 | // create is today's. |
| 336 | sel, selected, err := (ProductionAuthority{}).SelectExecutor(context.Background(), hostclaim.Observed{}, Record{}) |
| 337 | if err != nil || selected || sel != (hostclaim.ExecutorSelection{}) { |
| 338 | t.Fatalf("no chooser: %+v %v %v", sel, selected, err) |
| 339 | } |
| 340 | // With a chooser the selection is the chooser's, read from its |
| 341 | // evidence: box-a is draining, so box-b with its roster enrollment; |
| 342 | // with both draining, no box and an error, never a selection. |
| 343 | x, _, snap, _ := chooserFixture(t, "box-a", "box-b") |
| 344 | snap.Boxes[0].Draining = true |
| 345 | absent := hostclaim.Observed{Host: "shop.bab.oncarlos.com", Absent: true} |
| 346 | sel, selected, err = (ProductionAuthority{Executors: &x, Namespace: "platform"}).SelectExecutor(context.Background(), absent, chooserRecord()) |
| 347 | if err != nil || !selected || sel.Box != "box-b" || sel.Enrollment != strings.Repeat("b", 32) { |
| 348 | t.Fatalf("with a chooser: %+v %v %v", sel, selected, err) |
| 349 | } |
| 350 | snap.Boxes[1].Draining = true |
| 351 | if sel, selected, err = (ProductionAuthority{Executors: &x, Namespace: "platform"}).SelectExecutor(context.Background(), absent, chooserRecord()); !errors.Is(err, ErrNoEligibleBox) || selected { |
| 352 | t.Fatalf("no eligible box: %+v %v %v", sel, selected, err) |
| 353 | } |
| 354 | } |
| 355 | |
| 356 | // Main's production create (claim Plan 5's StartCreate) selects through |
| 357 | // selectExecutor. Without a chooser it is Plan 5's choice over the roster |
| 358 | // alone; with one the choice is the chooser's, read from the edges |
| 359 | // evidence, and Plan 5's scope (this console's namespace, the record's |
| 360 | // pool) still binds it: a chosen box outside that scope refuses. |
| 361 | func TestProductionCreateSelectsThroughTheChooser(t *testing.T) { |
| 362 | ctx := context.Background() |
| 363 | x, s, snap, _ := chooserFixture(t, "box-a", "box-b") |
| 364 | r := chooserRecord() |
| 365 | absent := hostclaim.Observed{Host: r.Holder.Host, Absent: true} |
| 366 | a := ProductionAuthority{Store: s, Directory: x.Directory, Namespace: "platform", Now: x.Now} |
| 367 | off, _, pool, err := a.selectExecutor(ctx, absent, r) |
| 368 | if err != nil || pool != "default" { |
| 369 | t.Fatalf("no chooser: %s %s %v", off, pool, err) |
| 370 | } |
| 371 | // The roster-only choice is draining: Plan 5 does not read the edges, |
| 372 | // so it still names it; the chooser names the other box. |
| 373 | other := map[string]string{"box-a": "box-b", "box-b": "box-a"}[off] |
| 374 | for i := range snap.Boxes { |
| 375 | snap.Boxes[i].Draining = snap.Boxes[i].BoxID == off |
| 376 | } |
| 377 | if again, _, _, err := a.selectExecutor(ctx, absent, r); err != nil || again != off { |
| 378 | t.Fatalf("no chooser, after a drain: %s %v, want %s", again, err, off) |
| 379 | } |
| 380 | a.Executors = &x |
| 381 | box, enrollment, pool, err := a.selectExecutor(ctx, absent, r) |
| 382 | if err != nil || box != other || enrollment != strings.Repeat(map[string]string{"box-a": "a", "box-b": "b"}[other], 32) || pool != "default" { |
| 383 | t.Fatalf("with a chooser: %s %s %s %v, want %s", box, enrollment, pool, err, other) |
| 384 | } |
| 385 | // Another console namespace: the chooser chooses within it, and it |
| 386 | // holds no box. |
| 387 | a.Namespace = "elsewhere" |
| 388 | if box, _, _, err := a.selectExecutor(ctx, absent, r); !errors.Is(err, ErrNoEligibleBox) { |
| 389 | t.Fatalf("a box outside the namespace: %s %v", box, err) |
| 390 | } |
| 391 | // Every box draining: the chooser's refusal, never Plan 5's fallback. |
| 392 | a.Namespace = "platform" |
| 393 | snap.Boxes[0].Draining, snap.Boxes[1].Draining = true, true |
| 394 | if box, _, _, err := a.selectExecutor(ctx, absent, r); !errors.Is(err, ErrNoEligibleBox) { |
| 395 | t.Fatalf("no eligible box: %s %v", box, err) |
| 396 | } |
| 397 | } |
| 398 | |
| 399 | // Fix round 1 (review Important 1): the chooser chooses within claim Plan |
| 400 | // 5's scope, so a region that also holds another namespace's or another |
| 401 | // pool's boxes places on the in-scope eligible box, never refusing because |
| 402 | // an out-of-scope box would have won. |
| 403 | func TestProductionCreateWithAChooserPlacesWithinPlan5sScope(t *testing.T) { |
| 404 | ctx := context.Background() |
| 405 | absent := hostclaim.Observed{Host: "shop.bab.oncarlos.com", Absent: true} |
| 406 | // The console's chooser carries no scope of its own: the authority |
| 407 | // supplies its namespace and the record's pool. |
| 408 | authority := func(x *ExecutorChooser, s release.Store) ProductionAuthority { |
| 409 | x.Namespace, x.Pool = "", "" |
| 410 | return ProductionAuthority{Store: s, Directory: x.Directory, Namespace: "platform", Now: x.Now, Executors: x} |
| 411 | } |
| 412 | t.Run("another namespace's box would win on load", func(t *testing.T) { |
| 413 | x, s, _, loads := chooserFixtureWith(t, func(r *hostactors.Roster) { |
| 414 | b := r.Boxes["box-b"] |
| 415 | b.Namespace = "other" |
| 416 | r.Boxes["box-b"] = b |
| 417 | }, "box-a", "box-b") |
| 418 | loads["box-a"] = boxchoice.Load{Hosts: 5, Apps: map[string]int{"bab/jam": 1}} |
| 419 | box, enrollment, pool, err := authority(&x, s).selectExecutor(ctx, absent, chooserRecord()) |
| 420 | if err != nil || box != "box-a" || enrollment != strings.Repeat("a", 32) || pool != "default" { |
| 421 | t.Fatalf("got %s %s %s %v, want the in-scope box-a", box, enrollment, pool, err) |
| 422 | } |
| 423 | }) |
| 424 | t.Run("a named pool", func(t *testing.T) { |
| 425 | // box-b serves only pool acme: its edges file does not serve the |
| 426 | // shared pool. box-a is the shared pool's, and idle. |
| 427 | x, s, snap, loads := chooserFixtureWith(t, func(r *hostactors.Roster) { |
| 428 | b := r.Boxes["box-b"] |
| 429 | b.Pool = "acme" |
| 430 | r.Boxes["box-b"] = b |
| 431 | }, "box-a", "box-b") |
| 432 | no := false |
| 433 | snap.Boxes[1].ServesSharedPool = &no |
| 434 | loads["box-b"] = boxchoice.Load{Hosts: 5} |
| 435 | named := chooserRecord() |
| 436 | named.Body = []byte(`{"host":"shop.bab.oncarlos.com","account":"bab","app":"jam","region":"ie","pool":"acme"}`) |
| 437 | box, enrollment, pool, err := authority(&x, s).selectExecutor(ctx, absent, named) |
| 438 | if err != nil || box != "box-b" || enrollment != strings.Repeat("b", 32) || pool != "acme" { |
| 439 | t.Fatalf("named pool: got %s %s %s %v, want box-b in acme", box, enrollment, pool, err) |
| 440 | } |
| 441 | // The shared pool still places on its own box, whatever the load. |
| 442 | loads["box-a"] = boxchoice.Load{Hosts: 50} |
| 443 | if box, _, pool, err := authority(&x, s).selectExecutor(ctx, absent, chooserRecord()); err != nil || box != "box-a" || pool != "default" { |
| 444 | t.Fatalf("shared pool: got %s %s %v, want box-a", box, pool, err) |
| 445 | } |
| 446 | // The named pool's box must still be eligible on its own evidence. |
| 447 | snap.Boxes[1].Draining = true |
| 448 | if box, _, _, err := authority(&x, s).selectExecutor(ctx, absent, named); !errors.Is(err, ErrNoEligibleBox) { |
| 449 | t.Fatalf("a draining named-pool box: %s %v", box, err) |
| 450 | } |
| 451 | }) |
| 452 | t.Run("an unreadable fence on another namespace's box", func(t *testing.T) { |
| 453 | x, s, _, _ := chooserFixtureWith(t, func(r *hostactors.Roster) { |
| 454 | b := r.Boxes["box-b"] |
| 455 | b.Namespace = "other" |
| 456 | r.Boxes["box-b"] = b |
| 457 | }, "box-a", "box-b") |
| 458 | if _, err := s.Put(ctx, "control/box-fences/"+strings.Repeat("b", 32)+".json", []byte("{"), ""); err != nil { |
| 459 | t.Fatal(err) |
| 460 | } |
| 461 | if box, _, _, err := authority(&x, s).selectExecutor(ctx, absent, chooserRecord()); err != nil || box != "box-a" { |
| 462 | t.Fatalf("got %s %v, want box-a: an out-of-scope fence never blocks", box, err) |
| 463 | } |
| 464 | // In scope, the same unreadable fence still refuses (absence is |
| 465 | // never evidence). |
| 466 | if _, err := s.Put(ctx, "control/box-fences/"+strings.Repeat("a", 32)+".json", []byte("{"), ""); err != nil { |
| 467 | t.Fatal(err) |
| 468 | } |
| 469 | if _, _, _, err := authority(&x, s).selectExecutor(ctx, absent, chooserRecord()); err == nil || errors.Is(err, ErrNoEligibleBox) { |
| 470 | t.Fatalf("an unreadable in-scope fence must refuse: %v", err) |
| 471 | } |
| 472 | }) |
| 473 | } |
| 474 | |
| 475 | // Main's home-region parks: a record placed in the home with no home name |
| 476 | // configured names it park.HomeRegion. With the home binding (HomeName, |
| 477 | // the create door's ProductionAuthority.HomeRegion) the chooser places it |
| 478 | // in the home's roster region, as main's StartCreate maps it; with none, |
| 479 | // park.HomeRegion is no roster region and the create is refused. |
| 480 | func TestExecutorChooserPlacesAHomeRecordUnderTheHomeBinding(t *testing.T) { |
| 481 | ctx := context.Background() |
| 482 | r := chooserRecord() |
| 483 | r.Holder.Region = park.HomeRegion |
| 484 | absent := hostclaim.Observed{Host: r.Holder.Host, Absent: true} |
| 485 | x, _, _, _ := chooserFixture(t, "box-a") |
| 486 | x.HomeName = "ie" |
| 487 | if got, err := x.Choose(ctx, absent, r); err != nil || got.Box != "box-a" { |
| 488 | t.Fatalf("bound: %+v %v", got, err) |
| 489 | } |
| 490 | x.HomeName = "" |
| 491 | if _, err := x.Choose(ctx, absent, r); !errors.Is(err, ErrContract) { |
| 492 | t.Fatalf("unbound: %v", err) |
| 493 | } |
| 494 | } |
| 495 | |
| 496 | // Region HA spec 2 §7 (H27): a door that chose before registering holds |
| 497 | // the create to its choice, for that host only; Plan 5's scope still |
| 498 | // binds it (TestProductionCreateSelectsThroughTheChooser). |
| 499 | func TestSelectExecutorHonoursTheDoorsChoice(t *testing.T) { |
| 500 | ctx := context.Background() |
| 501 | x, _, snap, _ := chooserFixture(t, "box-a", "box-b") |
| 502 | a := ProductionAuthority{Namespace: "platform", Executors: &x} |
| 503 | r := chooserRecord() |
| 504 | absent := hostclaim.Observed{Host: r.Holder.Host, Absent: true} |
| 505 | chosen := hostclaim.ExecutorSelection{Box: "box-b", Enrollment: strings.Repeat("b", 32)} |
| 506 | // Every box is now ineligible, yet the door's earlier choice stands. |
| 507 | for i := range snap.Boxes { |
| 508 | snap.Boxes[i].Draining = true |
| 509 | } |
| 510 | sel, ok, err := a.SelectExecutor(WithChosenExecutor(ctx, r.Holder.Host, chosen), absent, r) |
| 511 | if err != nil || !ok || sel != chosen { |
| 512 | t.Fatalf("with the door's choice: %+v %v %v", sel, ok, err) |
| 513 | } |
| 514 | // Another host's choice is not this create's. |
| 515 | if _, _, err := a.SelectExecutor(WithChosenExecutor(ctx, "other.bab.oncarlos.com", chosen), absent, r); !errors.Is(err, ErrNoEligibleBox) { |
| 516 | t.Fatalf("another host's choice was used: %v", err) |
| 517 | } |
| 518 | // No chooser (region HA off): no selection, whatever the context says. |
| 519 | if _, ok, err := (ProductionAuthority{}).SelectExecutor(WithChosenExecutor(ctx, r.Holder.Host, chosen), absent, r); ok || err != nil { |
| 520 | t.Fatalf("region HA off selected: %v %v", ok, err) |
| 521 | } |
| 522 | } |
| 523 | |
| 524 | // Region HA 2a Task 11 ruling: an incomplete placement read (an unreadable |
| 525 | // or malformed edges file, an unread status listing, a fresh box with no |
| 526 | // edges file: boxchoice.Snapshot.Complete false) is ErrEvidenceIncomplete |
| 527 | // from every chooser question, never ErrNoEligibleBox or an empty set, so |
| 528 | // a door never answers or pages a partial read as "no room". The same |
| 529 | // placeable box is chosen, permitted and counted once the read completes. |
| 530 | func TestExecutorChooserAnIncompleteReadIsNotNoRoom(t *testing.T) { |
| 531 | ctx := context.Background() |
| 532 | x, _, snap, _ := chooserFixture(t, "box-a") |
| 533 | absent := hostclaim.Observed{Host: "shop.bab.oncarlos.com", Absent: true} |
| 534 | snap.Complete = false |
| 535 | incomplete := func(what string, err error) { |
| 536 | t.Helper() |
| 537 | if !errors.Is(err, ErrEvidenceIncomplete) || errors.Is(err, ErrNoEligibleBox) { |
| 538 | t.Fatalf("%s on a partial read = %v, want ErrEvidenceIncomplete", what, err) |
| 539 | } |
| 540 | } |
| 541 | _, err := x.Choose(ctx, absent, chooserRecord()) |
| 542 | incomplete("Choose", err) |
| 543 | incomplete("Permits", x.Permits(ctx, absent, chooserRecord(), "box-a")) |
| 544 | boxes, err := x.PlaceableBoxes(ctx, "ie") |
| 545 | incomplete("PlaceableBoxes", err) |
| 546 | if boxes != nil { |
| 547 | t.Fatalf("PlaceableBoxes named %v from a partial read", boxes) |
| 548 | } |
| 549 | _, _, err = x.Capacity(ctx, "ie") |
| 550 | incomplete("Capacity", err) |
| 551 | _, _, err = (ProductionAuthority{Namespace: "platform", Executors: &x}).SelectExecutor(ctx, absent, chooserRecord()) |
| 552 | incomplete("SelectExecutor", err) |
| 553 | |
| 554 | snap.Complete = true |
| 555 | if sel, err := x.Choose(ctx, absent, chooserRecord()); err != nil || sel.Box != "box-a" { |
| 556 | t.Fatalf("Choose on a complete read: %+v %v", sel, err) |
| 557 | } |
| 558 | if err := x.Permits(ctx, absent, chooserRecord(), "box-a"); err != nil { |
| 559 | t.Fatalf("Permits on a complete read: %v", err) |
| 560 | } |
| 561 | if n, aws, err := x.Capacity(ctx, "ie"); err != nil || n != 1 || aws != "eu-west-1" { |
| 562 | t.Fatalf("Capacity on a complete read: %d in %q %v", n, aws, err) |
| 563 | } |
| 564 | // A complete read with no placeable box is still no room. |
| 565 | snap.Boxes[0].Fresh = false |
| 566 | if _, err := x.Choose(ctx, absent, chooserRecord()); !errors.Is(err, ErrNoEligibleBox) { |
| 567 | t.Fatalf("Choose with every box stale: %v, want ErrNoEligibleBox", err) |
| 568 | } |
| 569 | if n, aws, err := x.Capacity(ctx, "ie"); err != nil || n != 0 || aws != "eu-west-1" { |
| 570 | t.Fatalf("Capacity with every box stale: %d in %q %v", n, aws, err) |
| 571 | } |
| 572 | } |
| 573 | |
amadan