| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "net/http" |
| 8 | "strings" |
| 9 | "testing" |
| 10 | "time" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/boxchoice" |
| 13 | "github.com/carlosframework/platform/internal/hostactors" |
| 14 | "github.com/carlosframework/platform/internal/hostclaim" |
| 15 | "github.com/carlosframework/platform/internal/hostops" |
| 16 | "github.com/carlosframework/platform/internal/provision" |
| 17 | ) |
| 18 | |
| 19 | // roomyEdges gives each box's edges file fresh, roomy headroom, then edit |
| 20 | // (when set) changes uk-box's. |
| 21 | func roomyEdges(t *testing.T, fx *consoleFixture, edit func(map[string]any), boxes ...string) { |
| 22 | t.Helper() |
| 23 | ctx := context.Background() |
| 24 | for _, box := range boxes { |
| 25 | key := edgesFilePrefix + box + ".json" |
| 26 | raw, etag, err := fx.rel.Get(ctx, key) |
| 27 | if err != nil { |
| 28 | t.Fatal(err) |
| 29 | } |
| 30 | var m map[string]any |
| 31 | if err := json.Unmarshal(raw, &m); err != nil { |
| 32 | t.Fatal(err) |
| 33 | } |
| 34 | m["headroom"] = map[string]any{"mem_free_mib": 4000, "mem_total_mib": 8000, "disk_free_mib": 50000, "disk_total_mib": 100000, "awake": 0} |
| 35 | m["generation"] = time.Now().UTC().Format(time.RFC3339) |
| 36 | if box == "uk-box" && edit != nil { |
| 37 | edit(m) |
| 38 | } |
| 39 | body, _ := json.Marshal(m) |
| 40 | if _, err := fx.rel.Put(ctx, key, body, etag); err != nil { |
| 41 | t.Fatal(err) |
| 42 | } |
| 43 | } |
| 44 | } |
| 45 | |
| 46 | // haRoster signs a roster naming both regions, every box of fleet, and |
| 47 | // each box's role from roles. |
| 48 | func haRoster(t *testing.T, fx *consoleFixture, roles map[string]string, fleet ...fleetBox) { |
| 49 | t.Helper() |
| 50 | haRosterIn(t, fx, roles, []string{"ie", "uk"}, fleet...) |
| 51 | } |
| 52 | |
| 53 | // haRosterIn is haRoster naming only the regions in codes (the roster's |
| 54 | // Regions), which decides whether it places boxes in a region. |
| 55 | func haRosterIn(t *testing.T, fx *consoleFixture, roles map[string]string, codes []string, fleet ...fleetBox) { |
| 56 | t.Helper() |
| 57 | boxes := map[string]hostactors.Box{} |
| 58 | for _, b := range fleet { |
| 59 | rb := moveRosterBox(b) |
| 60 | rb.Role = roles[b.Box] |
| 61 | boxes[b.Box] = rb |
| 62 | } |
| 63 | regions := map[string]hostactors.Region{} |
| 64 | for _, c := range codes { |
| 65 | regions[c] = map[string]hostactors.Region{ |
| 66 | "ie": {Account: moveTestAccount, AWSRegion: "eu-west-1"}, |
| 67 | "uk": {Account: moveTestAccount, AWSRegion: "eu-west-2"}, |
| 68 | }[c] |
| 69 | } |
| 70 | putMoveRoster(t, fx, hostactors.Roster{Boxes: boxes, Regions: regions}) |
| 71 | } |
| 72 | |
| 73 | // bindRegionHA turns region HA on for fx's claim runtime, with the region |
| 74 | // HA chooser bound as the create door binds it. |
| 75 | func bindRegionHA(fx *consoleFixture) { |
| 76 | fx.srv.cfg.RegionHA = true |
| 77 | a := fx.srv.cfg.ClaimAuthority |
| 78 | a.Namespace = "platform" |
| 79 | a.Executors = fx.srv.executorChooser(a.Directory) |
| 80 | } |
| 81 | |
| 82 | // haMoveFleet is armMoveFleet's ie-box and uk-box (and any more boxes), |
| 83 | // each with fresh, roomy headroom, a signed roster naming both regions |
| 84 | // with each box's role from roles, and, with ha, a region HA chooser. |
| 85 | func haMoveFleet(t *testing.T, ha bool, roles map[string]string, more ...fleetBox) *consoleFixture { |
| 86 | t.Helper() |
| 87 | return haFleetOf(t, ha, roles, []string{"ie", "uk"}, append([]fleetBox{ieBox("ie-box"), ukBox("uk-box")}, more...)...) |
| 88 | } |
| 89 | |
| 90 | // haFleetOf is haMoveFleet for exactly fleet, with a roster naming only |
| 91 | // the regions in codes. |
| 92 | func haFleetOf(t *testing.T, ha bool, roles map[string]string, codes []string, fleet ...fleetBox) *consoleFixture { |
| 93 | t.Helper() |
| 94 | fx := newConsoleFixture(t) |
| 95 | armMoveFleet(t, fx, fleet...) |
| 96 | names := []string{} |
| 97 | for _, b := range fleet { |
| 98 | names = append(names, b.Box) |
| 99 | } |
| 100 | roomyEdges(t, fx, nil, names...) |
| 101 | haRosterIn(t, fx, roles, codes, fleet...) |
| 102 | fx.srv.cfg.HostClaim = true |
| 103 | if ha { |
| 104 | d := hostactors.Directory{Store: fx.rel, PublicKey: fx.pub, Bucket: moveTestBucket} |
| 105 | fx.srv.cfg.ClaimAuthority = &hostops.ProductionAuthority{Directory: d} |
| 106 | bindRegionHA(fx) |
| 107 | } |
| 108 | return fx |
| 109 | } |
| 110 | |
| 111 | // Region HA spec 2 §7 (H27), census ruling region-ha-2a rev 2 item 4: the |
| 112 | // move door's room check only refuses. With room, the selection is the |
| 113 | // one region HA off makes; with none, it is the no-room refusal with its |
| 114 | // alert; and it never narrows the candidates to pick a destination main's |
| 115 | // selection would not. |
| 116 | func TestMoveRoomCheckOnlyRefuses(t *testing.T) { |
| 117 | const host, app = "acme.go.tito.io", "bab/titogo" |
| 118 | ctx := context.Background() |
| 119 | t.Run("room: the selection region HA off makes", func(t *testing.T) { |
| 120 | off, err := haMoveFleet(t, false, nil).srv.selectMoveBoxes(ctx, host, "uk", "ie", app) |
| 121 | if err != nil { |
| 122 | t.Fatal(err) |
| 123 | } |
| 124 | on, err := haMoveFleet(t, true, nil).srv.selectMoveBoxes(ctx, host, "uk", "ie", app) |
| 125 | if err != nil || on != off || on.Target != "uk-box" || on.Source != "ie-box" { |
| 126 | t.Fatalf("region HA on %+v %v, off %+v", on, err, off) |
| 127 | } |
| 128 | }) |
| 129 | t.Run("no room: the only destination is legacy", func(t *testing.T) { |
| 130 | pages := pagesTo(t) |
| 131 | fx := haMoveFleet(t, true, map[string]string{"uk-box": hostactors.RoleLegacy}) |
| 132 | _, err := fx.srv.selectMoveBoxes(ctx, host, "uk", "ie", app) |
| 133 | var full *noCapacityError |
| 134 | if !errors.As(err, &full) || full.Region != "uk" { |
| 135 | t.Fatalf("err = %v, want no room in uk", err) |
| 136 | } |
| 137 | if ref := claimResponseRefusal(err); ref.Status != http.StatusServiceUnavailable || ref.Msg != noCapacityRefusal("uk") { |
| 138 | t.Fatalf("refusal %+v", ref) |
| 139 | } |
| 140 | expectPages(t, pages, noRoomSubject("uk")) |
| 141 | }) |
| 142 | t.Run("a legacy box beside a default one is not filtered out", func(t *testing.T) { |
| 143 | roles := map[string]string{"uk-box": hostactors.RoleLegacy} |
| 144 | off, err := haMoveFleet(t, false, roles, ukBox("uk-box-2")).srv.selectMoveBoxes(ctx, host, "uk", "ie", app) |
| 145 | if err != nil || !off.Many { |
| 146 | t.Fatalf("region HA off: %+v %v, want main's several-boxes refusal", off, err) |
| 147 | } |
| 148 | on, err := haMoveFleet(t, true, roles, ukBox("uk-box-2")).srv.selectMoveBoxes(ctx, host, "uk", "ie", app) |
| 149 | if err != nil || on != off { |
| 150 | t.Fatalf("region HA on %+v %v, want region HA off's %+v", on, err, off) |
| 151 | } |
| 152 | }) |
| 153 | t.Run("a platform source still moves out", func(t *testing.T) { |
| 154 | fx := haMoveFleet(t, true, map[string]string{"ie-box": hostactors.RolePlatform}) |
| 155 | got, err := fx.srv.selectMoveBoxes(ctx, host, "uk", "ie", app) |
| 156 | if err != nil || got.Source != "ie-box" || got.Target != "uk-box" { |
| 157 | t.Fatalf("selection %+v %v", got, err) |
| 158 | } |
| 159 | }) |
| 160 | } |
| 161 | |
| 162 | // Astra 2a round 1 finding 4: with host claims on and region HA off, the |
| 163 | // move door is main's. A persisted role excludes nothing, and the door |
| 164 | // reads no roster, so an unreadable one cannot refuse the move. |
| 165 | func TestMoveDestinationWithRegionHAOffIsMains(t *testing.T) { |
| 166 | const host, app = "acme.go.tito.io", "bab/titogo" |
| 167 | fx := haMoveFleet(t, false, map[string]string{"uk-box": hostactors.RoleLegacy}) |
| 168 | got, err := fx.srv.selectMoveBoxes(context.Background(), host, "uk", "ie", app) |
| 169 | if err != nil || got.Target != "uk-box" { |
| 170 | t.Fatalf("selection %+v %v", got, err) |
| 171 | } |
| 172 | fx.srv.cfg.Releases = failingGet{Store: fx.rel, key: hostactors.Key} |
| 173 | if got, err := fx.srv.selectMoveBoxes(context.Background(), host, "uk", "ie", app); err != nil || got.Target != "uk-box" { |
| 174 | t.Fatalf("with the roster unreadable: %+v %v", got, err) |
| 175 | } |
| 176 | } |
| 177 | |
| 178 | // haMoveDoor is moveDoorWithClaim's world on a host-claim console with the |
| 179 | // claim runtime armed: roomy edges files (uk-box's changed by edit), a |
| 180 | // roster naming both regions and, with ha, region HA on. |
| 181 | func haMoveDoor(t *testing.T, code string, ha bool, edit func(map[string]any)) (*consoleFixture, string, string) { |
| 182 | t.Helper() |
| 183 | fx, token, moveURL := moveDoorWithClaim(t, code) |
| 184 | roomyEdges(t, fx, edit, "ie-box", "uk-box") |
| 185 | haRoster(t, fx, nil, ieBox("ie-box"), ukBox("uk-box")) |
| 186 | armClaimMoves(t, fx, true, true, nil) |
| 187 | if ha { |
| 188 | bindRegionHA(fx) |
| 189 | } |
| 190 | return fx, token, moveURL |
| 191 | } |
| 192 | |
| 193 | // haMoveDoorOf is moveDoorWithClaim's world over exactly fleet (the claim |
| 194 | // is under fleet's first box, ie-box), with roomy edges files, each box's |
| 195 | // role from roles and region HA on. |
| 196 | func haMoveDoorOf(t *testing.T, code string, roles map[string]string, fleet ...fleetBox) (*consoleFixture, string, string) { |
| 197 | t.Helper() |
| 198 | fx, token := operatorInstanceFixture(t, code) |
| 199 | m := titoManifest() |
| 200 | m.Regions = nil |
| 201 | m.Pool = "" |
| 202 | enableInstances(t, fx, "titogo", m) |
| 203 | u := instancesURL(fx, "bab", "titogo") |
| 204 | if c, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); c != http.StatusCreated { |
| 205 | t.Fatalf("create = %d %s", c, out) |
| 206 | } |
| 207 | armMoveFleet(t, fx, fleet...) |
| 208 | claimFor(t, fx, "acme.go.tito.io", "ie-box", "ie") |
| 209 | names := []string{} |
| 210 | for _, b := range fleet { |
| 211 | names = append(names, b.Box) |
| 212 | } |
| 213 | roomyEdges(t, fx, nil, names...) |
| 214 | haRoster(t, fx, roles, fleet...) |
| 215 | armClaimMoves(t, fx, true, true, nil) |
| 216 | bindRegionHA(fx) |
| 217 | return fx, token, u + "/acme.go.tito.io/move" |
| 218 | } |
| 219 | |
| 220 | // Astra 2a round 2 finding 1: through the whole HTTP door, a destination |
| 221 | // with no box that has room (no edges file, or only blank public |
| 222 | // addresses) is the no-room refusal, alert included, and writes nothing: |
| 223 | // not the placement checks' 400. With region HA off it is main's 400; with |
| 224 | // room the move is admitted to uk-box either way. |
| 225 | func TestMoveDoorWithNoRoomInTheDestinationIsNoRoom(t *testing.T) { |
| 226 | for _, c := range []struct { |
| 227 | name string |
| 228 | edit func(map[string]any) |
| 229 | }{ |
| 230 | {"only a blank public address", func(m map[string]any) { m["eip"] = "" }}, |
| 231 | {"no edges file", nil}, |
| 232 | } { |
| 233 | t.Run(c.name, func(t *testing.T) { |
| 234 | noFile := c.edit == nil |
| 235 | world := func(t *testing.T, code string, ha bool) (*consoleFixture, string, string) { |
| 236 | fx, token, moveURL := haMoveDoor(t, code, ha, c.edit) |
| 237 | if noFile { |
| 238 | // A box that never reported: no edges file and no status |
| 239 | // snapshot. (A fresh snapshot with no edges file is an |
| 240 | // incomplete read, pinned below.) |
| 241 | for _, key := range []string{edgesFilePrefix + "uk-box.json", "status/boxes/uk-box.json"} { |
| 242 | if err := fx.rel.Delete(context.Background(), key); err != nil { |
| 243 | t.Fatal(err) |
| 244 | } |
| 245 | } |
| 246 | } |
| 247 | return fx, token, moveURL |
| 248 | } |
| 249 | _, token, moveURL := world(t, "code-ha-move-off", false) |
| 250 | if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusBadRequest { |
| 251 | t.Fatalf("region HA off: move = %d %s, want main's 400", code, out) |
| 252 | } |
| 253 | pages := pagesTo(t) |
| 254 | fx, token, moveURL := world(t, "code-ha-move-on", true) |
| 255 | claim := claimBytes(t, fx, "acme.go.tito.io") |
| 256 | code, hdr, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`) |
| 257 | if code != http.StatusServiceUnavailable || !strings.Contains(out, noCapacityRefusal("uk")) { |
| 258 | t.Fatalf("move = %d %s", code, out) |
| 259 | } |
| 260 | if got := hdr.Get("Retry-After"); got != noCapacityRetryAfter { |
| 261 | t.Fatalf("Retry-After = %q, want %q", got, noCapacityRetryAfter) |
| 262 | } |
| 263 | expectPages(t, pages, noRoomSubject("uk")) |
| 264 | if claimBytes(t, fx, "acme.go.tito.io") != claim { |
| 265 | t.Fatal("a refused move wrote the claim") |
| 266 | } |
| 267 | a, err := hostops.ReadAdmission(context.Background(), fx.rel, "bab", "titogo", claimMoveAppID) |
| 268 | if err != nil || len(a.Value.Registrations) != 0 { |
| 269 | t.Fatalf("a refused move registered: %+v %v", a.Value.Registrations, err) |
| 270 | } |
| 271 | if rec, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io"); err != nil || rec.Move != nil { |
| 272 | t.Fatalf("a refused move journaled: %+v %v", rec.Move, err) |
| 273 | } |
| 274 | }) |
| 275 | } |
| 276 | t.Run("room", func(t *testing.T) { |
| 277 | fx, token, moveURL := haMoveDoor(t, "code-ha-move-room", true, nil) |
| 278 | if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusAccepted { |
| 279 | t.Fatalf("move = %d %s", code, out) |
| 280 | } |
| 281 | o, err := hostclaim.Read(context.Background(), fx.rel, "acme.go.tito.io") |
| 282 | if err != nil || o.Claim == nil || o.Claim.Op.Intent["target"] != "uk-box" { |
| 283 | t.Fatalf("claim %+v %v, want a move to uk-box", o.Claim, err) |
| 284 | } |
| 285 | }) |
| 286 | } |
| 287 | |
| 288 | // Census ruling region-ha-2a rev 2 item 3: an incomplete placement read |
| 289 | // (here a malformed edges file in the destination) is not no room. The |
| 290 | // move door answers the existing 503 "temporarily unavailable", pages |
| 291 | // nobody, and writes nothing. |
| 292 | func TestMoveDoorWithAnIncompleteReadIsUnavailableNotNoRoom(t *testing.T) { |
| 293 | for _, c := range []struct { |
| 294 | name string |
| 295 | breakIt func(*testing.T, *consoleFixture) |
| 296 | }{ |
| 297 | {"a malformed edges file", func(t *testing.T, fx *consoleFixture) { |
| 298 | key := edgesFilePrefix + "uk-box.json" |
| 299 | _, etag, err := fx.rel.Get(context.Background(), key) |
| 300 | if err != nil { |
| 301 | t.Fatal(err) |
| 302 | } |
| 303 | if _, err := fx.rel.Put(context.Background(), key, []byte("{not json"), etag); err != nil { |
| 304 | t.Fatal(err) |
| 305 | } |
| 306 | }}, |
| 307 | {"a fresh status snapshot and no edges file", func(t *testing.T, fx *consoleFixture) { |
| 308 | if err := fx.rel.Delete(context.Background(), edgesFilePrefix+"uk-box.json"); err != nil { |
| 309 | t.Fatal(err) |
| 310 | } |
| 311 | }}, |
| 312 | } { |
| 313 | t.Run(c.name, func(t *testing.T) { moveDoorIncompleteRead(t, c.breakIt) }) |
| 314 | } |
| 315 | } |
| 316 | |
| 317 | func moveDoorIncompleteRead(t *testing.T, breakIt func(*testing.T, *consoleFixture)) { |
| 318 | pages := pagesTo(t) |
| 319 | fx, token, moveURL := haMoveDoor(t, "code-ha-move-incomplete", true, nil) |
| 320 | ctx := context.Background() |
| 321 | breakIt(t, fx) |
| 322 | claim := claimBytes(t, fx, "acme.go.tito.io") |
| 323 | code, hdr, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`) |
| 324 | if code != http.StatusServiceUnavailable || !strings.Contains(out, claimResponseUnavailable) || strings.Contains(out, noCapacityRefusal("uk")) { |
| 325 | t.Fatalf("move = %d %s, want the temporarily unavailable 503", code, out) |
| 326 | } |
| 327 | if got := hdr.Get("Retry-After"); got == noCapacityRetryAfter { |
| 328 | t.Fatalf("Retry-After = %q, the no-room value", got) |
| 329 | } |
| 330 | expectPages(t, pages) |
| 331 | if claimBytes(t, fx, "acme.go.tito.io") != claim { |
| 332 | t.Fatal("a refused move wrote the claim") |
| 333 | } |
| 334 | if rec, err := provision.GetRecord(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io"); err != nil || rec.Move != nil { |
| 335 | t.Fatalf("a refused move journaled: %+v %v", rec.Move, err) |
| 336 | } |
| 337 | } |
| 338 | |
| 339 | // Census ruling region-ha-2a (fix round 1): the room check refuses only |
| 340 | // where placement is known to have no room. A roster that does not place |
| 341 | // the destination (hostops.ErrContract) and a destination with no |
| 342 | // candidates are main's: the selection equals the one region HA off makes, |
| 343 | // and nobody is paged. |
| 344 | func TestMoveRoomCheckRefusesOnlyWhereRoomIsKnownMissing(t *testing.T) { |
| 345 | const host, app = "acme.go.tito.io", "bab/titogo" |
| 346 | ctx := context.Background() |
| 347 | for _, c := range []struct { |
| 348 | name string |
| 349 | codes []string |
| 350 | fleet []fleetBox |
| 351 | want moveSelection |
| 352 | }{ |
| 353 | {"the roster does not place the destination", []string{"ie"}, []fleetBox{ieBox("ie-box"), ukBox("uk-box")}, |
| 354 | moveSelection{Target: "uk-box", Source: "ie-box"}}, |
| 355 | {"the destination has no candidates", []string{"ie", "uk"}, []fleetBox{ieBox("ie-box"), func() fleetBox { b := ukBox("uk-box"); b.Unverified = true; return b }()}, |
| 356 | moveSelection{Source: "ie-box"}}, |
| 357 | } { |
| 358 | t.Run(c.name, func(t *testing.T) { |
| 359 | pages := pagesTo(t) |
| 360 | off, err := haFleetOf(t, false, nil, c.codes, c.fleet...).srv.selectMoveBoxes(ctx, host, "uk", "ie", app) |
| 361 | if err != nil || off.Target != c.want.Target || off.Source != c.want.Source { |
| 362 | t.Fatalf("region HA off: %+v %v, want target %q source %q", off, err, c.want.Target, c.want.Source) |
| 363 | } |
| 364 | // Every field, the floor's included, is main's. |
| 365 | on, err := haFleetOf(t, true, nil, c.codes, c.fleet...).srv.selectMoveBoxes(ctx, host, "uk", "ie", app) |
| 366 | if err != nil || on != off { |
| 367 | t.Fatalf("region HA on: %+v %v, want region HA off's %+v", on, err, off) |
| 368 | } |
| 369 | expectPages(t, pages) |
| 370 | }) |
| 371 | } |
| 372 | t.Run("the door answers main's 400 for a destination with no candidates", func(t *testing.T) { |
| 373 | pages := pagesTo(t) |
| 374 | unverified := ukBox("uk-box") |
| 375 | unverified.Unverified = true |
| 376 | _, token, moveURL := haMoveDoorOf(t, "code-ha-move-nocands", nil, ieBox("ie-box"), unverified) |
| 377 | code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`) |
| 378 | if code != http.StatusBadRequest || !strings.Contains(out, "no box in uk publishes an edges file yet") { |
| 379 | t.Fatalf("move = %d %s, want main's 400", code, out) |
| 380 | } |
| 381 | expectPages(t, pages) |
| 382 | }) |
| 383 | } |
| 384 | |
| 385 | // The door's own selection branch (apiinstancemove.go, after |
| 386 | // selectMoveBoxes): a destination whose placeable box is not a candidate |
| 387 | // (uk-box-2 has room but no verified identity) while its only candidate is |
| 388 | // excluded by role passes roomIn and is refused by moveRoomCheck, with the |
| 389 | // no-room answer, its Retry-After and its alert, and writes nothing. |
| 390 | func TestMoveDoorRefusesWhenNoCandidateHasRoom(t *testing.T) { |
| 391 | pages := pagesTo(t) |
| 392 | stranger := ukBox("uk-box-2") |
| 393 | stranger.Unverified = true |
| 394 | fx, token, moveURL := haMoveDoorOf(t, "code-ha-move-nocand-room", map[string]string{"uk-box": hostactors.RoleLegacy}, |
| 395 | ieBox("ie-box"), ukBox("uk-box"), stranger) |
| 396 | claim := claimBytes(t, fx, "acme.go.tito.io") |
| 397 | code, hdr, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`) |
| 398 | if code != http.StatusServiceUnavailable || !strings.Contains(out, noCapacityRefusal("uk")) { |
| 399 | t.Fatalf("move = %d %s", code, out) |
| 400 | } |
| 401 | if got := hdr.Get("Retry-After"); got != noCapacityRetryAfter { |
| 402 | t.Fatalf("Retry-After = %q, want %q", got, noCapacityRetryAfter) |
| 403 | } |
| 404 | expectPages(t, pages, noRoomSubject("uk")) |
| 405 | if claimBytes(t, fx, "acme.go.tito.io") != claim { |
| 406 | t.Fatal("a refused move wrote the claim") |
| 407 | } |
| 408 | if rec, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io"); err != nil || rec.Move != nil { |
| 409 | t.Fatalf("a refused move journaled: %+v %v", rec.Move, err) |
| 410 | } |
| 411 | } |
| 412 | |
| 413 | // Fix round 1: a placement read moveRoomCheck cannot complete (the second |
| 414 | // read the door makes, after roomIn's) is the claim doors' temporarily |
| 415 | // unavailable 503, with no page and no Retry-After of no room. It is not |
| 416 | // the moves protocol floor's answer, and an incomplete snapshot is still |
| 417 | // hostops.ErrEvidenceIncomplete under the wrapper. |
| 418 | func TestMoveRoomCheckReadFailureIsUnavailable(t *testing.T) { |
| 419 | for _, c := range []struct { |
| 420 | name string |
| 421 | fail func(boxchoice.Snapshot, error) (boxchoice.Snapshot, error) |
| 422 | }{ |
| 423 | {"an error", func(boxchoice.Snapshot, error) (boxchoice.Snapshot, error) { |
| 424 | return boxchoice.Snapshot{}, errors.New("edges store down") |
| 425 | }}, |
| 426 | {"an incomplete snapshot", func(snap boxchoice.Snapshot, err error) (boxchoice.Snapshot, error) { |
| 427 | snap.Complete = false |
| 428 | return snap, err |
| 429 | }}, |
| 430 | } { |
| 431 | t.Run(c.name, func(t *testing.T) { |
| 432 | pages := pagesTo(t) |
| 433 | fx, token, moveURL := haMoveDoor(t, "code-ha-move-read", true, nil) |
| 434 | edges, calls := fx.srv.cfg.ClaimAuthority.Executors.Edges, 0 |
| 435 | fx.srv.cfg.ClaimAuthority.Executors.Edges = func(ctx context.Context) (boxchoice.Snapshot, error) { |
| 436 | snap, err := edges(ctx) |
| 437 | if calls++; calls >= 2 { // roomIn's read passes; moveRoomCheck's does not |
| 438 | return c.fail(snap, err) |
| 439 | } |
| 440 | return snap, err |
| 441 | } |
| 442 | claim := claimBytes(t, fx, "acme.go.tito.io") |
| 443 | code, hdr, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`) |
| 444 | if code != http.StatusServiceUnavailable || !strings.Contains(out, claimResponseUnavailable) { |
| 445 | t.Fatalf("move = %d %s, want the temporarily unavailable 503", code, out) |
| 446 | } |
| 447 | if calls != 2 || hdr.Get("Retry-After") == noCapacityRetryAfter { |
| 448 | t.Fatalf("chooser reads = %d, Retry-After = %q", calls, hdr.Get("Retry-After")) |
| 449 | } |
| 450 | expectPages(t, pages) |
| 451 | if claimBytes(t, fx, "acme.go.tito.io") != claim { |
| 452 | t.Fatal("a refused move wrote the claim") |
| 453 | } |
| 454 | }) |
| 455 | } |
| 456 | t.Run("an incomplete snapshot stays ErrEvidenceIncomplete", func(t *testing.T) { |
| 457 | fx := haMoveFleet(t, true, nil) |
| 458 | edges := fx.srv.cfg.ClaimAuthority.Executors.Edges |
| 459 | fx.srv.cfg.ClaimAuthority.Executors.Edges = func(ctx context.Context) (boxchoice.Snapshot, error) { |
| 460 | snap, err := edges(ctx) |
| 461 | snap.Complete = false |
| 462 | return snap, err |
| 463 | } |
| 464 | _, err := fx.srv.selectMoveBoxes(context.Background(), "acme.go.tito.io", "uk", "ie", "bab/titogo") |
| 465 | var check *roomCheckError |
| 466 | if !errors.As(err, &check) || !errors.Is(err, hostops.ErrEvidenceIncomplete) { |
| 467 | t.Fatalf("err = %v, want a room check error wrapping ErrEvidenceIncomplete", err) |
| 468 | } |
| 469 | }) |
| 470 | } |
| 471 | |
amadan