amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package console
2
3import (
4 "context"
5 "crypto/sha256"
6 "encoding/hex"
7 "encoding/json"
8 "fmt"
9 "net/http"
10 "strings"
11 "sync"
12 "testing"
13 "time"
14
15 "github.com/carlosframework/platform/internal/domains"
16 "github.com/carlosframework/platform/internal/hostclaim"
17 "github.com/carlosframework/platform/internal/hostops"
18 "github.com/carlosframework/platform/internal/moves"
19 "github.com/carlosframework/platform/internal/provision"
20 "github.com/carlosframework/platform/internal/release"
21 "github.com/carlosframework/platform/internal/status"
22 "github.com/carlosframework/platform/internal/svcrole"
23)
24
25// The offline operator move's doors, driven in-process against the same
26// console fixture the move doors' tests use. The boxes' side is played by
27// reports written at control/offline-moves/ exactly where the real boxes
28// write them (moves.OfflineReportKey); the record moves only through
29// provision's offline transitions.
30
31const offlineHost = "acme.shop.example.com"
32
33// offlineManifest is the enabled manifest of the app shop, whose instances
34// live under shop.example.com.
35func offlineManifest() provision.Manifest {
36 return provision.Manifest{Schema: 1, App: "shop", Domain: "shop.example.com",
37 Pool: "shop", Channel: "stable", Regions: []string{"ie"}, Seed: []string{"seed"}}
38}
39
40// offlineClaimFor is claimFor for an instance of the app shop: it writes the
41// host's serving claim for the record's own incarnation.
42func offlineClaimFor(t *testing.T, fx *consoleFixture, host, box, region string) {
43 t.Helper()
44 rec, err := provision.GetRecord(context.Background(), fx.rel, "bab", "shop", host)
45 if err != nil {
46 t.Fatal(err)
47 }
48 inc, err := hostclaim.Incarnation("bab", "shop", host, rec.CreatedAt)
49 if err != nil {
50 t.Fatal(err)
51 }
52 _, err = hostclaim.Update(context.Background(), fx.rel, hostclaim.Observed{Host: host, Absent: true}, func(c *hostclaim.Claim) error {
53 c.Holder = &hostclaim.Holder{Family: "instance", Key: provision.RecordKey("bab", "shop", host), Account: "bab", App: "shop",
54 Incarnation: inc, CreatedAt: rec.CreatedAt, Location: "instances/" + host, Stamp: 1}
55 c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: box}
56 c.Touched = hostclaim.Touched{Complete: true, Regions: []string{region}, Boxes: []string{box}}
57 return nil
58 })
59 if err != nil {
60 t.Fatal(err)
61 }
62}
63
64// offlineBox is one box the fleet publishes: an edges file and a status
65// snapshot.
66type offlineBox struct {
67 Box, Placement, Bucket string
68 NoCap bool // no offline-move-1 (nor offline-stage-1)
69 NoStageCap bool // no offline-stage-1
70 ClaimsOn bool // its status report says claims are on
71 NoMetrics bool // its status report carries no metrics
72 NotSharedPool bool // serves_shared_pool false
73 NoHibernation bool // hibernation_configured false
74 Hosts []string // hosts its edges file lists
75 FP string // secrets_key_fp; testKeyFP when empty
76}
77
78func ieOff(box string) offlineBox { return offlineBox{Box: box, Placement: "ie", Bucket: "data-ie"} }
79func ukOff(box string) offlineBox { return offlineBox{Box: box, Placement: "uk", Bucket: "data-uk"} }
80
81type offlineWorld struct {
82 t *testing.T
83 fx *consoleFixture
84 token string
85 base string // .../instances/<host>/move/offline
86 id string // the started move's id
87}
88
89// newOfflineWorld is an operator console with one hibernating instance in
90// region ie and the fleet published.
91func newOfflineWorld(t *testing.T, boxes ...offlineBox) *offlineWorld {
92 t.Helper()
93 return newOfflineWorldIn(t, "", "ie", boxes...)
94}
95
96// newOfflineWorldIn is newOfflineWorld with the console's home named and the
97// instance created in the given region.
98func newOfflineWorldIn(t *testing.T, home, region string, boxes ...offlineBox) *offlineWorld {
99 t.Helper()
100 fx, token := instanceFixture(t, "code-offline-"+strings.ReplaceAll(t.Name(), "/", "-"))
101 if err := fx.st.SetOperator("alice@keymail.dev", true); err != nil {
102 t.Fatal(err)
103 }
104 if _, err := fx.st.CreateAppIn(OpsAccountID, "shop", "paul@keymail.dev", time.Now()); err != nil {
105 t.Fatal(err)
106 }
107 fx.srv.cfg.HomeRegion = home
108 m := offlineManifest()
109 m.Regions = nil
110 m.Pool = ""
111 enableInstances(t, fx, "shop", m)
112 u := instancesURL(fx, "bab", "shop")
113 if c, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"`+offlineHost+`","region":"`+region+`"}`); c != http.StatusCreated {
114 t.Fatalf("create = %d %s", c, out)
115 }
116 w := &offlineWorld{t: t, fx: fx, token: token, base: u + "/" + offlineHost + "/move/offline"}
117 w.fleet(boxes...)
118 return w
119}
120
121// putRaw writes a bucket object as a fixture, replacing what is there.
122func offlinePut(t *testing.T, rel release.Store, key string, body []byte) {
123 t.Helper()
124 _, etag, err := rel.Get(context.Background(), key)
125 if err != nil {
126 etag = ""
127 }
128 if _, err := rel.Put(context.Background(), key, body, etag); err != nil {
129 t.Fatal(err)
130 }
131}
132
133// fleet publishes the boxes, replacing any earlier publication of the same
134// box.
135func (w *offlineWorld) fleet(boxes ...offlineBox) {
136 w.t.Helper()
137 w.fx.srv.cfg.SecretsPub = testKeyPub
138 for _, b := range boxes {
139 aws := "eu-west-1"
140 if b.Placement == "uk" {
141 aws = "eu-west-2"
142 }
143 caps := []string{"region-ha-1"}
144 if !b.NoCap {
145 caps = append(caps, offlineMoveCapability)
146 if !b.NoStageCap {
147 caps = append(caps, offlineStageCapability)
148 }
149 }
150 hosts := map[string]any{}
151 for _, h := range b.Hosts {
152 hosts[h] = map[string]any{"role": "home"}
153 }
154 fp := b.FP
155 if fp == "" {
156 fp = testKeyFP
157 }
158 body, err := json.Marshal(map[string]any{
159 "v": 1, "box": b.Box, "box_id": b.Box, "eip": "1.2.3.4", "region": aws, "placement": b.Placement,
160 "data_bucket": b.Bucket, "data_bucket_region": aws, "hibernation_configured": !b.NoHibernation, "serves_shared_pool": !b.NotSharedPool,
161 "accepts_inbound": true, "secrets_key_fp": fp, "capabilities": caps, "hosts": hosts,
162 })
163 if err != nil {
164 w.t.Fatal(err)
165 }
166 offlinePut(w.t, w.fx.rel, edgesFilePrefix+b.Box+".json", body)
167 sb := status.Box{Label: b.Box, GeneratedAt: time.Now()}
168 if !b.NoMetrics {
169 sb.Metrics = &status.BoxMetrics{HostClaimEnabled: b.ClaimsOn}
170 }
171 if err := status.WriteBox(context.Background(), w.fx.rel, sb); err != nil {
172 w.t.Fatal(err)
173 }
174 }
175}
176
177func (w *offlineWorld) req(method, path, body string) (int, string) {
178 w.t.Helper()
179 code, _, out := rawReq(w.t, method, w.base+path, w.token, body)
180 return code, out
181}
182
183// bucketState is every object in the bucket, for the "a refusal writes
184// nothing" check.
185func bucketState(t *testing.T, rel release.Store) map[string]string {
186 t.Helper()
187 keys, err := rel.List(context.Background(), "")
188 if err != nil {
189 t.Fatal(err)
190 }
191 out := map[string]string{}
192 for _, k := range keys {
193 b, _, err := rel.Get(context.Background(), k)
194 if err != nil {
195 t.Fatal(err)
196 }
197 out[k] = string(b)
198 }
199 return out
200}
201
202// refused posts and wants exactly this refusal, with nothing written.
203func (w *offlineWorld) refused(method, path, body string, wantCode int, wantMsg string) {
204 w.t.Helper()
205 before := bucketState(w.t, w.fx.rel)
206 code, out := w.req(method, path, body)
207 if code != wantCode || errorText(out) != wantMsg {
208 w.t.Fatalf("%s %s = %d %s\nwant %d %q", method, path, code, out, wantCode, wantMsg)
209 }
210 after := bucketState(w.t, w.fx.rel)
211 if len(before) != len(after) {
212 w.t.Fatalf("%s %s: a refusal changed the bucket: %d objects before, %d after", method, path, len(before), len(after))
213 }
214 for k, v := range before {
215 if after[k] != v {
216 w.t.Fatalf("%s %s: a refusal changed %s", method, path, k)
217 }
218 }
219}
220
221func (w *offlineWorld) view(code int, out string) offlineView {
222 w.t.Helper()
223 var v offlineView
224 if err := json.Unmarshal([]byte(out), &v); err != nil {
225 w.t.Fatalf("%v in %s", err, out)
226 }
227 _ = code
228 return v
229}
230
231// ok posts and wants a 202 whose view has the state.
232func (w *offlineWorld) ok(path, body, wantState string) offlineView {
233 w.t.Helper()
234 code, out := w.req(http.MethodPost, path, body)
235 if code != http.StatusAccepted {
236 w.t.Fatalf("POST %s = %d %s", path, code, out)
237 }
238 v := w.view(code, out)
239 if v.State != wantState {
240 w.t.Fatalf("POST %s: state %q, want %q: %s", path, v.State, wantState, out)
241 }
242 return v
243}
244
245func (w *offlineWorld) status() offlineView {
246 w.t.Helper()
247 code, out := w.req(http.MethodGet, "", "")
248 if code != http.StatusOK {
249 w.t.Fatalf("GET status = %d %s", code, out)
250 }
251 return w.view(code, out)
252}
253
254func (w *offlineWorld) record() *provision.Record {
255 w.t.Helper()
256 rec, err := provision.GetRecord(context.Background(), w.fx.rel, "bab", "shop", offlineHost)
257 if err != nil {
258 w.t.Fatal(err)
259 }
260 return rec
261}
262
263// edit applies record transitions as a fixture, through the record writer.
264func (w *offlineWorld) edit(muts ...func(*provision.Record) error) {
265 w.t.Helper()
266 for _, m := range muts {
267 if _, err := provision.SetRecordMove(context.Background(), w.fx.rel, "bab", "shop", offlineHost, m); err != nil {
268 w.t.Fatal(err)
269 }
270 }
271}
272
273func serialBytes(t *testing.T, rel release.Store) string {
274 t.Helper()
275 b, _, err := rel.Get(context.Background(), "control/serial")
276 if err != nil {
277 return ""
278 }
279 return string(b)
280}
281
282// start runs the start door and keeps the id.
283func (w *offlineWorld) start() offlineView {
284 w.t.Helper()
285 before := serialBytes(w.t, w.fx.rel)
286 v := w.ok("", `{"region":"uk"}`, provision.OfflineStopping)
287 if v.ID == "" {
288 w.t.Fatalf("start answered no id: %+v", v)
289 }
290 if serialBytes(w.t, w.fx.rel) == before {
291 w.t.Fatal("start did not bump the serial")
292 }
293 w.id = v.ID
294 return v
295}
296
297// report writes a box's report where the real box writes it, and returns
298// its bytes.
299func (w *offlineWorld) report(role, box, phase string, edit func(*moves.OfflineReport)) []byte {
300 w.t.Helper()
301 r := moves.OfflineReport{V: moves.OfflineReportVersion, Op: w.id, Host: offlineHost, Box: box, Role: role, Phase: phase,
302 At: time.Now().UTC(), Held: true, Fenced: true, Quiesced: true}
303 if edit != nil {
304 edit(&r)
305 }
306 body, err := json.Marshal(r)
307 if err != nil {
308 w.t.Fatal(err)
309 }
310 key := moves.OfflineReportKey(offlineHost, w.id, role, box)
311 if key == "" {
312 w.t.Fatalf("no report key for %s/%s", role, box)
313 }
314 offlinePut(w.t, w.fx.rel, key, body)
315 return body
316}
317
318// stopped writes every source box's report as a stopped box writes it.
319func (w *offlineWorld) stopped(edit func(box string, r *moves.OfflineReport), boxes ...string) {
320 w.t.Helper()
321 for _, b := range boxes {
322 w.report(moves.OfflineRoleSource, b, moves.OfflinePhaseStopped, func(r *moves.OfflineReport) {
323 if edit != nil {
324 edit(b, r)
325 }
326 })
327 }
328}
329
330func okChecks(owner string) map[string]moves.Check {
331 out := map[string]moves.Check{}
332 for _, it := range moves.OfflineOwned(owner) {
333 out[it.ID] = moves.Check{OK: true, Reason: "ok"}
334 }
335 return out
336}
337
338// sealerChecks is what a sealer's report carries when the seal went well:
339// every item the source side reports.
340func sealerChecks() map[string]moves.Check {
341 out := map[string]moves.Check{}
342 for _, it := range moves.OfflineSourced() {
343 out[it.ID] = moves.Check{OK: true, Reason: "ok"}
344 }
345 return out
346}
347
348// putSeal writes seal.json as the sealer does: once and create-only when it
349// is not there, and replaced only by a test that tampers with it.
350func (w *offlineWorld) putSeal(sealer string, edit func(*moves.OfflineReport)) []byte {
351 w.t.Helper()
352 r := moves.OfflineReport{V: moves.OfflineReportVersion, Op: w.id, Host: offlineHost, Box: sealer, Role: moves.OfflineRoleSealer,
353 Phase: moves.OfflinePhaseSealed, At: time.Now().UTC(), Items: sealerChecks(), FinalTXID: 7, TombstoneOwner: moves.TombstoneOwner(w.id)}
354 if edit != nil {
355 edit(&r)
356 }
357 body, err := json.Marshal(r)
358 if err != nil {
359 w.t.Fatal(err)
360 }
361 key := moves.OfflineSealKey(offlineHost, w.id)
362 if _, _, err := w.fx.rel.Get(context.Background(), key); err != nil {
363 if _, err := w.fx.rel.Put(context.Background(), key, body, ""); err != nil {
364 w.t.Fatal(err)
365 }
366 } else {
367 offlinePut(w.t, w.fx.rel, key, body)
368 }
369 return body
370}
371
372// sealed plays the sealer: the seal, its living report, and the transition
373// that records where the seal is. It returns the seal's bytes.
374func (w *offlineWorld) sealed(sealer string) []byte {
375 w.t.Helper()
376 seal := w.putSeal(sealer, nil)
377 w.sealerFresh(sealer)
378 w.edit(provision.OfflineSealedBy(w.id, sealer, provision.MoveSource{FinalTXID: 7, Evidence: moves.OfflineSealEvidence(offlineHost, w.id, seal)}))
379 return seal
380}
381
382// landed plays the target: its landing report and the transition.
383func (w *offlineWorld) landed(target string) {
384 w.t.Helper()
385 w.report(moves.OfflineRoleTarget, target, moves.OfflinePhaseLanded, func(r *moves.OfflineReport) {
386 r.Items = okChecks(moves.OfflineOwnerLanding)
387 })
388 w.edit(provision.OfflineLandedBy(w.id, "uk"))
389}
390
391// toLanded drives a started move through seal and flip to offline-landed
392// with fresh reports from the one ie box and the one uk box.
393func (w *offlineWorld) toLanded() {
394 w.t.Helper()
395 w.start()
396 w.stopped(nil, "ie-box")
397 w.ok("/seal", "", provision.OfflineSealing)
398 w.sealed("ie-box")
399 // The sealer rewrites its living report on every pass, so by the time
400 // the operator flips it is not the bytes it was at the seal. The seal is.
401 w.sealerFresh("ie-box")
402 w.stopped(nil, "ie-box")
403 w.ok("/flip", "", provision.OfflineLanding)
404 w.landed("uk-box")
405 w.stopped(nil, "ie-box")
406 w.sealerFresh("ie-box")
407}
408
409// sealerFresh rewrites the sealer's report as the sealer does every pass.
410func (w *offlineWorld) sealerFresh(sealer string) {
411 w.t.Helper()
412 w.report(moves.OfflineRoleSealer, sealer, moves.OfflinePhaseSealed, func(r *moves.OfflineReport) {
413 r.Items = sealerChecks()
414 r.TombstoneOwner = moves.TombstoneOwner(w.id)
415 r.TombstoneUntil = time.Now().Add(moves.TombstoneTTL)
416 })
417}
418
419var oneEach = []offlineBox{ieOff("ie-box"), ukOff("uk-box")}
420
421const (
422 msgO20 = "offline moves are not available while host claims or Phase 1 moves are on"
423 msgO29 = offlineHost + " has a host claim. offline moves are only for hosts without one"
424)
425
426func msgO21(box string) string {
427 return box + " cannot take part in an offline move. it runs an older release or has host claims on"
428}
429func msgO25(state string) string {
430 return "this move is " + state + ". run carlos move offline status --host " + offlineHost + " to see what it waits for"
431}
432func msgMC10(box string) string {
433 return "Waiting for " + box + " in ie to confirm it has stopped " + offlineHost + ". " + offlineHost + " is offline."
434}
435func msgO178(box string) string {
436 return "the report from " + box + " is not current. it reports again within a minute. try again then"
437}
438func msgO31(box string) string {
439 return box + " still runs a process for " + offlineHost + ". wait for it to stop, then flip"
440}
441
442var offlineRoutes = []struct{ method, path, body string }{
443 {http.MethodGet, "", ""},
444 {http.MethodPost, "", `{"region":"uk"}`},
445 {http.MethodPost, "/stage", `{"region":"uk"}`},
446 {http.MethodPost, "/seal", ""},
447 {http.MethodPost, "/flip", ""},
448 {http.MethodPost, "/open", ""},
449 {http.MethodPost, "/finish", ""},
450 {http.MethodPost, "/abort", ""},
451}
452
453// Every door is the operator's: a member's token and an Instance-role
454// service token meet the same bytes http.NotFound writes, before MV-O20 or
455// anything else is said.
456func TestOfflineDoorsAreOperatorOnly(t *testing.T) {
457 w := newOfflineWorld(t, oneEach...)
458 member := w.token
459 router := serviceToken(t, w.fx.st, OpsAccountID, "router", "shop", svcrole.Instance)
460 if err := w.fx.st.SetOperator("alice@keymail.dev", false); err != nil {
461 t.Fatal(err)
462 }
463 want := canonicalNotFound(t)
464 for _, who := range []struct{ name, token string }{{"a member", member}, {"an instance token", router}} {
465 for _, rt := range offlineRoutes {
466 code, h, body := rawReq(t, rt.method, w.base+rt.path, who.token, rt.body)
467 assertNotFound(t, rt.method+" "+rt.path+" as "+who.name, want, code, h, body)
468 }
469 }
470 // The switch being on does not move the 404 behind MV-O20.
471 w.fx.srv.cfg.MovesEnabled = true
472 for _, rt := range offlineRoutes {
473 code, h, body := rawReq(t, rt.method, w.base+rt.path, member, rt.body)
474 assertNotFound(t, rt.method+" "+rt.path+" as a member with Phase 1 on", want, code, h, body)
475 }
476 if rec := w.record(); rec.Move != nil {
477 t.Fatalf("a refused request wrote a move: %+v", rec.Move)
478 }
479}
480
481// MV-O20, from the console's own configuration and from the deployment's
482// backfill plan; each leaves the bucket as it found it.
483func TestOfflineDoorsRefuseUnderClaims(t *testing.T) {
484 w := newOfflineWorld(t, oneEach...)
485 each := func(label string) {
486 t.Helper()
487 for _, rt := range offlineRoutes {
488 w.refused(rt.method, rt.path, rt.body, http.StatusConflict, msgO20)
489 }
490 _ = label
491 }
492 w.fx.srv.cfg.HostClaim = true
493 each("claims, no authority")
494 w.fx.srv.cfg.ClaimAuthority = &hostops.ProductionAuthority{}
495 each("claims, with an authority")
496 w.fx.srv.cfg.HostClaim, w.fx.srv.cfg.ClaimAuthority = false, nil
497 w.fx.srv.cfg.MovesEnabled = true
498 each("Phase 1")
499 w.fx.srv.cfg.MovesEnabled = false
500
501 plan := func(phase string) {
502 t.Helper()
503 raw, err := release.SignRootEnvelope(release.RootSigner(w.fx.priv), hostops.BackfillPlan{V: 1, ID: "plan-1", Phase: phase, CreatedAt: time.Now().UTC()})
504 if err != nil {
505 t.Fatal(err)
506 }
507 offlinePut(t, w.fx.rel, hostops.BackfillKey, raw)
508 }
509 plan("collecting")
510 w.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusConflict, msgO20)
511 plan("adopting")
512 w.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusConflict, msgO20)
513 offlinePut(t, w.fx.rel, hostops.BackfillKey, []byte("not an envelope"))
514 w.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusConflict, msgO20)
515 // A plan at done is allowed, with any number of hosts.
516 plan("done")
517 w.start()
518}
519
520func TestOfflineStartRefusals(t *testing.T) {
521 type tc struct {
522 name string
523 boxes []offlineBox
524 setup func(w *offlineWorld)
525 body string
526 code int
527 msg string
528 }
529 const body = `{"region":"uk"}`
530 other := func(w *offlineWorld) {
531 // A second instance, moving offline: started through its own door, so
532 // it holds the lock.
533 v, code, out := w.startOther("beta.shop.example.com")
534 if code != http.StatusAccepted {
535 t.Fatalf("beta start = %d %s (%+v)", code, out, v)
536 }
537 }
538 cases := []tc{
539 {name: "a replica", boxes: oneEach, body: body, code: 400, msg: "a replica does not move. delete it and create one in the new region",
540 setup: func(w *offlineWorld) { w.edit(func(r *provision.Record) error { r.Role = "replica"; return nil }) }},
541 {name: "replicas", boxes: oneEach, body: body, code: 400, msg: "a home with replicas does not move. remove its replicas first",
542 setup: func(w *offlineWorld) {
543 w.edit(func(r *provision.Record) error { r.Replicas = []string{"uk"}; return nil })
544 }},
545 {name: "steering", boxes: oneEach, body: body, code: 400, msg: offlineHost + " is steered or runs under a unit. offline moves cannot carry either yet",
546 setup: func(w *offlineWorld) { w.edit(func(r *provision.Record) error { r.Steering = "geo"; return nil }) }},
547 {name: "a unit", boxes: oneEach, body: body, code: 400, msg: offlineHost + " is steered or runs under a unit. offline moves cannot carry either yet",
548 setup: func(w *offlineWorld) { w.edit(func(r *provision.Record) error { r.Unit = "app.service"; return nil }) }},
549 {name: "a record acquisition", boxes: oneEach, body: body, code: 409, msg: msgO29,
550 setup: func(w *offlineWorld) { w.edit(func(r *provision.Record) error { r.Acquisition = "acq-1"; return nil }) }},
551 {name: "a host claim", boxes: oneEach, body: body, code: 409, msg: msgO29,
552 setup: func(w *offlineWorld) { offlineClaimFor(t, w.fx, offlineHost, "ie-box", "ie") }},
553 {name: "a domains claim with an acquisition", boxes: oneEach, body: body, code: 409, msg: msgO29,
554 setup: func(w *offlineWorld) {
555 c := domains.Claim{Schema: 1, Host: offlineHost, Account: "bab", App: "shop", Acquisition: "acq-1"}
556 raw, _ := json.Marshal(c)
557 offlinePut(t, w.fx.rel, domains.ClaimKey(offlineHost), raw)
558 }},
559 {name: "an admission", boxes: oneEach, body: body, code: 409, msg: msgO29,
560 setup: func(w *offlineWorld) {
561 key, err := hostops.AdmissionKey("bab", "shop")
562 if err != nil {
563 t.Fatal(err)
564 }
565 offlinePut(t, w.fx.rel, key, []byte(`{}`))
566 }},
567 {name: "a second move in flight", boxes: oneEach, body: body, code: 409, msg: "beta.shop.example.com is moving offline. move one instance at a time", setup: other},
568 {name: "no region", boxes: oneEach, body: `{"region":""}`, code: 400, msg: "invalid region"},
569 {name: "a bad region", boxes: oneEach, body: `{"region":"UK!"}`, code: 400, msg: "invalid region"},
570 {name: "the current region", boxes: oneEach, body: `{"region":"ie"}`, code: 409, msg: "the instance is already in ie"},
571 {name: "the manifest's regions", boxes: oneEach, body: body, code: 400, msg: "region uk is not one of this app's regions: ie",
572 setup: func(w *offlineWorld) {
573 m := offlineManifest()
574 m.Pool = ""
575 enableInstances(t, w.fx, "shop", m)
576 }},
577 {name: "an unreadable edges file", boxes: oneEach, body: body, code: 409, msg: "the fleet's edges files could not all be read. try again",
578 setup: func(w *offlineWorld) { offlinePut(t, w.fx.rel, edgesFilePrefix+"broken.json", []byte("{")) }},
579 {name: "no source box", boxes: []offlineBox{ukOff("uk-box")}, body: body, code: 400, msg: "no eligible box in ie publishes an edges file"},
580 {name: "no target box", boxes: []offlineBox{ieOff("ie-box")}, body: body, code: 409, msg: "uk has 0 boxes. an offline move needs exactly one box in the region it moves to"},
581 {name: "two target boxes", boxes: []offlineBox{ieOff("ie-box"), ukOff("uk-a"), ukOff("uk-b")}, body: body, code: 409,
582 msg: "uk has 2 boxes. an offline move needs exactly one box in the region it moves to"},
583 {name: "a source box without the capability", boxes: []offlineBox{func() offlineBox { b := ieOff("ie-box"); b.NoCap = true; return b }(), ukOff("uk-box")},
584 body: body, code: 409, msg: msgO21("ie-box")},
585 {name: "the second source box without the capability", boxes: []offlineBox{ieOff("ie-a"), func() offlineBox { b := ieOff("ie-b"); b.NoCap = true; return b }(), ukOff("uk-box")},
586 body: body, code: 409, msg: msgO21("ie-b")},
587 {name: "a target box without the capability", boxes: []offlineBox{ieOff("ie-box"), func() offlineBox { b := ukOff("uk-box"); b.NoCap = true; return b }()},
588 body: body, code: 409, msg: msgO21("uk-box")},
589 {name: "a target that does not hold the secrets key", boxes: []offlineBox{ieOff("ie-box"), func() offlineBox { b := ukOff("uk-box"); b.FP = "not-it"; return b }()},
590 body: body, code: 409, msg: "uk-box does not hold this deployment's secrets key"},
591 {name: "a target that lists the host", boxes: []offlineBox{ieOff("ie-box"), func() offlineBox { b := ukOff("uk-box"); b.Hosts = []string{offlineHost}; return b }()},
592 body: body, code: 409, msg: "uk-box already serves " + offlineHost + ". an offline move needs the region it moves to to be empty of it"},
593 {name: "a target with no data bucket", boxes: []offlineBox{ieOff("ie-box"), func() offlineBox { b := ukOff("uk-box"); b.Bucket = ""; return b }()},
594 body: body, code: 400, msg: "region uk cannot take a move in this deployment. it publishes no staging bucket. Regions that can: ie"},
595 // Rule 3: a claims-on box in either region, judged from its published state.
596 {name: "a claims-on box in the source region that serves the shared pool", body: body, code: 409, msg: msgO21("ie-claims"),
597 boxes: []offlineBox{ieOff("ie-box"), func() offlineBox { b := ieOff("ie-claims"); b.ClaimsOn = true; b.NoCap = true; return b }(), ukOff("uk-box")}},
598 {name: "a claims-on box in the target region whose pools do not match (no census evidence is published)", body: body, code: 409, msg: msgO21("uk-claims"),
599 boxes: []offlineBox{ieOff("ie-box"), ukOff("uk-box"), func() offlineBox {
600 b := ukOff("uk-claims")
601 b.ClaimsOn = true
602 b.NoCap = true
603 b.NotSharedPool = true
604 return b
605 }()}},
606 {name: "a claims-on box that publishes a row for the host", body: body, code: 409, msg: msgO21("ie-claims"),
607 boxes: []offlineBox{ieOff("ie-box"), func() offlineBox {
608 b := ieOff("ie-claims")
609 b.ClaimsOn, b.NoCap, b.NotSharedPool, b.Hosts = true, true, true, []string{offlineHost}
610 return b
611 }(), ukOff("uk-box")}},
612 {name: "a box whose report says claims are on although it carries the capability", body: body, code: 409, msg: msgO21("uk-box"),
613 boxes: []offlineBox{ieOff("ie-box"), func() offlineBox { b := ukOff("uk-box"); b.ClaimsOn = true; return b }()}},
614 {name: "a box outside both regions that runs claims is not consulted", body: "", code: 202,
615 boxes: []offlineBox{ieOff("ie-box"), ukOff("uk-box"), func() offlineBox {
616 b := offlineBox{Box: "se-box", Placement: "se", Bucket: "data-se"}
617 b.ClaimsOn, b.NoCap = true, true
618 return b
619 }()}},
620 {name: "a box with no metrics and no capability in the region cannot be judged", body: body, code: 409, msg: msgO21("ie-old"),
621 boxes: []offlineBox{ieOff("ie-box"), func() offlineBox {
622 b := ieOff("ie-old")
623 b.NoMetrics, b.NoCap, b.NotSharedPool = true, true, true
624 return b
625 }(), ukOff("uk-box")}},
626 }
627 for _, c := range cases {
628 t.Run(c.name, func(t *testing.T) {
629 w := newOfflineWorld(t, c.boxes...)
630 if c.setup != nil {
631 c.setup(w)
632 }
633 if c.code == 202 {
634 w.start()
635 return
636 }
637 w.refused(http.MethodPost, "", c.body, c.code, c.msg)
638 if rec := w.record(); rec.Move != nil && c.name != "a second move in flight" {
639 t.Fatalf("a refused start left a move: %+v", rec.Move)
640 }
641 })
642 }
643}
644
645// A done backfill plan does not refuse, and the claims-on truth table is
646// all three conditions: one missing is a refusal.
647func TestOfflineClaimsOnBoxNeedsAllThree(t *testing.T) {
648 yes, no := true, false
649 cases := []struct {
650 name string
651 box offlineClaimsOnBox
652 ok bool
653 }{
654 {"all three", offlineClaimsOnBox{Census: &yes, SharedPool: &no}, true},
655 {"a row for the host", offlineClaimsOnBox{HostRow: true, Census: &yes, SharedPool: &no}, false},
656 {"census not present", offlineClaimsOnBox{Census: &no, SharedPool: &no}, false},
657 {"census not said", offlineClaimsOnBox{SharedPool: &no}, false},
658 {"pools match", offlineClaimsOnBox{Census: &yes, SharedPool: &yes}, false},
659 {"pools not said", offlineClaimsOnBox{Census: &yes}, false},
660 }
661 for _, c := range cases {
662 if got := c.box.ok(); got != c.ok {
663 t.Errorf("%s: ok = %v, want %v", c.name, got, c.ok)
664 }
665 }
666 // No report field carries the census today, so no claims-on box passes.
667 if offlineCensusReported(&status.BoxMetrics{HostClaimEnabled: true}) != nil {
668 t.Error("a box report says something about census.json this build does not know how to read")
669 }
670}
671
672func TestOfflineStartWritesTheMoveAndBumpsTheSerial(t *testing.T) {
673 w := newOfflineWorld(t, oneEach...)
674 if v := w.status(); v.State != "none" || v.Host != offlineHost {
675 t.Fatalf("status before any move = %+v", v)
676 }
677 v := w.start()
678 if v.From != "ie" || v.To != "uk" || v.Host != offlineHost || !v.CanAbort || v.RequestedBy != "alice@keymail.dev" {
679 t.Fatalf("start view = %+v", v)
680 }
681 rec := w.record()
682 if rec.Move == nil || rec.Move.Kind != provision.MoveKindOffline || rec.Move.State != provision.OfflineStopping ||
683 rec.Move.From != "ie" || rec.Move.To != "uk" || rec.Move.ID != v.ID || rec.Region != "ie" {
684 t.Fatalf("record after start = %+v region %q", rec.Move, rec.Region)
685 }
686 // Waiting for the boxes, in the move's own words; no verb is next.
687 if v.Next != "" || v.Waiting != msgMC10("ie-box") {
688 t.Fatalf("view waits for %q (next %q)", v.Waiting, v.Next)
689 }
690 // A second start is refused by the record.
691 w.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusConflict, "a move to uk is already offline-stopping")
692 // The status now carries the manifest with the door-owned class row ok.
693 got := w.status()
694 if got.State != provision.OfflineStopping || len(got.Items) != len(moves.OfflineManifest) {
695 t.Fatalf("status = %+v", got)
696 }
697 for _, it := range got.Items {
698 switch it.Owner {
699 case moves.OfflineOwnerDoor:
700 if it.Status != "ok" {
701 t.Errorf("%s: %s", it.ID, it.Status)
702 }
703 case moves.OfflineOwnerNone:
704 if it.Status != "unchecked" {
705 t.Errorf("%s: %s", it.ID, it.Status)
706 }
707 default:
708 if it.Status != "waiting" {
709 t.Errorf("%s: %s", it.ID, it.Status)
710 }
711 }
712 }
713}
714
715func TestOfflineSealDoor(t *testing.T) {
716 w := newOfflineWorld(t, ieOff("ie-a"), ieOff("ie-b"), ukOff("uk-box"))
717 w.refused(http.MethodPost, "/seal", "", http.StatusConflict, offlineHost+" has no offline move in flight")
718 w.start()
719 // No report yet: the first box is named.
720 w.refused(http.MethodPost, "/seal", "", http.StatusConflict, msgMC10("ie-a"))
721 // A box that has not held, fenced and quiesced is named.
722 for _, lost := range []string{"held", "fenced", "quiesced"} {
723 w.stopped(func(box string, r *moves.OfflineReport) {
724 if box == "ie-b" {
725 switch lost {
726 case "held":
727 r.Held = false
728 case "fenced":
729 r.Fenced = false
730 case "quiesced":
731 r.Quiesced = false
732 }
733 }
734 }, "ie-a", "ie-b")
735 w.refused(http.MethodPost, "/seal", "", http.StatusConflict, msgMC10("ie-b"))
736 }
737 // A stale report is not a report.
738 w.stopped(func(box string, r *moves.OfflineReport) {
739 if box == "ie-b" {
740 r.At = time.Now().Add(-5 * time.Minute)
741 }
742 }, "ie-a", "ie-b")
743 w.refused(http.MethodPost, "/seal", "", http.StatusConflict, msgO178("ie-b"))
744 // A box still listing the host has not stopped serving it.
745 w.stopped(nil, "ie-a", "ie-b")
746 still := ieOff("ie-b")
747 still.Hosts = []string{offlineHost}
748 w.fleet(still)
749 w.refused(http.MethodPost, "/seal", "", http.StatusConflict, msgMC10("ie-b"))
750 w.fleet(ieOff("ie-b"))
751 // A sidecar anywhere.
752 w.stopped(func(box string, r *moves.OfflineReport) { r.Sidecar = box == "ie-b" }, "ie-a", "ie-b")
753 w.refused(http.MethodPost, "/seal", "", http.StatusConflict, "ie-b keeps a sidecar file for "+offlineHost+". instances with sidecars cannot move yet")
754 // Local data on two boxes.
755 w.stopped(func(box string, r *moves.OfflineReport) { r.LocalDB = true; r.Files = box == "ie-b" }, "ie-a", "ie-b")
756 w.refused(http.MethodPost, "/seal", "", http.StatusConflict, offlineHost+" keeps local data on more than one box: ie-a, ie-b. remove the stale copy first")
757 // One box with data is the sealer, even when it is not the first.
758 before := serialBytes(t, w.fx.rel)
759 w.stopped(func(box string, r *moves.OfflineReport) { r.Env = box == "ie-b" }, "ie-a", "ie-b")
760 v := w.ok("/seal", "", provision.OfflineSealing)
761 if v.Sealer != "ie-b" || strings.Join(v.Boxes, ",") != "ie-a,ie-b" {
762 t.Fatalf("sealer %q boxes %v", v.Sealer, v.Boxes)
763 }
764 if serialBytes(t, w.fx.rel) == before {
765 t.Fatal("seal did not bump the serial")
766 }
767 rec := w.record()
768 if rec.Move.State != provision.OfflineSealing || rec.Move.Sealer != "ie-b" || len(rec.Move.Boxes) != 2 {
769 t.Fatalf("record = %+v", rec.Move)
770 }
771 // seal is the verb of stopping only.
772 w.refused(http.MethodPost, "/seal", "", http.StatusConflict, msgO25(provision.OfflineSealing))
773 w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealing))
774}
775
776func TestOfflineSealWithNoDataTakesTheFirstBox(t *testing.T) {
777 w := newOfflineWorld(t, ieOff("ie-a"), ieOff("ie-b"), ukOff("uk-box"))
778 w.start()
779 w.stopped(func(box string, r *moves.OfflineReport) { r.RowPresent = false }, "ie-a", "ie-b")
780 if v := w.ok("/seal", "", provision.OfflineSealing); v.Sealer != "ie-a" {
781 t.Fatalf("sealer = %q", v.Sealer)
782 }
783}
784
785func TestOfflineFlipDoor(t *testing.T) {
786 w := newOfflineWorld(t, oneEach...)
787 w.start()
788 w.stopped(nil, "ie-box")
789 w.ok("/seal", "", provision.OfflineSealing)
790 w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealing))
791 seal := w.sealed("ie-box")
792 w.stopped(nil, "ie-box")
793 sealKey := moves.OfflineSealKey(offlineHost, w.id)
794 // seal.json is create-only: a second write does not replace it.
795 if _, err := w.fx.rel.Put(context.Background(), sealKey, []byte("{}"), ""); err == nil {
796 t.Fatal("the seal was written twice")
797 }
798 // The sealer's living report is rewritten on every pass and the seal is
799 // not: nothing the living report does moves the hash flip checks.
800 w.sealerFresh("ie-box")
801 w.report(moves.OfflineRoleSealer, "ie-box", moves.OfflinePhaseSealed, func(r *moves.OfflineReport) {
802 r.Items = map[string]moves.Check{} // the living report carries no items at all now
803 r.TombstoneOwner = moves.TombstoneOwner(w.id)
804 r.TombstoneUntil = time.Now().Add(time.Hour)
805 r.FinalTXID = 99
806 })
807 // A seal that is not the one the record names.
808 tampered := w.putSeal("ie-box", func(r *moves.OfflineReport) { r.FinalTXID = 99 })
809 if string(tampered) == string(seal) {
810 t.Fatal("the tampered seal is the seal")
811 }
812 w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealed))
813 offlinePut(t, w.fx.rel, sealKey, seal)
814 // No seal at all.
815 if err := w.fx.rel.Delete(context.Background(), sealKey); err != nil {
816 t.Fatal(err)
817 }
818 w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealed))
819 offlinePut(t, w.fx.rel, sealKey, seal)
820 // A seal item that is not OK, and a seal of another move's tombstone, each
821 // with a hash the record names.
822 bad := w.putSeal("ie-box", func(r *moves.OfflineReport) {
823 r.Items["files"] = moves.Check{OK: false, Reason: "a file the archive refuses"}
824 })
825 w.rehash(bad)
826 w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealed))
827 bad = w.putSeal("ie-box", func(r *moves.OfflineReport) { r.TombstoneOwner = moves.TombstoneOwner("mv_other") })
828 w.rehash(bad)
829 w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealed))
830 // A seal written by a box other than the recorded sealer.
831 bad = w.putSeal("ie-x", nil)
832 w.rehash(bad)
833 w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealed))
834 offlinePut(t, w.fx.rel, sealKey, seal)
835 w.rehash(seal)
836 // A process left, a layout journal, a stale report.
837 w.stopped(func(_ string, r *moves.OfflineReport) { r.Procs = 1 }, "ie-box")
838 w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO31("ie-box"))
839 w.stopped(func(_ string, r *moves.OfflineReport) { r.LayoutJournal = true }, "ie-box")
840 w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO31("ie-box"))
841 // A stale report is not a process (harness Run 9, finding 7): MV-O178,
842 // never MV-O31.
843 w.stopped(func(_ string, r *moves.OfflineReport) { r.At = time.Now().Add(-3 * time.Minute) }, "ie-box")
844 w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO178("ie-box"))
845 w.stopped(func(_ string, r *moves.OfflineReport) { r.Fenced = false }, "ie-box")
846 w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgMC10("ie-box"))
847 // All clear.
848 w.stopped(nil, "ie-box")
849 before := serialBytes(t, w.fx.rel)
850 v := w.ok("/flip", "", provision.OfflineLanding)
851 if v.Next != "" {
852 t.Fatalf("next after flip = %q", v.Next)
853 }
854 if serialBytes(t, w.fx.rel) == before {
855 t.Fatal("flip did not bump the serial")
856 }
857 if rec := w.record(); rec.Region != "uk" || rec.Move.State != provision.OfflineLanding {
858 t.Fatalf("after flip: region %q move %+v", rec.Region, rec.Move)
859 }
860}
861
862// rehash points the record's source evidence at a seal's bytes, as the
863// sealer's transition records its own seal.
864func (w *offlineWorld) rehash(seal []byte) {
865 w.t.Helper()
866 w.edit(func(r *provision.Record) error {
867 r.Move.Source.Evidence = moves.OfflineSealEvidence(offlineHost, w.id, seal)
868 return nil
869 })
870}
871
872func TestOfflineOpenDoorRechecksTheStop(t *testing.T) {
873 w := newOfflineWorld(t, oneEach...)
874 w.toLanded()
875 // The listing check of a source box is not the door's: the door reads its report.
876 reset := func() {
877 w.stopped(nil, "ie-box")
878 w.sealerFresh("ie-box")
879 w.report(moves.OfflineRoleTarget, "uk-box", moves.OfflinePhaseLanded, func(r *moves.OfflineReport) { r.Items = okChecks(moves.OfflineOwnerLanding) })
880 }
881 reset()
882 // A stale sealer report.
883 w.report(moves.OfflineRoleSealer, "ie-box", moves.OfflinePhaseSealed, func(r *moves.OfflineReport) {
884 r.Items = sealerChecks()
885 r.At = time.Now().Add(-3 * time.Minute)
886 r.TombstoneOwner = moves.TombstoneOwner(w.id)
887 r.TombstoneUntil = time.Now().Add(time.Hour)
888 })
889 w.refused(http.MethodPost, "/open", "", http.StatusConflict, msgO178("ie-box"))
890 reset()
891 // A tombstone that is another move's, or none, or run out.
892 for name, edit := range map[string]func(*moves.OfflineReport){
893 "none": func(r *moves.OfflineReport) { r.TombstoneOwner = "" },
894 "another": func(r *moves.OfflineReport) { r.TombstoneOwner = moves.TombstoneOwner("mv_other") },
895 "expired": func(r *moves.OfflineReport) { r.TombstoneUntil = time.Now().Add(-time.Minute) },
896 "unstated": func(r *moves.OfflineReport) { r.TombstoneUntil = time.Time{} },
897 } {
898 w.report(moves.OfflineRoleSealer, "ie-box", moves.OfflinePhaseSealed, func(r *moves.OfflineReport) {
899 r.Items = sealerChecks()
900 r.TombstoneOwner = moves.TombstoneOwner(w.id)
901 r.TombstoneUntil = time.Now().Add(time.Hour)
902 edit(r)
903 })
904 code, out := w.req(http.MethodPost, "/open", "")
905 if code != http.StatusConflict || errorText(out) != msgO25(provision.OfflineLanded) {
906 t.Fatalf("tombstone %s: %d %s", name, code, out)
907 }
908 reset()
909 }
910 // A source report that lost Fenced, or went stale.
911 w.stopped(func(_ string, r *moves.OfflineReport) { r.Fenced = false }, "ie-box")
912 w.refused(http.MethodPost, "/open", "", http.StatusConflict, msgMC10("ie-box"))
913 w.stopped(func(_ string, r *moves.OfflineReport) { r.At = time.Now().Add(-10 * time.Minute) }, "ie-box")
914 w.refused(http.MethodPost, "/open", "", http.StatusConflict, msgO178("ie-box"))
915 reset()
916 // The seal changed under the move.
917 sealKey := moves.OfflineSealKey(offlineHost, w.id)
918 sealRaw, _, err := w.fx.rel.Get(context.Background(), sealKey)
919 if err != nil {
920 t.Fatal(err)
921 }
922 offlinePut(t, w.fx.rel, sealKey, append(append([]byte{}, sealRaw...), ' '))
923 w.refused(http.MethodPost, "/open", "", http.StatusConflict, msgO25(provision.OfflineLanded))
924 offlinePut(t, w.fx.rel, sealKey, sealRaw)
925 // A landing item that is not OK.
926 w.report(moves.OfflineRoleTarget, "uk-box", moves.OfflinePhaseLanded, func(r *moves.OfflineReport) {
927 r.Items = okChecks(moves.OfflineOwnerLanding)
928 r.Items["config"] = moves.Check{OK: false, Reason: "the render waits"}
929 })
930 w.refused(http.MethodPost, "/open", "", http.StatusConflict, msgO25(provision.OfflineLanded))
931 // The status view says the same, and names no verb.
932 if v := w.status(); v.Next != "" || v.Waiting != msgO25(provision.OfflineLanded) {
933 t.Fatalf("status = next %q waiting %q", v.Next, v.Waiting)
934 } else {
935 for _, it := range v.Items {
936 if it.ID == "config" && (it.Status != "waiting" || it.Detail != "the render waits") {
937 t.Fatalf("config item = %+v", it)
938 }
939 }
940 }
941 reset()
942 if v := w.status(); v.Next != offlineVerbOpen {
943 t.Fatalf("status next = %q (%s)", v.Next, v.Waiting)
944 }
945 before := serialBytes(t, w.fx.rel)
946 w.ok("/open", "", provision.OfflineOpen)
947 if serialBytes(t, w.fx.rel) == before {
948 t.Fatal("open did not bump the serial")
949 }
950 if rec := w.record(); rec.Move.State != provision.OfflineOpen || rec.Region != "uk" || rec.Move.OpenedAt == nil {
951 t.Fatalf("after open: %+v region %q", rec.Move, rec.Region)
952 }
953}
954
955func TestOfflineFinishDoor(t *testing.T) {
956 w := newOfflineWorld(t, oneEach...)
957 w.toLanded()
958 w.ok("/open", "", provision.OfflineOpen)
959 // open to finishing: the target's opened and every source's parked.
960 w.refused(http.MethodPost, "/finish", "", http.StatusConflict, msgO25(provision.OfflineOpen))
961 w.report(moves.OfflineRoleTarget, "uk-box", moves.OfflinePhaseOpened, func(r *moves.OfflineReport) { r.Items = okChecks(moves.OfflineOwnerLanding) })
962 w.refused(http.MethodPost, "/finish", "", http.StatusConflict, msgO25(provision.OfflineOpen))
963 // A parked report whose inventory is not settled is not parked.
964 for name, retired := range map[string][]moves.RetiredPath{
965 "an unknown action": {{Path: "/data/acme.shop.example.com", Action: "deleted"}},
966 "left, no reason": {{Path: "/opt/carlos/live/acme.shop.example.com", Action: moves.RetiredLeft}},
967 "no path": {{Action: moves.RetiredParked}},
968 } {
969 w.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseParked, func(r *moves.OfflineReport) { r.Retired = retired })
970 code, out := w.req(http.MethodPost, "/finish", "")
971 if code != http.StatusConflict || errorText(out) != msgO25(provision.OfflineOpen) {
972 t.Fatalf("%s: %d %s", name, code, out)
973 }
974 }
975 // A path the box could not park is terminal when it says why, and the
976 // status view shows it.
977 w.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseParked, func(r *moves.OfflineReport) {
978 r.Retired = []moves.RetiredPath{
979 {Path: "/data/acme.shop.example.com", Action: moves.RetiredParked},
980 {Path: "/data/acme.shop.example.com.env", Action: moves.RetiredAbsent},
981 {Path: "/opt/carlos/live/acme.shop.example.com", Action: moves.RetiredLeft, Reason: "root-owned"},
982 }
983 })
984 if v := w.status(); v.Next != offlineVerbFinish || len(v.Left) != 1 || v.Left[0] != (offlineLeftView{Box: "ie-box", Path: "/opt/carlos/live/acme.shop.example.com", Reason: "root-owned"}) {
985 t.Fatalf("status = next %q left %+v waiting %q", v.Next, v.Left, v.Waiting)
986 }
987 before := serialBytes(t, w.fx.rel)
988 w.ok("/finish", "", provision.OfflineFinishing)
989 if serialBytes(t, w.fx.rel) == before {
990 t.Fatal("finish did not bump the serial")
991 }
992 // finishing to cleared: the sealer lifted and every source gone.
993 w.refused(http.MethodPost, "/finish", "", http.StatusConflict, msgO25(provision.OfflineFinishing))
994 w.report(moves.OfflineRoleSealer, "ie-box", moves.OfflinePhaseLifted, nil)
995 w.refused(http.MethodPost, "/finish", "", http.StatusConflict, msgO25(provision.OfflineFinishing))
996 w.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseGone, nil)
997 v := w.ok("/finish", "", "none")
998 if v.Last == nil || v.Last.Outcome != provision.OfflineOutcomeDone || v.Last.ID != w.id {
999 t.Fatalf("last = %+v", v.Last)
1000 }
1001 if rec := w.record(); rec.Move != nil || rec.Region != "uk" || rec.LastMove == nil {
1002 t.Fatalf("after finish: move %+v region %q", rec.Move, rec.Region)
1003 }
1004 w.refused(http.MethodPost, "/finish", "", http.StatusConflict, offlineHost+" has no offline move in flight")
1005 // The status of a host with no move is the outcome of the last.
1006 if s := w.status(); s.State != "none" || s.Last == nil {
1007 t.Fatalf("status = %+v", s)
1008 }
1009}
1010
1011func TestOfflineAbortDoor(t *testing.T) {
1012 // Before the flip: the region is untouched.
1013 w := newOfflineWorld(t, oneEach...)
1014 w.refused(http.MethodPost, "/abort", "", http.StatusConflict, offlineHost+" has no offline move in flight")
1015 w.start()
1016 before := serialBytes(t, w.fx.rel)
1017 w.ok("/abort", "", provision.OfflineReverting)
1018 if serialBytes(t, w.fx.rel) == before {
1019 t.Fatal("abort did not bump the serial")
1020 }
1021 if rec := w.record(); rec.Region != "ie" || rec.Move.State != provision.OfflineReverting {
1022 t.Fatalf("after abort: %q %+v", rec.Region, rec.Move)
1023 }
1024 // A second abort writes nothing.
1025 state := bucketState(t, w.fx.rel)
1026 w.ok("/abort", "", provision.OfflineReverting)
1027 for k, v := range bucketState(t, w.fx.rel) {
1028 if state[k] != v {
1029 t.Fatalf("a second abort changed %s", k)
1030 }
1031 }
1032 // reverting to cleared: every source box's reverted. Aborted before the
1033 // seal there is no sealer to ask, and no target either.
1034 w.refused(http.MethodPost, "/finish", "", http.StatusConflict, msgO25(provision.OfflineReverting))
1035 w.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseReverted, nil)
1036 if v := w.ok("/finish", "", "none"); v.Last == nil || v.Last.Outcome != provision.OfflineOutcomeAborted {
1037 t.Fatalf("last = %+v", v.Last)
1038 }
1039
1040 // After the flip: the record goes back to the source.
1041 w2 := newOfflineWorld(t, oneEach...)
1042 w2.start()
1043 w2.stopped(nil, "ie-box")
1044 w2.ok("/seal", "", provision.OfflineSealing)
1045 w2.sealed("ie-box")
1046 w2.stopped(nil, "ie-box")
1047 w2.ok("/flip", "", provision.OfflineLanding)
1048 if rec := w2.record(); rec.Region != "uk" {
1049 t.Fatalf("region after flip = %q", rec.Region)
1050 }
1051 w2.ok("/abort", "", provision.OfflineReverting)
1052 if rec := w2.record(); rec.Region != "ie" {
1053 t.Fatalf("region after abort = %q", rec.Region)
1054 }
1055 // Sealed and then aborted: the sealer's reverted is asked for too, and
1056 // the target's, which may be a box that died, is not.
1057 w2.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseReverted, nil)
1058 w2.refused(http.MethodPost, "/finish", "", http.StatusConflict, msgO25(provision.OfflineReverting))
1059 w2.report(moves.OfflineRoleSealer, "ie-box", moves.OfflinePhaseReverted, nil)
1060 if v := w2.ok("/finish", "", "none"); v.Last == nil || v.Last.Outcome != provision.OfflineOutcomeAborted {
1061 t.Fatalf("last = %+v", v.Last)
1062 }
1063 if rec := w2.record(); rec.Region != "ie" || rec.Move != nil {
1064 t.Fatalf("after finish: %q %+v", rec.Region, rec.Move)
1065 }
1066
1067 // From open on, abort is MV-O27 and writes nothing.
1068 w3 := newOfflineWorld(t, oneEach...)
1069 w3.toLanded()
1070 w3.ok("/open", "", provision.OfflineOpen)
1071 w3.refused(http.MethodPost, "/abort", "", http.StatusConflict, offlineHost+" is already served from uk. move it back with a new move")
1072 w3.edit(provision.OfflineFinishingTransition(w3.id))
1073 w3.refused(http.MethodPost, "/abort", "", http.StatusConflict, offlineHost+" is already served from uk. move it back with a new move")
1074}
1075
1076// The status view is the move, every report, the manifest with each item's
1077// latest check and the next legal verb.
1078func TestOfflineStatusView(t *testing.T) {
1079 w := newOfflineWorld(t, oneEach...)
1080 w.toLanded()
1081 v := w.status()
1082 if v.State != provision.OfflineLanded || v.ID != w.id || v.Sealer != "ie-box" || v.From != "ie" || v.To != "uk" || !v.CanAbort || v.Next != offlineVerbOpen {
1083 t.Fatalf("status = %+v", v)
1084 }
1085 if v.Failed {
1086 t.Error("a healthy move is marked failed")
1087 }
1088 roles := map[string]bool{}
1089 for _, r := range v.Reports {
1090 roles[r.Role+"/"+r.Box] = true
1091 if !r.Fresh {
1092 t.Errorf("report %s/%s is not fresh", r.Role, r.Box)
1093 }
1094 }
1095 for _, want := range []string{"source/ie-box", "sealer/ie-box", "target/uk-box"} {
1096 if !roles[want] {
1097 t.Errorf("no %s report in the view: %v", want, roles)
1098 }
1099 }
1100 if len(v.Items) != len(moves.OfflineManifest) {
1101 t.Fatalf("%d items, want %d", len(v.Items), len(moves.OfflineManifest))
1102 }
1103 for _, it := range v.Items {
1104 if (it.Owner == moves.OfflineOwnerSeal || it.Owner == moves.OfflineOwnerLanding) && it.Status != "ok" {
1105 t.Errorf("item %s is %s", it.ID, it.Status)
1106 }
1107 }
1108 // The step that owns an item decides it: a landing item is the target's
1109 // to prove, a seal item the sealer's. A failed phase marks a non-OK
1110 // item failed; a landing phase marks it waiting.
1111 w.report(moves.OfflineRoleSealer, "ie-box", moves.OfflinePhaseFailed, func(r *moves.OfflineReport) {
1112 r.Items = sealerChecks()
1113 r.Items["db-tail"] = moves.Check{OK: false, Reason: "the replica is behind"}
1114 r.Items["files"] = moves.Check{OK: false, Reason: "the sealer's own view does not decide it"}
1115 })
1116 w.report(moves.OfflineRoleTarget, "uk-box", moves.OfflinePhaseLanded, func(r *moves.OfflineReport) {
1117 r.Items = okChecks(moves.OfflineOwnerLanding)
1118 delete(r.Items, "release")
1119 r.Items["files"] = moves.Check{OK: false, Reason: "the digest differs"}
1120 })
1121 if !w.status().Failed {
1122 t.Error("a sealer that reported failed is not marked failed in the view")
1123 }
1124 seen := map[string]offlineItemView{}
1125 for _, it := range w.status().Items {
1126 seen[it.ID] = it
1127 }
1128 if it := seen["db-tail"]; it.Status != "failed" || it.Detail != "the replica is behind" {
1129 t.Errorf("db-tail = %+v", it)
1130 }
1131 if it := seen["files"]; it.Status != "waiting" || it.Detail != "the digest differs" || it.Owner != moves.OfflineOwnerLanding || it.Source != moves.OfflineSourceSeal {
1132 t.Errorf("files = %+v", it)
1133 }
1134 if it := seen["release"]; it.Status != "waiting" {
1135 t.Errorf("release = %+v", it)
1136 }
1137 // A report that is not this move's, or not at its own key, is not in the view.
1138 other := moves.OfflineReport{V: 1, Op: "mv_other", Host: offlineHost, Box: "ie-box", Role: moves.OfflineRoleSource, Phase: moves.OfflinePhaseStopped, At: time.Now()}
1139 body, _ := json.Marshal(other)
1140 offlinePut(t, w.fx.rel, moves.OfflineReportKey(offlineHost, w.id, moves.OfflineRoleSource, "ie-x"), body)
1141 for _, r := range w.status().Reports {
1142 if r.Box == "ie-x" {
1143 t.Errorf("a report at the wrong key is in the view: %+v", r)
1144 }
1145 }
1146}
1147
1148// Evidence matching takes the report's hash bare or after its key.
1149func TestOfflineEvidenceMatches(t *testing.T) {
1150 raw := []byte(`{"x":1}`)
1151 sum := sha256.Sum256(raw)
1152 h := hex.EncodeToString(sum[:])
1153 key := moves.OfflineReportKey(offlineHost, "mv_00000000000000000000000000", moves.OfflineRoleSealer, "ie-box")
1154 for _, ok := range []string{h, "sha256:" + h, key + "@" + h, key + ":" + h, key + " " + h, key + "#sha256:" + h} {
1155 if !offlineEvidenceMatches(ok, key, raw) {
1156 t.Errorf("%q should match", ok)
1157 }
1158 }
1159 for _, bad := range []string{"", key, key + "@", key + "@" + strings.Repeat("0", 64), "other/key@" + h, h + "0", key + h} {
1160 if offlineEvidenceMatches(bad, key, raw) {
1161 t.Errorf("%q should not match", bad)
1162 }
1163 }
1164}
1165
1166// A transition's refusal reaches the operator as a recorded sentence and
1167// never as the error's own text: the offline transitions' rows pass as they
1168// are (the values in them are the record's own), and anything else is
1169// MV-N46.
1170func TestOfflineConflictMapsOntoRecordedSentences(t *testing.T) {
1171 w := newOfflineWorld(t, oneEach...)
1172 w.start()
1173 apply := func(mut func(*provision.Record) error) error {
1174 _, err := provision.SetRecordMove(context.Background(), w.fx.rel, "bab", "shop", offlineHost, mut)
1175 return err
1176 }
1177 const other = "mv_ZZZZZZZZZZZZZZZZZZZZZZZZZZ"
1178 cases := []struct {
1179 name string
1180 mut func(*provision.Record) error
1181 want string
1182 }{
1183 {"wrong state", provision.OfflineFlip(w.id), "this offline move is offline-stopping. this step follows offline-sealed"},
1184 {"another move's id", provision.OfflineSeal(other, "ie-box", []string{"ie-box"}), "the record carries a different offline move"},
1185 {"a seal without its sealer among the boxes", provision.OfflineSeal(w.id, "ie-x", []string{"ie-box"}),
1186 "a seal names the box that seals and the boxes that stopped the instance. the box that seals is one of them"},
1187 {"an abort already past", provision.OfflineFinish(w.id, time.Now()), "this offline move is offline-stopping. it finishes from finishing or reverting"},
1188 }
1189 for _, c := range cases {
1190 err := apply(c.mut)
1191 if err == nil {
1192 t.Fatalf("%s: accepted", c.name)
1193 }
1194 if got := offlineConflict(err); got != c.want {
1195 t.Errorf("%s: %q, want %q", c.name, got, c.want)
1196 }
1197 }
1198 // A refusal this table does not know is the fixed sentence.
1199 if got := offlineConflict(fmt.Errorf("%w: something new and unreviewed", provision.ErrMoveConflict)); got != "this move's state does not allow that. read its state and try again" {
1200 t.Errorf("unknown refusal = %q", got)
1201 }
1202 // Once open, an abort is refused with its recorded sentence.
1203 w.stopped(nil, "ie-box")
1204 w.ok("/seal", "", provision.OfflineSealing)
1205 w.sealed("ie-box")
1206 w.stopped(nil, "ie-box")
1207 w.ok("/flip", "", provision.OfflineLanding)
1208 w.landed("uk-box")
1209 w.edit(provision.OfflineOpenTransition(w.id, time.Now()))
1210 err := apply(provision.OfflineAbort(w.id))
1211 if got := offlineConflict(err); got != "this offline move is offline-open. it cannot be aborted once it is open" {
1212 t.Errorf("abort after open = %q", got)
1213 }
1214 // A legacy move meets "this is not an offline move".
1215 if got := offlineConflict(fmt.Errorf("%w: %s", provision.ErrMoveConflict, "this is not an offline move")); got != "this is not an offline move" {
1216 t.Errorf("not offline = %q", got)
1217 }
1218}
1219
1220// The flagship's shape: the home placement is the record's empty region and
1221// the console names it. A move out of home reads the placementless boxes, a
1222// move into it the same, and the operator reads the home by its name.
1223func TestOfflineMovesOutOfAndIntoHome(t *testing.T) {
1224 homeBox := offlineBox{Box: "home-box", Bucket: "data-ie"}
1225 w := newOfflineWorldIn(t, "ie", "ie", homeBox, ukOff("uk-box"))
1226 if rec := w.record(); rec.Region != "" {
1227 t.Fatalf("the record's region = %q, want the home's empty code", rec.Region)
1228 }
1229 w.refused(http.MethodPost, "", `{"region":"ie"}`, http.StatusConflict, "the instance is already in ie")
1230 v := w.start()
1231 if v.From != "ie" || v.To != "uk" || v.Waiting != "Waiting for home-box in ie to confirm it has stopped "+offlineHost+". "+offlineHost+" is offline." {
1232 t.Fatalf("view = %+v", v)
1233 }
1234 if rec := w.record(); rec.Move.From != "" || rec.Move.To != "uk" || rec.Region != "" {
1235 t.Fatalf("record = %+v region %q", rec.Move, rec.Region)
1236 }
1237 w.report(moves.OfflineRoleSource, "home-box", moves.OfflinePhaseStopped, nil)
1238 if v := w.ok("/seal", "", provision.OfflineSealing); v.Sealer != "home-box" {
1239 t.Fatalf("sealer = %q", v.Sealer)
1240 }
1241
1242 // Back the other way: from uk into home.
1243 b := newOfflineWorldIn(t, "ie", "uk", homeBox, ukOff("uk-box"))
1244 bv := b.ok("", `{"region":"ie"}`, provision.OfflineStopping)
1245 if bv.From != "uk" || bv.To != "ie" {
1246 t.Fatalf("view = %+v", bv)
1247 }
1248 if rec := b.record(); rec.Move.From != "uk" || rec.Move.To != "" || rec.Region != "uk" {
1249 t.Fatalf("record = %+v region %q", rec.Move, rec.Region)
1250 }
1251
1252 // A box that sets CARLOS_REGION to the home's name is home to the console
1253 // and not to itself: it cannot take a move into home.
1254 named := offlineBox{Box: "ie-named", Placement: "ie", Bucket: "data-ie"}
1255 c := newOfflineWorldIn(t, "ie", "uk", homeBox, ukOff("uk-box"), named)
1256 c.refused(http.MethodPost, "", `{"region":"ie"}`, http.StatusConflict, "ie-named sets CARLOS_REGION, so it cannot take a move into the home region")
1257
1258 // And out of home, where its published placement is not the record's.
1259 d := newOfflineWorldIn(t, "ie", "ie", homeBox, ukOff("uk-box"), named)
1260 d.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusConflict, "ie-named sets CARLOS_REGION, so it cannot take a move into the home region")
1261}
1262
1263// startOther creates a second instance and starts it through its own door.
1264func (w *offlineWorld) startOther(host string) (offlineView, int, string) {
1265 w.t.Helper()
1266 u := instancesURL(w.fx, "bab", "shop")
1267 if c, _, out := rawReq(w.t, http.MethodPost, u, w.token, `{"host":"`+host+`","region":"ie"}`); c != http.StatusCreated && c != http.StatusConflict {
1268 w.t.Fatalf("create %s = %d %s", host, c, out)
1269 }
1270 code, _, out := rawReq(w.t, http.MethodPost, u+"/"+host+"/move/offline", w.token, `{"region":"uk"}`)
1271 var v offlineView
1272 _ = json.Unmarshal([]byte(out), &v)
1273 return v, code, out
1274}
1275
1276func (w *offlineWorld) lock() (moves.OfflineActive, bool) {
1277 w.t.Helper()
1278 raw, _, err := w.fx.rel.Get(context.Background(), moves.OfflineActiveKey)
1279 if err != nil {
1280 return moves.OfflineActive{}, false
1281 }
1282 var a moves.OfflineActive
1283 if err := json.Unmarshal(raw, &a); err != nil {
1284 w.t.Fatal(err)
1285 }
1286 return a, true
1287}
1288
1289// Two starts at once for different hosts: exactly one wins the lock.
1290func TestOfflineConcurrentStartsOneWins(t *testing.T) {
1291 w := newOfflineWorld(t, oneEach...)
1292 u := instancesURL(w.fx, "bab", "shop")
1293 if c, _, out := rawReq(t, http.MethodPost, u, w.token, `{"host":"beta.shop.example.com","region":"ie"}`); c != http.StatusCreated {
1294 t.Fatalf("create = %d %s", c, out)
1295 }
1296 hosts := []string{offlineHost, "beta.shop.example.com"}
1297 type answer struct {
1298 code int
1299 out string
1300 }
1301 got := make([]answer, len(hosts))
1302 var wg sync.WaitGroup
1303 gate := make(chan struct{})
1304 for i, h := range hosts {
1305 wg.Add(1)
1306 go func() {
1307 defer wg.Done()
1308 <-gate
1309 c, _, out := rawReq(t, http.MethodPost, u+"/"+h+"/move/offline", w.token, `{"region":"uk"}`)
1310 got[i] = answer{c, out}
1311 }()
1312 }
1313 close(gate)
1314 wg.Wait()
1315 won, lost := -1, -1
1316 for i, a := range got {
1317 switch a.code {
1318 case http.StatusAccepted:
1319 won = i
1320 case http.StatusConflict:
1321 lost = i
1322 default:
1323 t.Fatalf("start %s = %d %s", hosts[i], a.code, a.out)
1324 }
1325 }
1326 if won < 0 || lost < 0 {
1327 t.Fatalf("want one winner and one loser: %+v", got)
1328 }
1329 if want := hosts[won] + " is moving offline. move one instance at a time"; errorText(got[lost].out) != want {
1330 t.Fatalf("the loser was told %q, want %q", errorText(got[lost].out), want)
1331 }
1332 if a, ok := w.lock(); !ok || a.Host != hosts[won] {
1333 t.Fatalf("lock = %+v %v, want the winner %s", a, ok, hosts[won])
1334 }
1335 loser, err := provision.GetRecord(context.Background(), w.fx.rel, "bab", "shop", hosts[lost])
1336 if err != nil || loser.Move != nil {
1337 t.Fatalf("the loser's record = %+v %v", loser.Move, err)
1338 }
1339}
1340
1341// The lock is released when the move ends, by finish and by the finish that
1342// ends an abort, and a stale lock does not stop the next start.
1343func TestOfflineLockLifecycle(t *testing.T) {
1344 w := newOfflineWorld(t, oneEach...)
1345 w.toLanded()
1346 if a, ok := w.lock(); !ok || a.Host != offlineHost || a.Op != w.id || a.Account != "bab" || a.App != "shop" {
1347 t.Fatalf("lock = %+v %v", a, ok)
1348 }
1349 w.ok("/open", "", provision.OfflineOpen)
1350 w.report(moves.OfflineRoleTarget, "uk-box", moves.OfflinePhaseOpened, func(r *moves.OfflineReport) { r.Items = okChecks(moves.OfflineOwnerLanding) })
1351 w.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseParked, nil)
1352 w.ok("/finish", "", provision.OfflineFinishing)
1353 if _, ok := w.lock(); !ok {
1354 t.Fatal("the lock was released before the move ended")
1355 }
1356 w.report(moves.OfflineRoleSealer, "ie-box", moves.OfflinePhaseLifted, nil)
1357 w.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseGone, nil)
1358 w.ok("/finish", "", "none")
1359 if _, ok := w.lock(); ok {
1360 t.Fatal("finish left the lock")
1361 }
1362
1363 // An abort ends at finish too.
1364 b := newOfflineWorld(t, oneEach...)
1365 b.start()
1366 b.ok("/abort", "", provision.OfflineReverting)
1367 if _, ok := b.lock(); !ok {
1368 t.Fatal("abort released the lock before the finish")
1369 }
1370 b.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseReverted, nil)
1371 b.ok("/finish", "", "none")
1372 if _, ok := b.lock(); ok {
1373 t.Fatal("the finish after an abort left the lock")
1374 }
1375 // Another start now takes it.
1376 b.start()
1377 if a, ok := b.lock(); !ok || a.Op != b.id {
1378 t.Fatalf("lock = %+v %v", a, ok)
1379 }
1380}
1381
1382func TestOfflineStaleLockIsReplaced(t *testing.T) {
1383 put := func(w *offlineWorld, a moves.OfflineActive) {
1384 body, _ := json.Marshal(a)
1385 offlinePut(t, w.fx.rel, moves.OfflineActiveKey, body)
1386 }
1387 // A lock naming a host with no record.
1388 w := newOfflineWorld(t, oneEach...)
1389 put(w, moves.OfflineActive{Host: "gone.shop.example.com", Op: "mv_aaaaaaaaaaaaaaaaaaaaaaaaaa", Account: "bab", App: "shop", At: time.Now().Add(-time.Hour)})
1390 w.start()
1391 if a, _ := w.lock(); a.Host != offlineHost || a.Op != w.id {
1392 t.Fatalf("lock = %+v", a)
1393 }
1394 // A lock naming a host whose record carries no such move (a failed release).
1395 x := newOfflineWorld(t, oneEach...)
1396 if c, _, out := rawReq(t, http.MethodPost, instancesURL(x.fx, "bab", "shop"), x.token, `{"host":"beta.shop.example.com","region":"ie"}`); c != http.StatusCreated {
1397 t.Fatalf("create = %d %s", c, out)
1398 }
1399 put(x, moves.OfflineActive{Host: "beta.shop.example.com", Op: "mv_aaaaaaaaaaaaaaaaaaaaaaaaaa", Account: "bab", App: "shop", At: time.Now().Add(-time.Hour)})
1400 x.start()
1401 // A lock whose record carries another move of that host is not this one's.
1402 y := newOfflineWorld(t, oneEach...)
1403 put(y, moves.OfflineActive{Host: offlineHost, Op: "mv_aaaaaaaaaaaaaaaaaaaaaaaaaa", Account: "bab", App: "shop", At: time.Now().Add(-time.Hour)})
1404 y.start()
1405 // A young lock is believed even before its record carries the move: the
1406 // start door takes the lock, then writes the record.
1407 f := newOfflineWorld(t, oneEach...)
1408 if c, _, out := rawReq(t, http.MethodPost, instancesURL(f.fx, "bab", "shop"), f.token, `{"host":"beta.shop.example.com","region":"ie"}`); c != http.StatusCreated {
1409 t.Fatalf("create = %d %s", c, out)
1410 }
1411 put(f, moves.OfflineActive{Host: "beta.shop.example.com", Op: "mv_aaaaaaaaaaaaaaaaaaaaaaaaaa", Account: "bab", App: "shop", At: time.Now()})
1412 f.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusConflict, "beta.shop.example.com is moving offline. move one instance at a time")
1413 // An old lock is live while its record carries the move.
1414 g := newOfflineWorld(t, oneEach...)
1415 if _, code, out := g.startOther("beta.shop.example.com"); code != http.StatusAccepted {
1416 t.Fatalf("beta start = %d %s", code, out)
1417 }
1418 a, _ := g.lock()
1419 a.At = time.Now().Add(-24 * time.Hour)
1420 put(g, a)
1421 g.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusConflict, "beta.shop.example.com is moving offline. move one instance at a time")
1422 // An unreadable lock is not guessed at.
1423 z := newOfflineWorld(t, oneEach...)
1424 offlinePut(t, z.fx.rel, moves.OfflineActiveKey, []byte("not json"))
1425 z.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusBadGateway, "could not read the instance")
1426}
1427
1428// A record nobody can read elsewhere does not block a start: the lock is the
1429// gate, not a listing of every record.
1430func TestOfflineStartIgnoresOtherRecords(t *testing.T) {
1431 w := newOfflineWorld(t, oneEach...)
1432 offlinePut(t, w.fx.rel, provision.RecordKey("bab", "shop", "junk.shop.example.com"), []byte("{not a record"))
1433 w.start()
1434}
1435
1436// A failed record write does not prove nothing was written, so start never
1437// releases its lock on one. A write that lands and then reports an error is a
1438// start that succeeded; a write that does not land leaves the lock, which
1439// goes stale by itself. Either way no other host starts meanwhile.
1440func TestOfflineStartKeepsTheLockWhenTheRecordWriteFails(t *testing.T) {
1441 for _, landed := range []bool{true, false} {
1442 t.Run(fmt.Sprintf("landed=%v", landed), func(t *testing.T) {
1443 w := newOfflineWorld(t, oneEach...)
1444 if c, _, out := rawReq(t, http.MethodPost, instancesURL(w.fx, "bab", "shop"), w.token, `{"host":"beta.shop.example.com","region":"ie"}`); c != http.StatusCreated {
1445 t.Fatalf("create = %d %s", c, out)
1446 }
1447 w.fx.srv.cfg.Releases = &recordWriteFault{Store: w.fx.rel, key: provision.RecordKey("bab", "shop", offlineHost), land: landed}
1448 code, out := w.req(http.MethodPost, "", `{"region":"uk"}`)
1449 if landed {
1450 // The record carries the move: the start succeeded.
1451 if code != http.StatusAccepted {
1452 t.Fatalf("start = %d %s", code, out)
1453 }
1454 if v := w.view(code, out); v.State != provision.OfflineStopping || v.ID == "" {
1455 t.Fatalf("view = %+v", v)
1456 }
1457 } else if code != http.StatusBadGateway || errorText(out) != "could not write the move. try again" {
1458 t.Fatalf("start = %d %s", code, out)
1459 }
1460 a, ok := w.lock()
1461 if !ok || a.Host != offlineHost {
1462 t.Fatalf("lock = %+v %v, want it kept", a, ok)
1463 }
1464 // A second host's start is refused whichever way the write went.
1465 bu := instancesURL(w.fx, "bab", "shop") + "/beta.shop.example.com/move/offline"
1466 c, _, o := rawReq(t, http.MethodPost, bu, w.token, `{"region":"uk"}`)
1467 if c != http.StatusConflict || errorText(o) != offlineHost+" is moving offline. move one instance at a time" {
1468 t.Fatalf("beta start = %d %s", c, o)
1469 }
1470 })
1471 }
1472}
1473