| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "net/http" |
| 6 | "slices" |
| 7 | "testing" |
| 8 | "time" |
| 9 | |
| 10 | "github.com/carlosframework/platform/internal/moves" |
| 11 | "github.com/carlosframework/platform/internal/provision" |
| 12 | ) |
| 13 | |
| 14 | // The pre-copy door (offline-staging), in-process against the offline |
| 15 | // doors' console fixture: the boxes' side is played by the reports they |
| 16 | // write, the record moves only through provision's offline transitions. |
| 17 | |
| 18 | func msgO179(to string) string { |
| 19 | return offlineHost + " is staged for " + to + ". abort the move to move it to another region" |
| 20 | } |
| 21 | |
| 22 | func msgO180(box string) string { |
| 23 | return box + " cannot stage an offline move. it runs an older release" |
| 24 | } |
| 25 | |
| 26 | // stage runs the stage door on a record at rest and keeps the id. |
| 27 | func (w *offlineWorld) stage() offlineView { |
| 28 | w.t.Helper() |
| 29 | before := serialBytes(w.t, w.fx.rel) |
| 30 | v := w.ok("/stage", `{"region":"uk"}`, provision.OfflineStaging) |
| 31 | if v.ID == "" { |
| 32 | w.t.Fatalf("stage answered no id: %+v", v) |
| 33 | } |
| 34 | if serialBytes(w.t, w.fx.rel) == before { |
| 35 | w.t.Fatal("stage did not bump the serial") |
| 36 | } |
| 37 | w.id = v.ID |
| 38 | return v |
| 39 | } |
| 40 | |
| 41 | // staged writes the stager's report as the stager writes it after a round. |
| 42 | func (w *offlineWorld) staged(box string, edit func(*moves.OfflineReport)) { |
| 43 | w.t.Helper() |
| 44 | w.report(moves.OfflineRoleStager, box, moves.OfflinePhaseStaged, func(r *moves.OfflineReport) { |
| 45 | r.Held, r.Fenced, r.Quiesced = false, false, false |
| 46 | r.ReplicaObjects, r.ReplicaBytes, r.StagedObjects, r.StagedBytes, r.RoundObjects, r.RoundBytes = 305, 1_580_000_000, 300, 1_570_000_000, 12, 9_000_000 |
| 47 | if edit != nil { |
| 48 | edit(r) |
| 49 | } |
| 50 | }) |
| 51 | } |
| 52 | |
| 53 | func TestOfflineStageThenStartThenSeal(t *testing.T) { |
| 54 | us := offlineBox{Box: "us-box", Placement: "us", Bucket: "data-us"} |
| 55 | w := newOfflineWorld(t, ieOff("ie-box"), ukOff("uk-box"), us) |
| 56 | v := w.stage() |
| 57 | rec := w.record() |
| 58 | if rec.Region != "ie" || rec.Move.State != provision.OfflineStaging || rec.Move.To != "uk" || rec.Move.StagedAt == nil || rec.Move.Stager != "ie-box" { |
| 59 | t.Fatalf("staged record: %q %+v", rec.Region, rec.Move) |
| 60 | } |
| 61 | if !v.CanAbort || v.Next != offlineVerbStart || v.Stage != nil { |
| 62 | t.Fatalf("staged view: next %q, can abort %v, stage %+v", v.Next, v.CanAbort, v.Stage) |
| 63 | } |
| 64 | if l, ok := w.lock(); !ok || l.Host != offlineHost || l.Op != w.id { |
| 65 | t.Fatalf("lock %+v %v, want this move's", l, ok) |
| 66 | } |
| 67 | |
| 68 | if !v.NeverOffline || v.StagedAt != rec.Move.StagedAt.UnixNano() { |
| 69 | t.Fatalf("staged view: never offline %v, staged at %d (record %v)", v.NeverOffline, v.StagedAt, rec.Move.StagedAt) |
| 70 | } |
| 71 | |
| 72 | // Again: one more round. The request moves forward, the rest of the |
| 73 | // move does not, and the serial moves. |
| 74 | serial := serialBytes(t, w.fx.rel) |
| 75 | again := w.ok("/stage", `{"region":"uk"}`, provision.OfflineStaging) |
| 76 | if again.ID != w.id || again.StagedAt <= v.StagedAt { |
| 77 | t.Fatalf("a second stage answered move %s staged at %d, want %s after %d", again.ID, again.StagedAt, w.id, v.StagedAt) |
| 78 | } |
| 79 | if serialBytes(t, w.fx.rel) == serial { |
| 80 | t.Fatal("a second stage did not bump the serial") |
| 81 | } |
| 82 | if after := w.record(); after.Move.State != provision.OfflineStaging || !after.Move.RequestedAt.Equal(rec.Move.RequestedAt) || after.Move.ID != w.id { |
| 83 | t.Fatalf("a second stage changed the move: %+v", after.Move) |
| 84 | } |
| 85 | v = again |
| 86 | |
| 87 | // Another region: refused for stage and for start, nothing written. |
| 88 | w.refused(http.MethodPost, "/stage", `{"region":"us"}`, http.StatusConflict, msgO179("uk")) |
| 89 | w.refused(http.MethodPost, "", `{"region":"us"}`, http.StatusConflict, msgO179("uk")) |
| 90 | // The other verbs wait for start. |
| 91 | for _, verb := range []string{"/seal", "/flip", "/open", "/finish"} { |
| 92 | w.refused(http.MethodPost, verb, "", http.StatusConflict, msgO25(provision.OfflineStaging)) |
| 93 | } |
| 94 | |
| 95 | // The status shows the stager's progress, and which request it answered. |
| 96 | w.staged("ie-box", func(r *moves.OfflineReport) { r.StageRequest = time.Unix(0, v.StagedAt).UTC() }) |
| 97 | st := w.status() |
| 98 | if st.Stage == nil || st.Stage.Box != "ie-box" || st.Stage.Phase != moves.OfflinePhaseStaged || st.Stage.StagedObjects != 300 || |
| 99 | st.Stage.ReplicaObjects != 305 || st.Stage.StagedBytes != 1_570_000_000 || st.Stage.RoundBytes != 9_000_000 || st.Stage.At == 0 || |
| 100 | st.Stage.Request != v.StagedAt { |
| 101 | t.Fatalf("stage view %+v (request %d)", st.Stage, v.StagedAt) |
| 102 | } |
| 103 | |
| 104 | // Start: staging to stopping, the same move and lock, offline from now. |
| 105 | started := w.ok("", `{"region":"uk"}`, provision.OfflineStopping) |
| 106 | if started.ID != w.id { |
| 107 | t.Fatalf("start answered move %s, want the staged %s", started.ID, w.id) |
| 108 | } |
| 109 | rec = w.record() |
| 110 | if !rec.Move.RequestedAt.After(*rec.Move.StagedAt) || rec.Move.RequestedBy == "" || started.NeverOffline { |
| 111 | t.Fatalf("started record %+v (never offline %v)", rec.Move, started.NeverOffline) |
| 112 | } |
| 113 | if l, _ := w.lock(); l.Op != w.id { |
| 114 | t.Fatalf("lock %+v after start", l) |
| 115 | } |
| 116 | if st := w.status(); st.Stage == nil || st.Stage.StagedObjects != 300 { |
| 117 | t.Fatalf("the stage's progress is gone from the status after start: %+v", st.Stage) |
| 118 | } |
| 119 | // From here it is the ordinary move: stage and start cannot be posted |
| 120 | // again, and seal follows the stop. |
| 121 | w.refused(http.MethodPost, "/stage", `{"region":"uk"}`, http.StatusConflict, "a move to uk is already "+provision.OfflineStopping) |
| 122 | w.stopped(nil, "ie-box") |
| 123 | w.stagerStopped("ie-box", time.Now()) |
| 124 | w.ok("/seal", "", provision.OfflineSealing) |
| 125 | } |
| 126 | |
| 127 | // The stage door's own rule: every box in both regions can stage. A box |
| 128 | // that does not publish offline-stage-1 would leave a staging record to |
| 129 | // nobody, so the door refuses before anything is written. The start door |
| 130 | // does not ask it. |
| 131 | func TestOfflineStageRefusesABoxThatCannotStage(t *testing.T) { |
| 132 | for _, c := range []struct { |
| 133 | name string |
| 134 | boxes []offlineBox |
| 135 | box string |
| 136 | }{ |
| 137 | {"source", []offlineBox{{Box: "ie-box", Placement: "ie", Bucket: "data-ie", NoStageCap: true}, ukOff("uk-box")}, "ie-box"}, |
| 138 | {"target", []offlineBox{ieOff("ie-box"), {Box: "uk-box", Placement: "uk", Bucket: "data-uk", NoStageCap: true}}, "uk-box"}, |
| 139 | } { |
| 140 | t.Run(c.name, func(t *testing.T) { |
| 141 | w := newOfflineWorld(t, c.boxes...) |
| 142 | w.refused(http.MethodPost, "/stage", `{"region":"uk"}`, http.StatusConflict, msgO180(c.box)) |
| 143 | if rec := w.record(); rec.Move != nil { |
| 144 | t.Fatalf("a refused stage wrote %+v", rec.Move) |
| 145 | } |
| 146 | if _, ok := w.lock(); ok { |
| 147 | t.Fatal("a refused stage took the lock") |
| 148 | } |
| 149 | w.start() |
| 150 | }) |
| 151 | } |
| 152 | // The start door's refusals hold for stage too (one of them here). |
| 153 | w := newOfflineWorld(t, ieOff("ie-box"), offlineBox{Box: "uk-box", Placement: "uk", Bucket: "data-uk", NoCap: true}) |
| 154 | w.refused(http.MethodPost, "/stage", `{"region":"uk"}`, http.StatusConflict, msgO21("uk-box")) |
| 155 | } |
| 156 | |
| 157 | // Abort from staging: reverting, the region untouched; finish waits for |
| 158 | // every source box's reverted (the stager's comes after it has deleted the |
| 159 | // staged copy), then ends a move that was never offline and frees the lock. |
| 160 | func TestOfflineAbortFromStagingDoor(t *testing.T) { |
| 161 | w := newOfflineWorld(t, oneEach...) |
| 162 | w.stage() |
| 163 | w.staged("ie-box", nil) |
| 164 | if v := w.ok("/abort", "", provision.OfflineReverting); !v.NeverOffline { |
| 165 | t.Fatal("an abort from staging does not say the instance was never offline") |
| 166 | } |
| 167 | if rec := w.record(); rec.Region != "ie" || rec.Move.State != provision.OfflineReverting { |
| 168 | t.Fatalf("after abort: %q %+v", rec.Region, rec.Move) |
| 169 | } |
| 170 | w.refused(http.MethodPost, "/finish", "", http.StatusConflict, msgO25(provision.OfflineReverting)) |
| 171 | w.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseReverted, nil) |
| 172 | // Every source box has reverted, but the pinned stager has not said |
| 173 | // the staged copy is gone (Astra round 1, findings 1 and 2): no finish, |
| 174 | // whatever its last round reported, or another box's stager report. |
| 175 | w.refused(http.MethodPost, "/finish", "", http.StatusConflict, msgO25(provision.OfflineReverting)) |
| 176 | w.report(moves.OfflineRoleStager, "ie-other", moves.OfflinePhaseReverted, nil) |
| 177 | w.refused(http.MethodPost, "/finish", "", http.StatusConflict, msgO25(provision.OfflineReverting)) |
| 178 | w.report(moves.OfflineRoleStager, "ie-box", moves.OfflinePhaseReverted, nil) |
| 179 | v := w.ok("/finish", "", "none") |
| 180 | if v.Last == nil || v.Last.Outcome != provision.OfflineOutcomeAborted || v.Last.OfflineS != 0 { |
| 181 | t.Fatalf("last = %+v", v.Last) |
| 182 | } |
| 183 | if _, ok := w.lock(); ok { |
| 184 | t.Fatal("the lock outlived the aborted stage") |
| 185 | } |
| 186 | // The instance can be staged again. |
| 187 | w.stage() |
| 188 | } |
| 189 | |
| 190 | // A failed round shows in the status with the box's own diagnostic. |
| 191 | func TestOfflineStageViewShowsAFailedRound(t *testing.T) { |
| 192 | w := newOfflineWorld(t, oneEach...) |
| 193 | w.stage() |
| 194 | w.staged("ie-box", func(r *moves.OfflineReport) { |
| 195 | r.Phase, r.StageError, r.At = moves.OfflinePhaseFailed, "mirror: copy instances/x: boom", time.Now().UTC() |
| 196 | }) |
| 197 | st := w.status() |
| 198 | if st.Stage == nil || st.Stage.Phase != moves.OfflinePhaseFailed || st.Stage.Error != "mirror: copy instances/x: boom" || st.Stage.StagedObjects != 300 { |
| 199 | t.Fatalf("stage view %+v", st.Stage) |
| 200 | } |
| 201 | } |
| 202 | |
| 203 | // The stage door pins the stager: the lexically first eligible source box, |
| 204 | // whatever order the fleet lists them in. |
| 205 | func TestOfflineStagePinsTheStager(t *testing.T) { |
| 206 | w := newOfflineWorld(t, ieOff("ie-c"), ieOff("ie-a"), ukOff("uk-box")) |
| 207 | v := w.stage() |
| 208 | if rec := w.record(); rec.Move.Stager != "ie-a" || v.Stager != "ie-a" { |
| 209 | t.Fatalf("stager %q (view %q), want ie-a", rec.Move.Stager, v.Stager) |
| 210 | } |
| 211 | } |
| 212 | |
| 213 | // ---- Astra round 2 ---- |
| 214 | |
| 215 | func msgO190(box string) string { |
| 216 | return box + ", which staged " + offlineHost + ", no longer publishes an edges file in ie. bring it back or abort the move" |
| 217 | } |
| 218 | |
| 219 | const ( |
| 220 | msgO191 = "uk publishes the same data bucket as ie. a staged copy there would replace the serving one" |
| 221 | msgO192 = "uk now publishes a different data bucket or region from what was recorded when " + offlineHost + " was staged. abort the move and stage it again" |
| 222 | ) |
| 223 | |
| 224 | // unpublish removes a box from the fleet's evidence, its edges file and |
| 225 | // its status report: the box left the fleet (it may still be running). |
| 226 | func (w *offlineWorld) unpublish(box string) { |
| 227 | w.t.Helper() |
| 228 | for _, key := range []string{edgesFilePrefix + box + ".json", "status/boxes/" + box + ".json"} { |
| 229 | if err := w.fx.rel.Delete(context.Background(), key); err != nil { |
| 230 | w.t.Fatal(err) |
| 231 | } |
| 232 | } |
| 233 | } |
| 234 | |
| 235 | // stagerStopped writes the pinned stager's stop acknowledgement. |
| 236 | func (w *offlineWorld) stagerStopped(box string, at time.Time) { |
| 237 | w.t.Helper() |
| 238 | w.report(moves.OfflineRoleStager, box, moves.OfflinePhaseStopped, func(r *moves.OfflineReport) { |
| 239 | r.Held, r.Fenced, r.Quiesced, r.At = false, false, false, at |
| 240 | }) |
| 241 | } |
| 242 | |
| 243 | // Finding 1: seal waits for the pinned stager's fresh acknowledgement that |
| 244 | // no round is in flight and none will start, beside every box's stop, and |
| 245 | // names the stager among the boxes it captures. |
| 246 | func TestOfflineSealWaitsForThePinnedStager(t *testing.T) { |
| 247 | w := newOfflineWorld(t, ieOff("ie-a"), ieOff("ie-b"), ukOff("uk-box")) |
| 248 | w.stage() |
| 249 | w.ok("", `{"region":"uk"}`, provision.OfflineStopping) |
| 250 | w.stopped(nil, "ie-a", "ie-b") |
| 251 | w.refused(http.MethodPost, "/seal", "", http.StatusConflict, "Waiting for ie-a in ie to confirm it has stopped "+offlineHost+". "+offlineHost+" is offline.") |
| 252 | w.stagerStopped("ie-a", time.Now().Add(-10*time.Minute)) |
| 253 | w.refused(http.MethodPost, "/seal", "", http.StatusConflict, msgO178("ie-a")) |
| 254 | w.stagerStopped("ie-a", time.Now()) |
| 255 | w.ok("/seal", "", provision.OfflineSealing) |
| 256 | if rec := w.record(); !slices.Contains(rec.Move.Boxes, "ie-a") { |
| 257 | t.Fatalf("the seal's boxes %v leave out the stager", rec.Move.Boxes) |
| 258 | } |
| 259 | } |
| 260 | |
| 261 | // Finding 1: a pinned stager that no longer publishes an edges file is never |
| 262 | // taken to have stopped. start and seal refuse with MV-O190. |
| 263 | func TestOfflineStagedMoveRefusesAMissingStager(t *testing.T) { |
| 264 | w := newOfflineWorld(t, ieOff("ie-a"), ieOff("ie-b"), ukOff("uk-box")) |
| 265 | w.stage() |
| 266 | w.unpublish("ie-a") |
| 267 | w.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusConflict, msgO190("ie-a")) |
| 268 | if st := w.status(); st.Next != "" || st.Waiting != msgO190("ie-a") { |
| 269 | t.Fatalf("status next %q waiting %q", st.Next, st.Waiting) |
| 270 | } |
| 271 | |
| 272 | w2 := newOfflineWorld(t, ieOff("ie-a"), ieOff("ie-b"), ukOff("uk-box")) |
| 273 | w2.stage() |
| 274 | w2.ok("", `{"region":"uk"}`, provision.OfflineStopping) |
| 275 | w2.stopped(nil, "ie-a", "ie-b") |
| 276 | w2.stagerStopped("ie-a", time.Now()) |
| 277 | w2.unpublish("ie-a") |
| 278 | w2.refused(http.MethodPost, "/seal", "", http.StatusConflict, msgO190("ie-a")) |
| 279 | } |
| 280 | |
| 281 | // Finding 2: the stage fixes its destination on the record, and refuses a |
| 282 | // target that publishes a source box's own bucket. |
| 283 | func TestOfflineStageFixesItsDestination(t *testing.T) { |
| 284 | w := newOfflineWorld(t, oneEach...) |
| 285 | w.stage() |
| 286 | if m := w.record().Move; m.StageBucket != "data-uk" || m.StageRegion != "eu-west-2" { |
| 287 | t.Fatalf("staging destination %q %q", m.StageBucket, m.StageRegion) |
| 288 | } |
| 289 | w2 := newOfflineWorld(t, ieOff("ie-box"), offlineBox{Box: "uk-box", Placement: "uk", Bucket: "data-ie"}) |
| 290 | w2.refused(http.MethodPost, "/stage", `{"region":"uk"}`, http.StatusConflict, msgO191) |
| 291 | } |
| 292 | |
| 293 | // Finding 2: once staged, a target that publishes other coordinates is |
| 294 | // refused by stage, start and seal, never followed. |
| 295 | func TestOfflineStagedMoveRefusesChangedCoordinates(t *testing.T) { |
| 296 | for _, c := range []struct { |
| 297 | name string |
| 298 | target offlineBox |
| 299 | }{ |
| 300 | {"another bucket", offlineBox{Box: "uk-box", Placement: "uk", Bucket: "data-uk-new"}}, |
| 301 | {"another box", offlineBox{Box: "uk-new", Placement: "uk", Bucket: "data-uk-new"}}, |
| 302 | } { |
| 303 | t.Run(c.name, func(t *testing.T) { |
| 304 | w := newOfflineWorld(t, oneEach...) |
| 305 | w.stage() |
| 306 | if c.target.Box != "uk-box" { |
| 307 | w.unpublish("uk-box") |
| 308 | } |
| 309 | w.fleet(c.target) |
| 310 | w.refused(http.MethodPost, "/stage", `{"region":"uk"}`, http.StatusConflict, msgO192) |
| 311 | w.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusConflict, msgO192) |
| 312 | |
| 313 | w2 := newOfflineWorld(t, oneEach...) |
| 314 | w2.stage() |
| 315 | w2.ok("", `{"region":"uk"}`, provision.OfflineStopping) |
| 316 | w2.stopped(nil, "ie-box") |
| 317 | w2.stagerStopped("ie-box", time.Now()) |
| 318 | if c.target.Box != "uk-box" { |
| 319 | w2.unpublish("uk-box") |
| 320 | } |
| 321 | w2.fleet(c.target) |
| 322 | w2.refused(http.MethodPost, "/seal", "", http.StatusConflict, msgO192) |
| 323 | }) |
| 324 | } |
| 325 | } |
| 326 | |
amadan