| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "encoding/json" |
| 7 | "errors" |
| 8 | "flag" |
| 9 | |
| 10 | "github.com/aws/aws-sdk-go-v2/aws" |
| 11 | awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" |
| 12 | "github.com/aws/aws-sdk-go-v2/service/ec2" |
| 13 | ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" |
| 14 | "github.com/carlosframework/platform/internal/awsinv" |
| 15 | "github.com/carlosframework/platform/internal/hostclaim" |
| 16 | "github.com/carlosframework/platform/internal/hostfence" |
| 17 | "github.com/carlosframework/platform/internal/hostresolve" |
| 18 | "github.com/carlosframework/platform/internal/provision" |
| 19 | "io" |
| 20 | "net/http" |
| 21 | "net/http/httptest" |
| 22 | "os" |
| 23 | "strings" |
| 24 | "sync" |
| 25 | "testing" |
| 26 | "time" |
| 27 | ) |
| 28 | |
| 29 | // The fake console below answers the move doors in Task 2's shapes: the flat |
| 30 | // view (state, target, updated_at, id, last, protocol) and, for detail=1, |
| 31 | // the same plus the detail object. Each flat GET returns the next scripted |
| 32 | // frame; a detail GET returns the frame the last flat GET returned. |
| 33 | |
| 34 | const ( |
| 35 | tMoveID = "mv_abcdefghijklmnopqrstuvwxyz" |
| 36 | tHost = "acme.go.tito.io" |
| 37 | tMovePth = "/api/cli/apps/bab/titogo/instances/acme.go.tito.io/move" |
| 38 | ) |
| 39 | |
| 40 | type frame map[string]any |
| 41 | |
| 42 | func flatOf(f frame) frame { |
| 43 | out := frame{} |
| 44 | for k, v := range f { |
| 45 | if k != "detail" { |
| 46 | out[k] = v |
| 47 | } |
| 48 | } |
| 49 | return out |
| 50 | } |
| 51 | |
| 52 | type fakeMoveConsole struct { |
| 53 | mu sync.Mutex |
| 54 | frames []frame |
| 55 | i int |
| 56 | last frame |
| 57 | tooMany int // the next flat GETs answered 429 |
| 58 | flatGets int |
| 59 | details int |
| 60 | posts []string // method path body |
| 61 | mutated bool // a mutation since the last flat GET |
| 62 | answer func(method, path, body string) (int, any) |
| 63 | // startDetail, when set, is the detail answer right after a mutation |
| 64 | // (the move's own start), whatever the next scripted frame is. |
| 65 | startDetail frame |
| 66 | // detailStatus, when set, answers every detail GET with that status. |
| 67 | detailStatus int |
| 68 | } |
| 69 | |
| 70 | func (c *fakeMoveConsole) ServeHTTP(w http.ResponseWriter, r *http.Request) { |
| 71 | c.mu.Lock() |
| 72 | defer c.mu.Unlock() |
| 73 | path := r.URL.Path |
| 74 | if !strings.HasPrefix(path, tMovePth) { |
| 75 | http.NotFound(w, r) |
| 76 | return |
| 77 | } |
| 78 | write := func(code int, v any) { |
| 79 | w.Header().Set("Content-Type", "application/json") |
| 80 | w.WriteHeader(code) |
| 81 | _ = json.NewEncoder(w).Encode(v) |
| 82 | } |
| 83 | if r.Method == http.MethodGet && path == tMovePth { |
| 84 | if r.URL.Query().Get("detail") == "1" { |
| 85 | c.details++ |
| 86 | if c.detailStatus != 0 { |
| 87 | write(c.detailStatus, map[string]string{"error": "detail refused by the test"}) |
| 88 | return |
| 89 | } |
| 90 | cur := c.last |
| 91 | if c.mutated && c.startDetail != nil { |
| 92 | cur = c.startDetail |
| 93 | } else if cur == nil || c.mutated { |
| 94 | // Right after a mutation the console shows its result: the |
| 95 | // next scripted frame. |
| 96 | n := c.i |
| 97 | if n >= len(c.frames) { |
| 98 | n = len(c.frames) - 1 |
| 99 | } |
| 100 | cur = c.frames[n] |
| 101 | } |
| 102 | write(http.StatusOK, cur) |
| 103 | return |
| 104 | } |
| 105 | c.flatGets++ |
| 106 | if c.tooMany > 0 { |
| 107 | c.tooMany-- |
| 108 | write(http.StatusTooManyRequests, map[string]string{"error": "too many instance requests. wait a minute and try again"}) |
| 109 | return |
| 110 | } |
| 111 | n := c.i |
| 112 | if n >= len(c.frames) { |
| 113 | n = len(c.frames) - 1 |
| 114 | } |
| 115 | c.last = c.frames[n] |
| 116 | c.mutated = false |
| 117 | c.i++ |
| 118 | write(http.StatusOK, flatOf(c.last)) |
| 119 | return |
| 120 | } |
| 121 | body, _ := io.ReadAll(r.Body) |
| 122 | c.posts = append(c.posts, r.Method+" "+path+" "+strings.TrimSpace(string(body))) |
| 123 | c.mutated = true |
| 124 | if c.answer == nil { |
| 125 | write(http.StatusAccepted, frame{"state": "preparing", "target": "uk", "id": tMoveID, "protocol": "phase1"}) |
| 126 | return |
| 127 | } |
| 128 | code, v := c.answer(r.Method, path, string(body)) |
| 129 | write(code, v) |
| 130 | } |
| 131 | |
| 132 | func (c *fakeMoveConsole) mutations() []string { |
| 133 | c.mu.Lock() |
| 134 | defer c.mu.Unlock() |
| 135 | return append([]string(nil), c.posts...) |
| 136 | } |
| 137 | |
| 138 | // runMoveAgainst runs the verb against the fake, as an operator who is not |
| 139 | // root, polling every millisecond. It returns the exit code and the output. |
| 140 | func runMoveAgainst(t *testing.T, c *fakeMoveConsole, args ...string) (int, string) { |
| 141 | t.Helper() |
| 142 | return runMoveWith(t, c, nil, args...) |
| 143 | } |
| 144 | |
| 145 | // runMoveWith is runMoveAgainst with deps adjusted first (stdin, the EC2 |
| 146 | // reader). |
| 147 | func runMoveWith(t *testing.T, c *fakeMoveConsole, adjust func(*moveDeps), args ...string) (int, string) { |
| 148 | t.Helper() |
| 149 | ts := httptest.NewServer(c) |
| 150 | t.Cleanup(ts.Close) |
| 151 | var out bytes.Buffer |
| 152 | deps := moveDeps{ |
| 153 | session: func(string) (*apiSession, error) { |
| 154 | return &apiSession{c: newConsoleClient(ts.URL, "cli-token"), console: ts.URL, address: "op@keymail.dev"}, nil |
| 155 | }, |
| 156 | euid: func() int { return 1000 }, |
| 157 | poll: time.Millisecond, |
| 158 | stdin: strings.NewReader(""), |
| 159 | ec2: func(context.Context, string) (awsinv.ExactEC2API, error) { |
| 160 | t.Error("the EC2 reader was asked for") |
| 161 | return nil, errors.New("no reader in this test") |
| 162 | }, |
| 163 | } |
| 164 | if adjust != nil { |
| 165 | adjust(&deps) |
| 166 | } |
| 167 | args = append(args, "--account", "bab", "--app", "titogo") |
| 168 | ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) |
| 169 | defer cancel() |
| 170 | code := moveMain(ctx, args, &out, deps) |
| 171 | return code, out.String() |
| 172 | } |
| 173 | |
| 174 | // The frames a move goes through, as the console shows them. |
| 175 | func none(last frame) frame { |
| 176 | f := frame{"state": "none", "protocol": "phase1"} |
| 177 | if last != nil { |
| 178 | f["last"] = last |
| 179 | } |
| 180 | f["detail"] = frame{"region": "ie", "log": []string{}, "evidence": frame{}} |
| 181 | return f |
| 182 | } |
| 183 | |
| 184 | func inFlight(state string, move frame, extra frame) frame { |
| 185 | m := frame{"id": tMoveID, "from": "ie", "to": "uk", "cut": "auto", "state": state} |
| 186 | for k, v := range move { |
| 187 | m[k] = v |
| 188 | } |
| 189 | d := frame{"move": m, "from": "ie", "region": "ie", "log": []string{}, "evidence": frame{}, |
| 190 | "claim": frame{"kind": "moving", "exec": "ie-box", "target": "uk-box"}} |
| 191 | for k, v := range extra { |
| 192 | d[k] = v |
| 193 | } |
| 194 | f := frame{"state": state, "target": "uk", "updated_at": time.Now().Unix(), "id": tMoveID, "protocol": "phase1", "detail": d} |
| 195 | if p := flatPhase(state, m); p != "" { |
| 196 | f["phase"] = p |
| 197 | } |
| 198 | return f |
| 199 | } |
| 200 | |
| 201 | // flatPhase is the flat view's phase, as the console derives it from the |
| 202 | // journal (moveViewOf). |
| 203 | func flatPhase(state string, m frame) string { |
| 204 | str := func(k string) string { v, _ := m[k].(string); return v } |
| 205 | switch state { |
| 206 | case "cutting": |
| 207 | return str("cut_phase") |
| 208 | case "landing": |
| 209 | return str("landing_phase") |
| 210 | case "rolling_back": |
| 211 | if rb, ok := m["rollback"].(frame); ok { |
| 212 | tp, _ := rb["target_phase"].(string) |
| 213 | sp, _ := rb["source_phase"].(string) |
| 214 | return tp + "/" + sp |
| 215 | } |
| 216 | } |
| 217 | return "" |
| 218 | } |
| 219 | |
| 220 | func outcome(id, what string, offline float64) frame { |
| 221 | return frame{"id": id, "outcome": what, "at": time.Now().UTC().Format(time.RFC3339), "offline_s": offline} |
| 222 | } |
| 223 | |
| 224 | const ( |
| 225 | mc1 = "Moving acme.go.tito.io from ie to uk." |
| 226 | mc2 = "Copying data. acme.go.tito.io keeps serving while this runs." |
| 227 | mc3 = "Checking that uk can run acme.go.tito.io." |
| 228 | mc5 = "uk is ready. Starting the cut. acme.go.tito.io will be offline for about 45 seconds." |
| 229 | mc6 = "Stopped in ie. Starting in uk." |
| 230 | mc7 = "Moved. acme.go.tito.io is served from uk. It was offline for 41 seconds." |
| 231 | mc8 = "uk could not start acme.go.tito.io: the landing ran past its deadline. Moving it back to ie." |
| 232 | mc9 = "acme.go.tito.io is back in ie and serving. Nothing was lost." |
| 233 | mc11 = "acme.go.tito.io could not start in uk or in ie. It is offline. Run carlos move resolve --host acme.go.tito.io --retry once the cause is fixed." |
| 234 | mc12 = "Ready to cut. Run carlos move cut --host acme.go.tito.io when you are ready." |
| 235 | ) |
| 236 | |
| 237 | func wantLines(t *testing.T, out string, lines ...string) { |
| 238 | t.Helper() |
| 239 | for _, l := range lines { |
| 240 | if !strings.Contains(out, l+"\n") { |
| 241 | t.Errorf("output lacks %q:\n%s", l, out) |
| 242 | } |
| 243 | } |
| 244 | } |
| 245 | |
| 246 | func wantNoLine(t *testing.T, out string, lines ...string) { |
| 247 | t.Helper() |
| 248 | for _, l := range lines { |
| 249 | if strings.Contains(out, l) { |
| 250 | t.Errorf("output holds %q:\n%s", l, out) |
| 251 | } |
| 252 | } |
| 253 | } |
| 254 | |
| 255 | // Exit codes (preflight L24): 0 done, 3 rolled back or aborted, 4 stuck or |
| 256 | // waiting, 5 refused. Each sentence comes from the state it describes, and |
| 257 | // the outcome only from last, bound to this move's id. |
| 258 | func TestMoveWaitExitCodes(t *testing.T) { |
| 259 | rb := frame{"id": "rb_x", "reason": "the landing ran past its deadline", "target_phase": "cleaning"} |
| 260 | report := frame{"checks": frame{"t1_protocol": frame{"ok": true, "reason": ""}}} |
| 261 | t.Run("rolled back", func(t *testing.T) { |
| 262 | // The detail is read on each change of state or phase (L19): the |
| 263 | // target's report is there when the move is first seen preparing. |
| 264 | frames := []frame{none(nil), inFlight("preparing", nil, frame{"target_report": report})} |
| 265 | c := &fakeMoveConsole{frames: append(frames, |
| 266 | inFlight("ready", nil, nil), |
| 267 | inFlight("cutting", frame{"cut_phase": "admitted", "cut_budget_s": 45}, nil), |
| 268 | inFlight("landing", frame{"cut_phase": "flipped", "landing_phase": "probing"}, nil), |
| 269 | inFlight("rolling_back", frame{"rollback": rb}, nil), |
| 270 | none(outcome(tMoveID, "rolled_back", 0)), |
| 271 | )} |
| 272 | code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk") |
| 273 | if code != 3 { |
| 274 | t.Fatalf("exit %d, want 3:\n%s", code, out) |
| 275 | } |
| 276 | wantLines(t, out, mc1, mc2, mc3, mc5, mc6, mc8, mc9) |
| 277 | if m := c.mutations(); len(m) != 1 || m[0] != "POST "+tMovePth+` {"cut":"auto","region":"uk"}` { |
| 278 | t.Fatalf("mutations %q", m) |
| 279 | } |
| 280 | }) |
| 281 | t.Run("done", func(t *testing.T) { |
| 282 | // After the close the record's region is the target's: M-C7 names it. |
| 283 | done := none(outcome(tMoveID, "done", 41.2)) |
| 284 | done["detail"].(frame)["region"] = "uk" |
| 285 | c := &fakeMoveConsole{frames: []frame{ |
| 286 | none(nil), |
| 287 | inFlight("preparing", nil, nil), |
| 288 | inFlight("cutting", frame{"cut_phase": "stopping", "cut_budget_s": 45}, nil), |
| 289 | inFlight("committed", frame{"cut_phase": "flipped"}, nil), |
| 290 | done, |
| 291 | }} |
| 292 | code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk") |
| 293 | if code != 0 { |
| 294 | t.Fatalf("exit %d, want 0:\n%s", code, out) |
| 295 | } |
| 296 | wantLines(t, out, mc1, mc5, mc6, mc7) |
| 297 | }) |
| 298 | t.Run("stuck", func(t *testing.T) { |
| 299 | c := &fakeMoveConsole{frames: []frame{ |
| 300 | none(nil), |
| 301 | inFlight("stuck", frame{"stuck_reason": "the recovery probe ran out of attempts"}, nil), |
| 302 | }} |
| 303 | code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk") |
| 304 | if code != 4 { |
| 305 | t.Fatalf("exit %d, want 4:\n%s", code, out) |
| 306 | } |
| 307 | wantLines(t, out, mc11, "the recovery probe ran out of attempts") |
| 308 | }) |
| 309 | t.Run("rolling back past the release deadline", func(t *testing.T) { |
| 310 | past := time.Now().Add(-time.Minute).UTC().Format(time.RFC3339) |
| 311 | late := frame{"id": "rb_x", "reason": "the landing ran past its deadline", "target_phase": "releasing", "release_deadline": past} |
| 312 | c := &fakeMoveConsole{frames: []frame{none(nil), inFlight("rolling_back", frame{"rollback": late}, nil)}} |
| 313 | code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk") |
| 314 | if code != 4 { |
| 315 | t.Fatalf("exit %d, want 4:\n%s", code, out) |
| 316 | } |
| 317 | wantLines(t, out, mc8, "Waiting for uk-box in uk to confirm it has stopped acme.go.tito.io. acme.go.tito.io is offline.") |
| 318 | }) |
| 319 | t.Run("refused at the request", func(t *testing.T) { |
| 320 | c := &fakeMoveConsole{frames: []frame{none(nil)}, answer: func(string, string, string) (int, any) { |
| 321 | return http.StatusConflict, map[string]string{"error": "the instance is already in uk"} |
| 322 | }} |
| 323 | code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk") |
| 324 | if code != 5 { |
| 325 | t.Fatalf("exit %d, want 5:\n%s", code, out) |
| 326 | } |
| 327 | wantLines(t, out, "the instance is already in uk") |
| 328 | }) |
| 329 | t.Run("abort", func(t *testing.T) { |
| 330 | c := &fakeMoveConsole{frames: []frame{ |
| 331 | inFlight("preparing", nil, nil), |
| 332 | inFlight("aborting", nil, nil), |
| 333 | none(outcome(tMoveID, "aborted", 0)), |
| 334 | }, answer: func(string, string, string) (int, any) { |
| 335 | return http.StatusAccepted, frame{"state": "aborting", "target": "uk", "id": tMoveID, "protocol": "phase1"} |
| 336 | }} |
| 337 | code, out := runMoveAgainst(t, c, "abort", "--host", tHost) |
| 338 | if code != 3 { |
| 339 | t.Fatalf("exit %d, want 3:\n%s", code, out) |
| 340 | } |
| 341 | wantLines(t, out, "The move was aborted. acme.go.tito.io stays in ie.") |
| 342 | if m := c.mutations(); len(m) != 1 || !strings.HasPrefix(m[0], "DELETE "+tMovePth+" ") { |
| 343 | t.Fatalf("mutations %q", m) |
| 344 | } |
| 345 | }) |
| 346 | t.Run("abort after the cut started", func(t *testing.T) { |
| 347 | const mc13 = "This move has started its cut. It will finish or move back on its own." |
| 348 | c := &fakeMoveConsole{frames: []frame{inFlight("landing", nil, nil)}, answer: func(string, string, string) (int, any) { |
| 349 | return http.StatusConflict, map[string]string{"error": mc13} |
| 350 | }} |
| 351 | code, out := runMoveAgainst(t, c, "abort", "--host", tHost) |
| 352 | if code != 5 { |
| 353 | t.Fatalf("exit %d, want 5:\n%s", code, out) |
| 354 | } |
| 355 | wantLines(t, out, mc13) |
| 356 | }) |
| 357 | t.Run("cut and resolve post to their doors", func(t *testing.T) { |
| 358 | for _, v := range []struct { |
| 359 | args []string |
| 360 | want string |
| 361 | }{ |
| 362 | {[]string{"cut", "--host", tHost, "--no-wait"}, "POST " + tMovePth + "/cut {}"}, |
| 363 | {[]string{"resolve", "--host", tHost, "--retry", "--no-wait"}, "POST " + tMovePth + `/resolve {"retry":true}`}, |
| 364 | } { |
| 365 | c := &fakeMoveConsole{frames: []frame{inFlight("ready", nil, nil)}} |
| 366 | code, out := runMoveAgainst(t, c, v.args...) |
| 367 | if code != 0 { |
| 368 | t.Fatalf("%v: exit %d:\n%s", v.args, code, out) |
| 369 | } |
| 370 | if m := c.mutations(); len(m) != 1 || m[0] != v.want { |
| 371 | t.Fatalf("%v: mutations %q, want %q", v.args, m, v.want) |
| 372 | } |
| 373 | } |
| 374 | }) |
| 375 | } |
| 376 | |
| 377 | // Review Focus 1: the outcome is never inferred from the last state seen. A |
| 378 | // wait that saw only preparing still reads rolled_back from last, bound to |
| 379 | // this move's id; a last of another move is "not recorded". |
| 380 | func TestMoveWaitSurvivesMissedStates(t *testing.T) { |
| 381 | c := &fakeMoveConsole{frames: []frame{none(nil), inFlight("preparing", nil, nil), none(outcome(tMoveID, "rolled_back", 0))}} |
| 382 | code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk") |
| 383 | if code != 3 { |
| 384 | t.Fatalf("exit %d, want 3:\n%s", code, out) |
| 385 | } |
| 386 | wantLines(t, out, mc9) |
| 387 | // The rollback's reason was never seen, so M-C8 (a present-tense line |
| 388 | // with a reason) is not invented after the fact. |
| 389 | wantNoLine(t, out, "could not start") |
| 390 | |
| 391 | for _, last := range []frame{outcome("mv_another", "done", 3), nil} { |
| 392 | c := &fakeMoveConsole{frames: []frame{none(nil), inFlight("preparing", nil, nil), none(last)}} |
| 393 | code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk") |
| 394 | if code != 4 { |
| 395 | t.Fatalf("last %v: exit %d, want 4:\n%s", last, code, out) |
| 396 | } |
| 397 | wantLines(t, out, "This move's outcome is not recorded.") |
| 398 | wantNoLine(t, out, "Moved.", "back in") |
| 399 | } |
| 400 | // Another move in flight, and ours closed: ours is read from last. |
| 401 | other := inFlight("preparing", nil, nil) |
| 402 | other["id"] = "mv_another" |
| 403 | other["last"] = outcome(tMoveID, "done", 41.2) |
| 404 | other["detail"].(frame)["region"] = "uk" |
| 405 | c = &fakeMoveConsole{frames: []frame{none(nil), other}} |
| 406 | code, out = runMoveAgainst(t, c, "--host", tHost, "--to", "uk") |
| 407 | if code != 0 { |
| 408 | t.Fatalf("exit %d, want 0:\n%s", code, out) |
| 409 | } |
| 410 | wantLines(t, out, mc7) |
| 411 | } |
| 412 | |
| 413 | func TestMoveManualCutPrintsM12AndStops(t *testing.T) { |
| 414 | c := &fakeMoveConsole{frames: []frame{none(nil), inFlight("preparing", frame{"cut": "manual"}, nil), inFlight("ready", frame{"cut": "manual"}, nil)}} |
| 415 | code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk", "--manual-cut") |
| 416 | if code != 4 { |
| 417 | t.Fatalf("exit %d, want 4:\n%s", code, out) |
| 418 | } |
| 419 | wantLines(t, out, mc12) |
| 420 | if m := c.mutations(); len(m) != 1 || m[0] != "POST "+tMovePth+` {"cut":"manual","region":"uk"}` { |
| 421 | t.Fatalf("mutations %q: want the request and no cut", m) |
| 422 | } |
| 423 | } |
| 424 | |
| 425 | // L4: a console whose answer lacks the Phase 1 marker gets no mutation. |
| 426 | func TestMoveRefusesAConsoleWithoutPhase1(t *testing.T) { |
| 427 | legacy := frame{"state": "none"} |
| 428 | for _, args := range [][]string{ |
| 429 | {"--host", tHost, "--to", "uk"}, {"cut", "--host", tHost}, {"abort", "--host", tHost}, |
| 430 | {"resolve", "--host", tHost, "--retry"}, {"status", "--host", tHost}, |
| 431 | } { |
| 432 | c := &fakeMoveConsole{frames: []frame{legacy}} |
| 433 | code, out := runMoveAgainst(t, c, args...) |
| 434 | if code != 5 { |
| 435 | t.Fatalf("%v: exit %d, want 5:\n%s", args, code, out) |
| 436 | } |
| 437 | wantLines(t, out, "Moves are not enabled on this console.") |
| 438 | if m := c.mutations(); len(m) != 0 { |
| 439 | t.Fatalf("%v: mutations %q, want none", args, m) |
| 440 | } |
| 441 | } |
| 442 | } |
| 443 | |
| 444 | // L18: relocating with waiting_for_box prints M-C22 and exits 4, never |
| 445 | // waiting for the box. |
| 446 | func TestMoveWaitReportsWaitingForABox(t *testing.T) { |
| 447 | c := &fakeMoveConsole{frames: []frame{none(nil), inFlight("relocating", nil, frame{"waiting_for_box": "ie"})}} |
| 448 | code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk") |
| 449 | if code != 4 { |
| 450 | t.Fatalf("exit %d, want 4:\n%s", code, out) |
| 451 | } |
| 452 | wantLines(t, out, "acme.go.tito.io is waiting for a box in ie. Its last box was terminated before the move finished. Its data up to the last completed upload is kept in ie. It will start when a box joins the region, and any writes after that upload will be reported then.") |
| 453 | c = &fakeMoveConsole{frames: []frame{none(nil), inFlight("relocating", nil, nil)}} |
| 454 | code, out = runMoveAgainst(t, c, "--host", tHost, "--to", "uk") |
| 455 | if code != 4 { |
| 456 | t.Fatalf("relocating with a box: exit %d, want 4:\n%s", code, out) |
| 457 | } |
| 458 | wantLines(t, out, "acme.go.tito.io is relocating after its box was terminated. This move will not finish.") |
| 459 | } |
| 460 | |
| 461 | // L19: the wait polls the flat view and reads detail only when the state |
| 462 | // changes (or now and then). A 429 backs off, says so once, and never ends |
| 463 | // the wait. |
| 464 | func TestMoveWaitBacksOffAndReadsDetailOnChange(t *testing.T) { |
| 465 | frames := []frame{none(nil)} |
| 466 | for i := 0; i < 10; i++ { |
| 467 | frames = append(frames, inFlight("preparing", nil, nil)) |
| 468 | } |
| 469 | done := none(outcome(tMoveID, "done", 41.2)) |
| 470 | done["detail"].(frame)["region"] = "uk" |
| 471 | frames = append(frames, done) |
| 472 | c := &fakeMoveConsole{frames: frames, tooMany: 2} |
| 473 | code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk") |
| 474 | if code != 0 { |
| 475 | t.Fatalf("exit %d, want 0:\n%s", code, out) |
| 476 | } |
| 477 | if n := strings.Count(out, "The console is limiting requests. Checking every 10 seconds."); n != 1 { |
| 478 | t.Fatalf("the back-off line appears %d times, want once:\n%s", n, out) |
| 479 | } |
| 480 | if c.details > 4 { |
| 481 | t.Fatalf("%d detail reads over %d flat polls; detail is read on a change", c.details, c.flatGets) |
| 482 | } |
| 483 | } |
| 484 | |
| 485 | // L25: carlos move refuses to run as root, before any request. |
| 486 | func TestMoveRefusesRoot(t *testing.T) { |
| 487 | c := &fakeMoveConsole{frames: []frame{none(nil)}} |
| 488 | ts := httptest.NewServer(c) |
| 489 | defer ts.Close() |
| 490 | var out bytes.Buffer |
| 491 | called := false |
| 492 | deps := moveDeps{session: func(string) (*apiSession, error) { called = true; return nil, nil }, euid: func() int { return 0 }, poll: time.Millisecond} |
| 493 | if code := moveMain(context.Background(), []string{"--host", tHost, "--to", "uk", "--app", "titogo"}, &out, deps); code != 5 { |
| 494 | t.Fatalf("exit %d, want 5:\n%s", code, out.String()) |
| 495 | } |
| 496 | if called || c.flatGets != 0 { |
| 497 | t.Fatal("a refused root run reached the console") |
| 498 | } |
| 499 | wantLines(t, out.String(), "Run carlos move as yourself, not as root. It uses your own credentials.") |
| 500 | } |
| 501 | |
| 502 | // --no-wait answers from the mutation's own answer, and reads nothing after |
| 503 | // it (fix round 2, ruling 2): a detail read that fails or is rate limited |
| 504 | // cannot change the exit of a command the console accepted. |
| 505 | func TestMoveNoWait(t *testing.T) { |
| 506 | c := &fakeMoveConsole{frames: []frame{none(nil), inFlight("preparing", nil, nil)}} |
| 507 | code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk", "--no-wait") |
| 508 | if code != 0 { |
| 509 | t.Fatalf("exit %d:\n%s", code, out) |
| 510 | } |
| 511 | wantLines(t, out, "Move "+tMoveID+" of acme.go.tito.io is preparing.") |
| 512 | wantNoLine(t, out, mc2) |
| 513 | } |
| 514 | |
| 515 | func TestMoveNoWaitReadsNothingAfterAcceptance(t *testing.T) { |
| 516 | for _, status := range []int{http.StatusInternalServerError, http.StatusTooManyRequests} { |
| 517 | for _, args := range [][]string{ |
| 518 | {"--host", tHost, "--to", "uk", "--no-wait"}, {"cut", "--host", tHost, "--no-wait"}, |
| 519 | {"abort", "--host", tHost, "--no-wait"}, {"resolve", "--host", tHost, "--retry", "--no-wait"}, |
| 520 | } { |
| 521 | c := &fakeMoveConsole{frames: []frame{inFlight("ready", nil, nil)}, detailStatus: status} |
| 522 | if args[0] == "--host" { |
| 523 | c.frames = []frame{none(nil)} |
| 524 | } |
| 525 | code, out := runMoveAgainst(t, c, args...) |
| 526 | if code != 0 || c.details != 0 { |
| 527 | t.Fatalf("%v with detail answering %d: exit %d after %d detail reads, want 0 and none:\n%s", args, status, code, c.details, out) |
| 528 | } |
| 529 | } |
| 530 | } |
| 531 | } |
| 532 | |
| 533 | // status: one read, the state in a line, exit 0; --wait waits for this |
| 534 | // move's outcome; with nothing in flight and nothing closed, it says so. |
| 535 | func TestMoveStatus(t *testing.T) { |
| 536 | c := &fakeMoveConsole{frames: []frame{inFlight("preparing", nil, nil)}} |
| 537 | code, out := runMoveAgainst(t, c, "status", "--host", tHost) |
| 538 | if code != 0 { |
| 539 | t.Fatalf("exit %d:\n%s", code, out) |
| 540 | } |
| 541 | wantLines(t, out, "Move "+tMoveID+" of acme.go.tito.io is preparing.", mc2) |
| 542 | if m := c.mutations(); len(m) != 0 { |
| 543 | t.Fatalf("status mutated: %q", m) |
| 544 | } |
| 545 | c = &fakeMoveConsole{frames: []frame{none(nil)}} |
| 546 | if code, out := runMoveAgainst(t, c, "status", "--host", tHost, "--wait"); code != 0 || !strings.Contains(out, "No move is in flight for acme.go.tito.io.\n") { |
| 547 | t.Fatalf("status --wait with nothing = %d:\n%s", code, out) |
| 548 | } |
| 549 | // Resumed after a disconnect: the first read is none, and last is the |
| 550 | // move this operator was waiting on. |
| 551 | closed := none(outcome(tMoveID, "rolled_back", 0)) |
| 552 | c = &fakeMoveConsole{frames: []frame{closed}} |
| 553 | // The wait never saw the move, so it never learnt its regions, and the |
| 554 | // record's region now is not taken for them (fix round 2, ruling 3). |
| 555 | if code, out := runMoveAgainst(t, c, "status", "--host", tHost, "--wait"); code != 3 || !strings.Contains(out, "Move "+tMoveID+" of acme.go.tito.io is rolled_back.\n") || strings.Contains(out, "back in") { |
| 556 | t.Fatalf("status --wait after a rollback = %d:\n%s", code, out) |
| 557 | } |
| 558 | } |
| 559 | |
| 560 | func TestParseMoveArgs(t *testing.T) { |
| 561 | for _, c := range []struct { |
| 562 | args []string |
| 563 | ok bool |
| 564 | verb string |
| 565 | }{ |
| 566 | {[]string{"--host", tHost, "--to", "uk"}, true, "request"}, |
| 567 | {[]string{"--host", tHost, "--to", "uk", "--manual-cut", "--no-wait"}, true, "request"}, |
| 568 | {[]string{"status", "--host", tHost}, true, "status"}, |
| 569 | {[]string{"status", "--host", tHost, "--wait"}, true, "status"}, |
| 570 | {[]string{"cut", "--host", tHost}, true, "cut"}, |
| 571 | {[]string{"abort", "--host", tHost}, true, "abort"}, |
| 572 | {[]string{"resolve", "--host", tHost, "--retry"}, true, "resolve"}, |
| 573 | {[]string{"--to", "uk"}, false, ""}, |
| 574 | {[]string{"--host", tHost}, false, ""}, |
| 575 | {[]string{"resolve", "--host", tHost}, false, ""}, |
| 576 | {[]string{"cut", "--host", tHost, "--to", "uk"}, false, ""}, |
| 577 | {[]string{"resolve", "--host", tHost, "--declare-lost", "uk-box"}, true, "resolve"}, |
| 578 | {[]string{"resolve", "--host", tHost, "--declare-lost", "uk-box", "--retry"}, false, ""}, |
| 579 | } { |
| 580 | got, err := parseMoveArgs(c.args, io.Discard) |
| 581 | if (err == nil) != c.ok || (c.ok && got.verb != c.verb) { |
| 582 | t.Errorf("parseMoveArgs(%q) = %+v, %v; want ok %v verb %q", c.args, got, err, c.ok, c.verb) |
| 583 | } |
| 584 | } |
| 585 | } |
| 586 | |
| 587 | // L16 (Task 6 folded in): every fenced `carlos move` line in the CLI page's |
| 588 | // section parses through the CLI's own parser, the section names every verb, |
| 589 | // and it states the four exit codes. |
| 590 | func TestMoveDocsNameEveryVerb(t *testing.T) { |
| 591 | raw, err := os.ReadFile("docs/site/cli.md") |
| 592 | if err != nil { |
| 593 | t.Fatal(err) |
| 594 | } |
| 595 | page := string(raw) |
| 596 | start := strings.Index(page, "\n### carlos move\n") |
| 597 | if start < 0 { |
| 598 | t.Fatal("docs/site/cli.md has no ### carlos move section") |
| 599 | } |
| 600 | section := page[start+1:] |
| 601 | if end := strings.Index(section[4:], "\n## "); end >= 0 { |
| 602 | section = section[:end+4] |
| 603 | } |
| 604 | if end := strings.Index(section[4:], "\n### "); end >= 0 { |
| 605 | section = section[:end+4] |
| 606 | } |
| 607 | verbs := map[string]bool{} |
| 608 | inFence := false |
| 609 | for _, line := range strings.Split(section, "\n") { |
| 610 | if strings.HasPrefix(line, "```") { |
| 611 | inFence = !inFence |
| 612 | continue |
| 613 | } |
| 614 | fields := strings.Fields(line) |
| 615 | if !inFence || len(fields) < 2 || fields[0] != "carlos" || fields[1] != "move" { |
| 616 | continue |
| 617 | } |
| 618 | args := fields[2:] |
| 619 | for i, a := range args { |
| 620 | // [--flag] is an optional flag, and a placeholder stands for a |
| 621 | // value. |
| 622 | a = strings.TrimSuffix(strings.TrimPrefix(a, "["), "]") |
| 623 | if strings.HasPrefix(a, "<") { |
| 624 | a = "x" |
| 625 | } |
| 626 | args[i] = a |
| 627 | } |
| 628 | got, err := parseMoveArgs(args, io.Discard) |
| 629 | if err != nil { |
| 630 | t.Errorf("the page's example %q does not parse: %v", line, err) |
| 631 | continue |
| 632 | } |
| 633 | verbs[got.verb] = true |
| 634 | } |
| 635 | for _, v := range []string{"request", "status", "cut", "abort", "resolve"} { |
| 636 | if !verbs[v] { |
| 637 | t.Errorf("the section has no example of %s", v) |
| 638 | } |
| 639 | } |
| 640 | for _, code := range []string{"`0`", "`3`", "`4`", "`5`"} { |
| 641 | if !strings.Contains(section, code) { |
| 642 | t.Errorf("the section does not state exit code %s", code) |
| 643 | } |
| 644 | } |
| 645 | } |
| 646 | |
| 647 | // Ruling 1 (fix round 1): a 429 from request, cut, abort or retry comes |
| 648 | // before the handler, so the same call is sent again after the back-off, |
| 649 | // said once; the verb never exits on it. |
| 650 | func TestMoveMutationRetriesOn429(t *testing.T) { |
| 651 | for _, v := range []struct { |
| 652 | args []string |
| 653 | want string |
| 654 | }{ |
| 655 | {[]string{"--host", tHost, "--to", "uk", "--no-wait"}, "POST " + tMovePth + ` {"cut":"auto","region":"uk"}`}, |
| 656 | {[]string{"cut", "--host", tHost, "--no-wait"}, "POST " + tMovePth + "/cut {}"}, |
| 657 | {[]string{"abort", "--host", tHost, "--no-wait"}, "DELETE " + tMovePth + " "}, |
| 658 | {[]string{"resolve", "--host", tHost, "--retry", "--no-wait"}, "POST " + tMovePth + `/resolve {"retry":true}`}, |
| 659 | } { |
| 660 | limited := 2 |
| 661 | c := &fakeMoveConsole{frames: []frame{none(nil), inFlight("preparing", nil, nil)}} |
| 662 | c.answer = func(string, string, string) (int, any) { |
| 663 | if limited > 0 { |
| 664 | limited-- |
| 665 | return http.StatusTooManyRequests, map[string]string{"error": "too many instance requests. wait a minute and try again"} |
| 666 | } |
| 667 | return http.StatusAccepted, frame{"state": "preparing", "target": "uk", "id": tMoveID, "protocol": "phase1"} |
| 668 | } |
| 669 | if v.args[0] == "cut" || v.args[0] == "abort" || v.args[0] == "resolve" { |
| 670 | c.frames = []frame{inFlight("ready", nil, nil)} |
| 671 | } |
| 672 | code, out := runMoveAgainst(t, c, v.args...) |
| 673 | if code != 0 { |
| 674 | t.Fatalf("%v: exit %d, want 0:\n%s", v.args, code, out) |
| 675 | } |
| 676 | m := c.mutations() |
| 677 | if len(m) != 3 || m[0] != v.want || m[1] != v.want || m[2] != v.want { |
| 678 | t.Fatalf("%v: mutations %q, want the same call three times", v.args, m) |
| 679 | } |
| 680 | if n := strings.Count(out, "The console is limiting requests. Checking every 10 seconds."); n != 1 { |
| 681 | t.Fatalf("%v: the back-off line appears %d times, want once:\n%s", v.args, n, out) |
| 682 | } |
| 683 | wantNoLine(t, out, "too many instance requests") |
| 684 | } |
| 685 | } |
| 686 | |
| 687 | // Ruling 3: the outcome's regions are this move's own, bound from its own |
| 688 | // answers, never from a later move's record. Here a second move flips to fr |
| 689 | // while the first one's outcome is read. |
| 690 | func TestMoveOutcomeRegionsAreThisMoves(t *testing.T) { |
| 691 | second := func(last frame) frame { |
| 692 | f := inFlight("landing", frame{"id": "mv_another", "from": "uk", "to": "fr", "landing_phase": "probing"}, frame{"from": "uk", "region": "fr"}) |
| 693 | f["id"], f["target"], f["last"] = "mv_another", "fr", last |
| 694 | return f |
| 695 | } |
| 696 | for _, c := range []struct { |
| 697 | name string |
| 698 | last frame |
| 699 | code int |
| 700 | want string |
| 701 | }{ |
| 702 | {"done", outcome(tMoveID, "done", 41.2), 0, mc7}, |
| 703 | {"rolled back", outcome(tMoveID, "rolled_back", 0), 3, mc9}, |
| 704 | {"aborted", outcome(tMoveID, "aborted", 0), 3, "The move was aborted. acme.go.tito.io stays in ie."}, |
| 705 | } { |
| 706 | fc := &fakeMoveConsole{frames: []frame{none(nil), inFlight("preparing", nil, nil), second(c.last)}} |
| 707 | code, out := runMoveAgainst(t, fc, "--host", tHost, "--to", "uk") |
| 708 | if code != c.code { |
| 709 | t.Fatalf("%s: exit %d, want %d:\n%s", c.name, code, c.code, out) |
| 710 | } |
| 711 | wantLines(t, out, c.want) |
| 712 | wantNoLine(t, out, "from fr", "in fr") |
| 713 | } |
| 714 | // Bound from the request's own start answer, before any poll: the |
| 715 | // first poll already shows the second move, flipped to fr. |
| 716 | fc := &fakeMoveConsole{frames: []frame{none(nil), second(outcome(tMoveID, "done", 41.2))}, startDetail: inFlight("preparing", nil, nil)} |
| 717 | code, out := runMoveAgainst(t, fc, "--host", tHost, "--to", "uk") |
| 718 | if code != 0 { |
| 719 | t.Fatalf("bound at the start: exit %d:\n%s", code, out) |
| 720 | } |
| 721 | wantLines(t, out, mc1, mc7) |
| 722 | wantNoLine(t, out, "from fr", "in fr") |
| 723 | } |
| 724 | |
| 725 | // Ruling 4: a rollback the wait missed says M-C8 only when the outcome |
| 726 | // carries the reason, and never invents one. |
| 727 | func TestMoveMissedRollbackSaysItsReasonOnlyFromTheOutcome(t *testing.T) { |
| 728 | withReason := outcome(tMoveID, "rolled_back", 0) |
| 729 | withReason["reason"] = "the landing ran past its deadline" |
| 730 | c := &fakeMoveConsole{frames: []frame{none(nil), inFlight("preparing", nil, nil), none(withReason)}} |
| 731 | code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk") |
| 732 | if code != 3 { |
| 733 | t.Fatalf("exit %d, want 3:\n%s", code, out) |
| 734 | } |
| 735 | wantLines(t, out, mc8, mc9) |
| 736 | if strings.Index(out, mc8) > strings.Index(out, mc9) { |
| 737 | t.Fatalf("M-C8 after M-C9:\n%s", out) |
| 738 | } |
| 739 | } |
| 740 | |
| 741 | // L19 strictly: the detail is read only when the state or the phase changes, |
| 742 | // whatever else (updated_at) does. A release deadline that passes while |
| 743 | // nothing changes is still seen, from the detail already read. |
| 744 | func TestMoveWaitReadsDetailOnlyOnStateOrPhase(t *testing.T) { |
| 745 | frames := []frame{none(nil)} |
| 746 | for i := 0; i < 6; i++ { |
| 747 | f := inFlight("preparing", nil, nil) |
| 748 | f["updated_at"] = int64(1000 + i) // an unrelated record update each poll |
| 749 | frames = append(frames, f) |
| 750 | } |
| 751 | for _, p := range []string{"admitted", "admitted", "stopping", "stopping"} { |
| 752 | frames = append(frames, inFlight("cutting", frame{"cut_phase": p, "cut_budget_s": 45}, nil)) |
| 753 | } |
| 754 | done := none(outcome(tMoveID, "done", 41.2)) |
| 755 | frames = append(frames, done) |
| 756 | c := &fakeMoveConsole{frames: frames} |
| 757 | code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk") |
| 758 | if code != 0 { |
| 759 | t.Fatalf("exit %d:\n%s", code, out) |
| 760 | } |
| 761 | // One detail read at the request's start, then one per change: preparing, |
| 762 | // cutting/admitted, cutting/stopping, none. |
| 763 | if c.details != 5 { |
| 764 | t.Fatalf("%d detail reads, want 5 (the start and four changes) over %d flat polls", c.details, c.flatGets) |
| 765 | } |
| 766 | |
| 767 | soon := time.Now().Add(150 * time.Millisecond).UTC().Format(time.RFC3339Nano) |
| 768 | rb := frame{"id": "rb_x", "reason": "the landing ran past its deadline", "target_phase": "releasing", "release_deadline": soon} |
| 769 | c = &fakeMoveConsole{frames: []frame{none(nil), inFlight("rolling_back", frame{"rollback": rb}, nil)}} |
| 770 | code, out = runMoveAgainst(t, c, "--host", tHost, "--to", "uk") |
| 771 | if code != 4 { |
| 772 | t.Fatalf("deadline passing: exit %d, want 4:\n%s", code, out) |
| 773 | } |
| 774 | wantLines(t, out, "Waiting for uk-box in uk to confirm it has stopped acme.go.tito.io. acme.go.tito.io is offline.") |
| 775 | if c.details > 2 { |
| 776 | t.Fatalf("%d detail reads while nothing changed, want at most 2", c.details) |
| 777 | } |
| 778 | } |
| 779 | |
| 780 | // Ruling 6: the flag help is said through moveFlagHelp, the copy gate's |
| 781 | // registered help emission: on -h, and after a bad flag, with the flag |
| 782 | // package's own error. |
| 783 | func TestMoveFlagHelp(t *testing.T) { |
| 784 | var out bytes.Buffer |
| 785 | if _, err := parseMoveArgs([]string{"-h"}, &out); !errors.Is(err, flag.ErrHelp) { |
| 786 | t.Fatalf("-h: %v", err) |
| 787 | } |
| 788 | for _, want := range []string{"-host", "the instance's host name", "-to", "-manual-cut"} { |
| 789 | if !strings.Contains(out.String(), want) { |
| 790 | t.Errorf("help lacks %q:\n%s", want, out.String()) |
| 791 | } |
| 792 | } |
| 793 | out.Reset() |
| 794 | if _, err := parseMoveArgs([]string{"resolve", "--host", tHost, "--instance-id", "i-1"}, &out); err == nil { |
| 795 | t.Fatal("--instance-id parsed") |
| 796 | } |
| 797 | if !strings.Contains(out.String(), "flag provided but not defined: -instance-id") || !strings.Contains(out.String(), "-retry") { |
| 798 | t.Errorf("a bad flag's help:\n%s", out.String()) |
| 799 | } |
| 800 | } |
| 801 | |
| 802 | // Fix round 2, ruling 3: a region the wait never learnt is not filled in. A |
| 803 | // known origin with an unknown destination says M-C9 alone, even when the |
| 804 | // outcome carries a reason, since M-C8 names the destination. |
| 805 | func TestMoveOutcomeNeverGuessesARegion(t *testing.T) { |
| 806 | var out bytes.Buffer |
| 807 | m := &mover{out: &out, host: tHost} |
| 808 | w := newMoveWatch(m, tMoveID) |
| 809 | w.fromRegion = "ie" |
| 810 | last := &moveOutcome{ID: tMoveID, Outcome: "rolled_back", Reason: "the landing ran past its deadline"} |
| 811 | a := moveAnswer{moveFlat: moveFlat{State: "none", Last: last}, Detail: &moveDetailView{Region: "fr"}} |
| 812 | if code := w.outcome(a); code != 3 { |
| 813 | t.Fatalf("exit %d", code) |
| 814 | } |
| 815 | if got := out.String(); got != mc9+"\n" { |
| 816 | t.Fatalf("output %q, want M-C9 alone", got) |
| 817 | } |
| 818 | } |
| 819 | |
| 820 | // Fix round 2, ruling 1: what the CLI says when the console is not reached, |
| 821 | // answers with no reason, or the session cannot be made, is this file's own |
| 822 | // copy; only the console's own sentence is passed on, as server text. |
| 823 | func TestMoveFixedSentences(t *testing.T) { |
| 824 | // Nothing listens: the transport fails. |
| 825 | ts := httptest.NewServer(http.NotFoundHandler()) |
| 826 | dead := ts.URL |
| 827 | ts.Close() |
| 828 | deps := moveDeps{ |
| 829 | session: func(string) (*apiSession, error) { |
| 830 | return &apiSession{c: newConsoleClient(dead, "cli-token"), console: dead, address: "op@keymail.dev"}, nil |
| 831 | }, |
| 832 | euid: func() int { return 1000 }, poll: time.Millisecond, |
| 833 | } |
| 834 | var out bytes.Buffer |
| 835 | if code := moveMain(context.Background(), []string{"status", "--host", tHost, "--account", "bab", "--app", "titogo"}, &out, deps); code != 1 { |
| 836 | t.Fatalf("unreachable console: exit %d:\n%s", code, out.String()) |
| 837 | } |
| 838 | wantLines(t, out.String(), "Could not reach the console. Check the connection and try again.") |
| 839 | |
| 840 | // The session cannot be made: one sentence of this file. |
| 841 | out.Reset() |
| 842 | deps.session = func(string) (*apiSession, error) { return nil, errors.New("unreviewed resolveMode text") } |
| 843 | if code := moveMain(context.Background(), []string{"status", "--host", tHost, "--account", "bab", "--app", "titogo"}, &out, deps); code != 1 { |
| 844 | t.Fatalf("no session: exit %d:\n%s", code, out.String()) |
| 845 | } |
| 846 | wantLines(t, out.String(), "carlos move could not sign in to a console. Run carlos auth login, or check --console, CARLOS_CONSOLE and CARLOS_SERVICE_TOKEN.") |
| 847 | wantNoLine(t, out.String(), "unreviewed resolveMode text") |
| 848 | |
| 849 | // A refusal with no JSON reason (the family's 404), and a 5xx with none. |
| 850 | for _, c := range []struct { |
| 851 | status int |
| 852 | code int |
| 853 | want string |
| 854 | }{ |
| 855 | {http.StatusNotFound, 5, "Not found. Check --account and --app, and that you are a member of the instance's account."}, |
| 856 | {http.StatusBadGateway, 1, "The console answered 502 and gave no reason."}, |
| 857 | } { |
| 858 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 859 | w.WriteHeader(c.status) |
| 860 | _, _ = w.Write([]byte("plain text")) |
| 861 | })) |
| 862 | deps.session = func(string) (*apiSession, error) { |
| 863 | return &apiSession{c: newConsoleClient(ts.URL, "cli-token"), console: ts.URL, address: "op@keymail.dev"}, nil |
| 864 | } |
| 865 | out.Reset() |
| 866 | code := moveMain(context.Background(), []string{"status", "--host", tHost, "--account", "bab", "--app", "titogo"}, &out, deps) |
| 867 | ts.Close() |
| 868 | if code != c.code { |
| 869 | t.Fatalf("%d with no reason: exit %d, want %d:\n%s", c.status, code, c.code, out.String()) |
| 870 | } |
| 871 | wantLines(t, out.String(), c.want) |
| 872 | wantNoLine(t, out.String(), "plain text") |
| 873 | } |
| 874 | } |
| 875 | |
| 876 | // ---- Task 1b: carlos move resolve --declare-lost (claim Plan 10's door) ---- |
| 877 | |
| 878 | // fakeEC2 answers DescribeInstances for one instance in one state. |
| 879 | type fakeEC2 struct { |
| 880 | account, instance, state string |
| 881 | calls *int |
| 882 | } |
| 883 | |
| 884 | func (f fakeEC2) DescribeInstances(context.Context, *ec2.DescribeInstancesInput, ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) { |
| 885 | *f.calls++ |
| 886 | out := &ec2.DescribeInstancesOutput{Reservations: []ec2types.Reservation{{ |
| 887 | OwnerId: aws.String(f.account), |
| 888 | Instances: []ec2types.Instance{{InstanceId: aws.String(f.instance), State: &ec2types.InstanceState{Name: ec2types.InstanceStateName(f.state)}}}, |
| 889 | }}} |
| 890 | awsmiddleware.SetRequestIDMetadata(&out.ResultMetadata, "aws-request-1b") |
| 891 | return out, nil |
| 892 | } |
| 893 | |
| 894 | const tSignedChallenge = `{"envelope":"signed by the test console"}` |
| 895 | |
| 896 | // lossChallenge is an inspection as claim Plan 10's move door answers it. |
| 897 | func lossChallenge(t *testing.T, transition string) hostresolve.Inspection { |
| 898 | t.Helper() |
| 899 | c := hostfence.Challenge{ |
| 900 | ID: "loss-0123456789abcdef", Actor: "op@keymail.dev", Host: tHost, |
| 901 | HolderIncarnation: "inc-1", HolderStamp: 3, HolderSession: 4, ClaimGen: 9, ClaimETag: "etag-claim", |
| 902 | Op: hostfence.Operation{ID: tMoveID, Kind: hostclaim.KindLanding, Phase: "probing"}, |
| 903 | Record: hostfence.RecordObservation{Key: "apps/bab/titogo/instances/acme.go.tito.io.json", ETag: "etag-record", CommitState: "present"}, |
| 904 | Box: "uk-box", Enrollment: "0123456789abcdef0123456789abcdef", |
| 905 | Cloud: hostfence.CloudBinding{Provider: "aws", Account: "111122223333", Region: "eu-west-2", InstanceID: "i-uk-box"}, |
| 906 | Consequences: hostfence.Consequences{Transition: transition, PossibleWriteLoss: true}, |
| 907 | } |
| 908 | if transition == "destroyed-local-only" { |
| 909 | c.Consequences = hostfence.Consequences{ |
| 910 | Transition: "destroyed-local-only", |
| 911 | DischargeLocal: []hostclaim.ObligationRef{{ID: "local", Op: tMoveID, Retirement: "retired-installation"}}, |
| 912 | DestroyedStorageAssertion: []string{tMoveID + "/local/retired-installation"}, |
| 913 | } |
| 914 | } |
| 915 | c, err := hostfence.PrepareChallenge(time.Now().Add(-time.Second), c) |
| 916 | if err != nil { |
| 917 | t.Fatal(err) |
| 918 | } |
| 919 | return hostresolve.Inspection{Challenge: json.RawMessage(tSignedChallenge), Details: c} |
| 920 | } |
| 921 | |
| 922 | // lossConsole is a console in landing whose move door answers the |
| 923 | // inspection, then the declaration (202, the flat view in rolling_back). |
| 924 | func lossConsole(t *testing.T, in hostresolve.Inspection, declared *hostfence.Submission) *fakeMoveConsole { |
| 925 | flipped := time.Date(2026, 10, 11, 9, 30, 0, 0, time.UTC) |
| 926 | c := &fakeMoveConsole{frames: []frame{inFlight("landing", frame{"flipped_at": flipped.Format(time.RFC3339)}, nil)}} |
| 927 | c.answer = func(method, path, body string) (int, any) { |
| 928 | var req struct { |
| 929 | DeclareLost string `json:"declare_lost"` |
| 930 | Submission *hostfence.Submission `json:"submission"` |
| 931 | } |
| 932 | if path != tMovePth+"/resolve" || json.Unmarshal([]byte(body), &req) != nil || req.DeclareLost != "uk-box" { |
| 933 | t.Errorf("unexpected %s %s %s", method, path, body) |
| 934 | return http.StatusBadRequest, map[string]string{"error": "invalid resolution request"} |
| 935 | } |
| 936 | if req.Submission == nil { |
| 937 | return http.StatusOK, in |
| 938 | } |
| 939 | *declared = *req.Submission |
| 940 | return http.StatusAccepted, frame{"state": "rolling_back", "target": "uk", "id": tMoveID, "protocol": "phase1"} |
| 941 | } |
| 942 | return c |
| 943 | } |
| 944 | |
| 945 | func TestMoveDeclareLost(t *testing.T) { |
| 946 | const mc18 = "This declares uk-box lost. Anything acme.go.tito.io wrote on uk-box after 2026-10-11 09:30:00 UTC is lost. Type the host name to confirm." |
| 947 | const localOnly = "This records that uk-box is gone with its disks. acme.go.tito.io keeps nothing there that it still needs. Type the host name to confirm." |
| 948 | args := []string{"resolve", "--host", tHost, "--declare-lost", "uk-box", "--no-wait"} |
| 949 | reader := func(state string, calls *int, region *string) func(*moveDeps) { |
| 950 | return func(d *moveDeps) { |
| 951 | d.stdin = strings.NewReader(tHost + "\n") |
| 952 | d.ec2 = func(_ context.Context, r string) (awsinv.ExactEC2API, error) { |
| 953 | *region = r |
| 954 | return fakeEC2{account: "111122223333", instance: "i-uk-box", state: state, calls: calls}, nil |
| 955 | } |
| 956 | } |
| 957 | } |
| 958 | |
| 959 | t.Run("move-target-lost", func(t *testing.T) { |
| 960 | in := lossChallenge(t, "move-target-lost") |
| 961 | var sub hostfence.Submission |
| 962 | calls, region := 0, "" |
| 963 | code, out := runMoveWith(t, lossConsole(t, in, &sub), reader("stopped", &calls, ®ion), args...) |
| 964 | if code != 0 { |
| 965 | t.Fatalf("exit %d:\n%s", code, out) |
| 966 | } |
| 967 | wantLines(t, out, mc18, "Move "+tMoveID+" of acme.go.tito.io is rolling_back.") |
| 968 | wantNoLine(t, out, localOnly) |
| 969 | if region != "eu-west-2" || calls != 1 { |
| 970 | t.Fatalf("EC2 read in %q, %d calls; want the challenge's region, once", region, calls) |
| 971 | } |
| 972 | want := hostfence.OperatorAttestation{MaintainComputeFence: true, ColdQuarantineStartOnly: true} |
| 973 | if string(sub.Challenge) != tSignedChallenge || sub.Confirmation != in.Details.ID || sub.RequestID != "aws-request-1b" || |
| 974 | sub.Observation.State != "stopped" || sub.Observation.InstanceID != "i-uk-box" || sub.Attestation.MaintainComputeFence != want.MaintainComputeFence || |
| 975 | sub.Attestation.ColdQuarantineStartOnly != want.ColdQuarantineStartOnly || len(sub.Attestation.Disks) != 0 { |
| 976 | t.Fatalf("submission %+v", sub) |
| 977 | } |
| 978 | }) |
| 979 | t.Run("destroyed-local-only", func(t *testing.T) { |
| 980 | in := lossChallenge(t, "destroyed-local-only") |
| 981 | var sub hostfence.Submission |
| 982 | calls, region := 0, "" |
| 983 | code, out := runMoveWith(t, lossConsole(t, in, &sub), reader("terminated", &calls, ®ion), args...) |
| 984 | if code != 0 { |
| 985 | t.Fatalf("exit %d:\n%s", code, out) |
| 986 | } |
| 987 | wantLines(t, out, localOnly) |
| 988 | wantNoLine(t, out, "This declares uk-box lost.") |
| 989 | if strings.Join(sub.Attestation.Disks, ",") != tMoveID+"/local/retired-installation" || sub.Confirmation != in.Details.ID { |
| 990 | t.Fatalf("submission %+v", sub) |
| 991 | } |
| 992 | }) |
| 993 | t.Run("a wrong host name refuses", func(t *testing.T) { |
| 994 | in := lossChallenge(t, "move-target-lost") |
| 995 | var sub hostfence.Submission |
| 996 | c := lossConsole(t, in, &sub) |
| 997 | code, out := runMoveWith(t, c, func(d *moveDeps) { d.stdin = strings.NewReader("acme.go.tito.io.evil\n") }, args...) |
| 998 | if code != 5 { |
| 999 | t.Fatalf("exit %d, want 5:\n%s", code, out) |
| 1000 | } |
| 1001 | wantLines(t, out, mc18, "That is not acme.go.tito.io. Nothing was declared.") |
| 1002 | if m := c.mutations(); len(m) != 1 || sub.Confirmation != "" { |
| 1003 | t.Fatalf("a refused confirmation declared: %q", m) |
| 1004 | } |
| 1005 | }) |
| 1006 | t.Run("a running box is refused before the declaration", func(t *testing.T) { |
| 1007 | in := lossChallenge(t, "move-target-lost") |
| 1008 | var sub hostfence.Submission |
| 1009 | calls, region := 0, "" |
| 1010 | c := lossConsole(t, in, &sub) |
| 1011 | code, out := runMoveWith(t, c, reader("running", &calls, ®ion), args...) |
| 1012 | if code != 5 { |
| 1013 | t.Fatalf("exit %d, want 5:\n%s", code, out) |
| 1014 | } |
| 1015 | wantLines(t, out, "AWS does not show uk-box stopped or terminated, so it cannot be declared lost.") |
| 1016 | if m := c.mutations(); len(m) != 1 { |
| 1017 | t.Fatalf("mutations %q: want the inspection only", m) |
| 1018 | } |
| 1019 | }) |
| 1020 | t.Run("no AWS credentials", func(t *testing.T) { |
| 1021 | in := lossChallenge(t, "move-target-lost") |
| 1022 | var sub hostfence.Submission |
| 1023 | c := lossConsole(t, in, &sub) |
| 1024 | code, out := runMoveWith(t, c, func(d *moveDeps) { |
| 1025 | d.stdin = strings.NewReader(tHost + "\n") |
| 1026 | d.ec2 = func(context.Context, string) (awsinv.ExactEC2API, error) { |
| 1027 | return nil, errors.New("no credential chain") |
| 1028 | } |
| 1029 | }, args...) |
| 1030 | if code != 5 { |
| 1031 | t.Fatalf("exit %d, want 5:\n%s", code, out) |
| 1032 | } |
| 1033 | wantLines(t, out, "declare-lost needs your AWS credentials to confirm the box is stopped or terminated.") |
| 1034 | wantNoLine(t, out, "no credential chain") |
| 1035 | }) |
| 1036 | // Fix round 1, finding 1: M-C18 is bound to the inspected move. A detail |
| 1037 | // that names another move, or another incarnation, than the inspection's |
| 1038 | // challenge refuses before anything is shown, read or declared. |
| 1039 | for _, c := range []struct { |
| 1040 | name string |
| 1041 | edit func(*hostresolve.Inspection) |
| 1042 | }{ |
| 1043 | {"another move", func(in *hostresolve.Inspection) { in.Details.Op.ID = "mv_bbbbbbbbbbbbbbbbbbbbbbbbbb" }}, |
| 1044 | {"another incarnation", func(in *hostresolve.Inspection) { in.Details.HolderIncarnation = "inc-other" }}, |
| 1045 | } { |
| 1046 | t.Run("the inspection names "+c.name, func(t *testing.T) { |
| 1047 | in := lossChallenge(t, "move-target-lost") |
| 1048 | c.edit(&in) |
| 1049 | var sub hostfence.Submission |
| 1050 | fc := lossConsole(t, in, &sub) |
| 1051 | fc.frames[0]["detail"].(frame)["move"].(frame)["incarnation"] = "inc-1" |
| 1052 | code, out := runMoveWith(t, fc, func(d *moveDeps) { d.stdin = strings.NewReader(tHost + "\n") }, args...) |
| 1053 | if code != 5 { |
| 1054 | t.Fatalf("exit %d, want 5:\n%s", code, out) |
| 1055 | } |
| 1056 | wantLines(t, out, "The move changed since it was read. Nothing was declared. Run the command again.") |
| 1057 | wantNoLine(t, out, "This declares uk-box lost.") |
| 1058 | if m := fc.mutations(); len(m) != 1 || sub.Confirmation != "" { |
| 1059 | t.Fatalf("mutations %q: want the inspection only", m) |
| 1060 | } |
| 1061 | }) |
| 1062 | } |
| 1063 | t.Run("the console refuses", func(t *testing.T) { |
| 1064 | c := &fakeMoveConsole{frames: []frame{inFlight("landing", nil, nil)}, answer: func(string, string, string) (int, any) { |
| 1065 | return http.StatusConflict, map[string]string{"error": "declare-lost is not enabled in this deployment yet"} |
| 1066 | }} |
| 1067 | code, out := runMoveAgainst(t, c, args...) |
| 1068 | if code != 5 { |
| 1069 | t.Fatalf("exit %d, want 5:\n%s", code, out) |
| 1070 | } |
| 1071 | wantLines(t, out, "declare-lost is not enabled in this deployment yet") |
| 1072 | }) |
| 1073 | } |
| 1074 | |
| 1075 | // The cut guard (Paul's ruling, 2026-10-11): a cut the source refused goes |
| 1076 | // back to ready with the reason recorded on the move, and `carlos move |
| 1077 | // status` says it. While the refusal is recorded (until a repin clears it) |
| 1078 | // the move is not ready to cut, so M-C12 is not said (fix round 1). A ready |
| 1079 | // move with no reason says M-C12 alone. |
| 1080 | func TestMoveStatusSaysWhyTheCutDidNotStart(t *testing.T) { |
| 1081 | for _, reason := range []string{provision.RefusedTargetUnarmed, provision.RefusedSourceUnarmed, provision.RefusedReplicaLayout} { |
| 1082 | c := &fakeMoveConsole{frames: []frame{inFlight("ready", frame{"cut": "manual", "reason": reason}, nil)}} |
| 1083 | code, out := runMoveAgainst(t, c, "status", "--host", tHost) |
| 1084 | if code != moveExitWaiting { |
| 1085 | t.Fatalf("exit %d:\n%s", code, out) |
| 1086 | } |
| 1087 | wantLines(t, out, "Move "+tMoveID+" of acme.go.tito.io is ready.", "The cut did not start: "+reason+".") |
| 1088 | wantNoLine(t, out, mc12) |
| 1089 | } |
| 1090 | c := &fakeMoveConsole{frames: []frame{inFlight("ready", frame{"cut": "manual"}, nil)}} |
| 1091 | code, out := runMoveAgainst(t, c, "status", "--host", tHost) |
| 1092 | if code != moveExitWaiting || strings.Contains(out, "did not start") { |
| 1093 | t.Fatalf("a ready move with no refusal = %d:\n%s", code, out) |
| 1094 | } |
| 1095 | wantLines(t, out, mc12) |
| 1096 | } |
| 1097 | |
| 1098 | // ---- carlos move offline ---- |
| 1099 | |
| 1100 | // tOffMovePth is the move door of the offline tests' instance, oHost of the |
| 1101 | // app shop. |
| 1102 | const tOffMovePth = "/api/cli/apps/bab/shop/instances/" + oHost + "/move" |
| 1103 | |
| 1104 | const tOfflinePth = tOffMovePth + "/offline" |
| 1105 | |
| 1106 | // offPost is one scripted answer of a mutating door; the last one repeats. |
| 1107 | type offPost struct { |
| 1108 | code int |
| 1109 | body any |
| 1110 | } |
| 1111 | |
| 1112 | // fakeOfflineConsole answers the offline doors in T6's shapes. Each GET of the |
| 1113 | // status door returns the next scripted view (the last repeats); each POST |
| 1114 | // returns the next scripted answer of its door. Anything outside the offline |
| 1115 | // doors is a 404, so a verb that read the Phase 1 door would fail. |
| 1116 | type fakeOfflineConsole struct { |
| 1117 | mu sync.Mutex |
| 1118 | gets []frame |
| 1119 | i int |
| 1120 | posts map[string][]offPost // keyed by the door's tail: "", "/seal", ... |
| 1121 | n map[string]int |
| 1122 | log []string // method path body |
| 1123 | } |
| 1124 | |
| 1125 | func (c *fakeOfflineConsole) ServeHTTP(w http.ResponseWriter, r *http.Request) { |
| 1126 | c.mu.Lock() |
| 1127 | defer c.mu.Unlock() |
| 1128 | write := func(code int, v any) { |
| 1129 | w.Header().Set("Content-Type", "application/json") |
| 1130 | w.WriteHeader(code) |
| 1131 | _ = json.NewEncoder(w).Encode(v) |
| 1132 | } |
| 1133 | body, _ := io.ReadAll(r.Body) |
| 1134 | c.log = append(c.log, r.Method+" "+strings.TrimPrefix(r.URL.Path, tOffMovePth)+" "+strings.TrimSpace(string(body))) |
| 1135 | if !strings.HasPrefix(r.URL.Path, tOfflinePth) { |
| 1136 | http.NotFound(w, r) |
| 1137 | return |
| 1138 | } |
| 1139 | if r.Method == http.MethodGet && r.URL.Path == tOfflinePth { |
| 1140 | n := c.i |
| 1141 | if n >= len(c.gets) { |
| 1142 | n = len(c.gets) - 1 |
| 1143 | } |
| 1144 | c.i++ |
| 1145 | write(http.StatusOK, c.gets[n]) |
| 1146 | return |
| 1147 | } |
| 1148 | tail := strings.TrimPrefix(r.URL.Path, tOfflinePth) |
| 1149 | list := c.posts[tail] |
| 1150 | if r.Method != http.MethodPost || len(list) == 0 { |
| 1151 | http.NotFound(w, r) |
| 1152 | return |
| 1153 | } |
| 1154 | if c.n == nil { |
| 1155 | c.n = map[string]int{} |
| 1156 | } |
| 1157 | k := c.n[tail] |
| 1158 | if k >= len(list) { |
| 1159 | k = len(list) - 1 |
| 1160 | } |
| 1161 | c.n[tail]++ |
| 1162 | write(list[k].code, list[k].body) |
| 1163 | } |
| 1164 | |
| 1165 | func (c *fakeOfflineConsole) requests() []string { |
| 1166 | c.mu.Lock() |
| 1167 | defer c.mu.Unlock() |
| 1168 | return append([]string(nil), c.log...) |
| 1169 | } |
| 1170 | |
| 1171 | // posted counts the POSTs a door got. |
| 1172 | func (c *fakeOfflineConsole) posted(tail string) int { |
| 1173 | c.mu.Lock() |
| 1174 | defer c.mu.Unlock() |
| 1175 | return c.n[tail] |
| 1176 | } |
| 1177 | |
| 1178 | func runOfflineAgainst(t *testing.T, c *fakeOfflineConsole, args ...string) (int, string) { |
| 1179 | t.Helper() |
| 1180 | ts := httptest.NewServer(c) |
| 1181 | t.Cleanup(ts.Close) |
| 1182 | var out bytes.Buffer |
| 1183 | deps := moveDeps{ |
| 1184 | session: func(string) (*apiSession, error) { |
| 1185 | return &apiSession{c: newConsoleClient(ts.URL, "cli-token"), console: ts.URL, address: "op@keymail.dev"}, nil |
| 1186 | }, |
| 1187 | euid: func() int { return 1000 }, |
| 1188 | poll: time.Millisecond, |
| 1189 | stdin: strings.NewReader(""), |
| 1190 | } |
| 1191 | args = append([]string{"offline"}, args...) |
| 1192 | args = append(args, "--host", oHost, "--account", "bab", "--app", "shop") |
| 1193 | ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) |
| 1194 | defer cancel() |
| 1195 | code := moveMain(ctx, args, &out, deps) |
| 1196 | return code, out.String() |
| 1197 | } |
| 1198 | |
| 1199 | // offView is the offline status view of a move in state, as T6 builds it. |
| 1200 | func offView(state string, extra frame) frame { |
| 1201 | f := frame{"host": oHost, "state": state, "id": tMoveID, "from": "ie", "to": "uk", |
| 1202 | "sealer": "ie-box", "boxes": []string{"ie-box"}} |
| 1203 | for k, v := range extra { |
| 1204 | f[k] = v |
| 1205 | } |
| 1206 | return f |
| 1207 | } |
| 1208 | |
| 1209 | func offDone(outcome string, offlineS float64) frame { |
| 1210 | return frame{"host": oHost, "state": "none", |
| 1211 | "last": frame{"id": tMoveID, "outcome": outcome, "at": "2026-10-20T10:00:00Z", "offline_s": offlineS}} |
| 1212 | } |
| 1213 | |
| 1214 | func offReport(box, role, phase string, fresh bool) frame { |
| 1215 | return frame{"box": box, "role": role, "phase": phase, "fresh": fresh} |
| 1216 | } |
| 1217 | |
| 1218 | const ( |
| 1219 | oHost = "acme.shop.example.com" |
| 1220 | |
| 1221 | oStarting = "Moving acme.shop.example.com from ie to uk." |
| 1222 | oOffline = "acme.shop.example.com is offline from now until the move opens or is aborted." |
| 1223 | oStopped = "Every box in ie has stopped acme.shop.example.com. Run carlos move offline seal --host acme.shop.example.com to copy it." |
| 1224 | oCopying = "Copying acme.shop.example.com to uk." |
| 1225 | oCopied = "Copied and checked. Run carlos move offline flip --host acme.shop.example.com to start it in uk." |
| 1226 | oFlipped = "Stopped in ie. Starting in uk." |
| 1227 | oLanded = "uk holds everything acme.shop.example.com needs. Nothing is served yet. Run carlos move offline open --host acme.shop.example.com to serve it from uk." |
| 1228 | oServed = "acme.shop.example.com is served from uk. Run carlos move offline finish --host acme.shop.example.com once ie has cleaned up." |
| 1229 | oBack = "Moving acme.shop.example.com back to ie." |
| 1230 | oPutBack = "Every box has put acme.shop.example.com back in ie. Run carlos move offline finish --host acme.shop.example.com to end the move." |
| 1231 | oWaitBox = "Waiting for ie-box in ie to confirm it has stopped acme.shop.example.com. acme.shop.example.com is offline." |
| 1232 | ) |
| 1233 | |
| 1234 | func TestParseMoveArgsOffline(t *testing.T) { |
| 1235 | for _, c := range []struct { |
| 1236 | args []string |
| 1237 | ok bool |
| 1238 | sub string |
| 1239 | }{ |
| 1240 | {[]string{"offline", "start", "--host", oHost, "--to", "uk"}, true, "start"}, |
| 1241 | {[]string{"offline", "stage", "--host", oHost, "--to", "uk"}, true, "stage"}, |
| 1242 | {[]string{"offline", "stage", "--host", oHost}, false, ""}, |
| 1243 | {[]string{"offline", "seal", "--host", oHost}, true, "seal"}, |
| 1244 | {[]string{"offline", "flip", "--host", oHost}, true, "flip"}, |
| 1245 | {[]string{"offline", "open", "--host", oHost}, true, "open"}, |
| 1246 | {[]string{"offline", "finish", "--host", oHost}, true, "finish"}, |
| 1247 | {[]string{"offline", "abort", "--host", oHost}, true, "abort"}, |
| 1248 | {[]string{"offline", "status", "--host", oHost, "--app", "x", "--account", "y"}, true, "status"}, |
| 1249 | {[]string{"offline", "start", "--host", oHost}, false, ""}, |
| 1250 | {[]string{"offline", "start", "--to", "uk"}, false, ""}, |
| 1251 | {[]string{"offline", "seal", "--host", oHost, "--to", "uk"}, false, ""}, |
| 1252 | {[]string{"offline", "seal", "--host", oHost, "--no-wait"}, false, ""}, |
| 1253 | {[]string{"offline", "seal", "--host", oHost, "extra"}, false, ""}, |
| 1254 | {[]string{"offline"}, false, ""}, |
| 1255 | {[]string{"offline", "cut", "--host", oHost}, false, ""}, |
| 1256 | {[]string{"offline", "--host", oHost}, false, ""}, |
| 1257 | } { |
| 1258 | got, err := parseMoveArgs(c.args, io.Discard) |
| 1259 | if (err == nil) != c.ok || (c.ok && (got.verb != "offline" || got.offline != c.sub)) { |
| 1260 | t.Errorf("parseMoveArgs(%q) = %+v, %v; want ok %v offline %q", c.args, got, err, c.ok, c.sub) |
| 1261 | } |
| 1262 | } |
| 1263 | var out bytes.Buffer |
| 1264 | if _, err := parseMoveArgs([]string{"offline", "cut", "--host", oHost}, &out); err == nil { |
| 1265 | t.Fatal("an unknown offline verb parsed") |
| 1266 | } |
| 1267 | wantLines(t, out.String(), "carlos move offline needs stage, start, seal, flip, open, finish, abort or status.") |
| 1268 | out.Reset() |
| 1269 | _, _ = parseMoveArgs([]string{"offline", "start", "--host", oHost}, &out) |
| 1270 | wantLines(t, out.String(), "carlos move needs --to <region>.") |
| 1271 | // The Phase 1 verbs are untouched. |
| 1272 | if got, err := parseMoveArgs([]string{"abort", "--host", oHost}, io.Discard); err != nil || got.verb != "abort" || got.offline != "" { |
| 1273 | t.Errorf("carlos move abort = %+v, %v", got, err) |
| 1274 | } |
| 1275 | } |
| 1276 | |
| 1277 | func TestMoveOfflineStart(t *testing.T) { |
| 1278 | c := &fakeOfflineConsole{ |
| 1279 | posts: map[string][]offPost{"": {{202, offView("offline-stopping", frame{"waiting": oWaitBox})}}}, |
| 1280 | gets: []frame{ |
| 1281 | offView("offline-stopping", frame{"waiting": oWaitBox}), |
| 1282 | offView("offline-stopping", frame{"waiting": oWaitBox}), |
| 1283 | offView("offline-stopping", frame{"next": "seal"}), |
| 1284 | }, |
| 1285 | } |
| 1286 | code, out := runOfflineAgainst(t, c, "start", "--to", "uk") |
| 1287 | if code != 0 { |
| 1288 | t.Fatalf("exit %d:\n%s", code, out) |
| 1289 | } |
| 1290 | want := oStarting + "\n" + oOffline + "\n" + oWaitBox + "\n" + oStopped + "\n" |
| 1291 | if out != want { |
| 1292 | t.Fatalf("output:\n%s\nwant:\n%s", out, want) |
| 1293 | } |
| 1294 | reqs := c.requests() |
| 1295 | if reqs[0] != `POST {"region":"uk"}` && reqs[0] != `POST /offline {"region":"uk"}` { |
| 1296 | t.Fatalf("first request %q", reqs[0]) |
| 1297 | } |
| 1298 | for _, r := range reqs { |
| 1299 | if !strings.Contains(r, "/offline") { |
| 1300 | t.Errorf("a request left the offline doors: %q", r) |
| 1301 | } |
| 1302 | } |
| 1303 | } |
| 1304 | |
| 1305 | func TestMoveOfflineRefusals(t *testing.T) { |
| 1306 | for _, c := range []struct { |
| 1307 | name string |
| 1308 | code int |
| 1309 | body any |
| 1310 | exit int |
| 1311 | say string |
| 1312 | }{ |
| 1313 | {"claim console", 409, frame{"error": "offline moves are not available while host claims or Phase 1 moves are on"}, 5, "offline moves are not available while host claims or Phase 1 moves are on"}, |
| 1314 | {"not found", 404, "404 page not found", 5, "Not found. Check --account and --app, and that you are a member of the instance's account."}, |
| 1315 | {"console error", 502, frame{"error": "could not write the move. try again"}, 1, "could not write the move. try again"}, |
| 1316 | {"no reason", 500, frame{}, 1, "The console answered 500 and gave no reason."}, |
| 1317 | } { |
| 1318 | t.Run(c.name, func(t *testing.T) { |
| 1319 | f := &fakeOfflineConsole{posts: map[string][]offPost{"": {{c.code, c.body}}, "/seal": {{c.code, c.body}}}, gets: []frame{offView("offline-stopping", nil)}} |
| 1320 | for _, args := range [][]string{{"start", "--to", "uk"}, {"seal"}} { |
| 1321 | code, out := runOfflineAgainst(t, f, args...) |
| 1322 | if code != c.exit || out != c.say+"\n" { |
| 1323 | t.Errorf("%v: exit %d, output %q; want %d, %q", args, code, out, c.exit, c.say) |
| 1324 | } |
| 1325 | } |
| 1326 | for _, r := range f.requests() { |
| 1327 | if strings.HasPrefix(r, "GET") { |
| 1328 | t.Errorf("a refused verb polled: %q", r) |
| 1329 | } |
| 1330 | } |
| 1331 | }) |
| 1332 | } |
| 1333 | } |
| 1334 | |
| 1335 | func TestMoveOfflineSeal(t *testing.T) { |
| 1336 | proc := "ie-box still runs a process for acme.shop.example.com. wait for it to stop, then flip" |
| 1337 | c := &fakeOfflineConsole{ |
| 1338 | posts: map[string][]offPost{"/seal": {{202, offView("offline-sealing", nil)}}}, |
| 1339 | gets: []frame{ |
| 1340 | offView("offline-sealing", nil), |
| 1341 | offView("offline-sealing", nil), |
| 1342 | offView("offline-sealed", frame{"waiting": proc}), |
| 1343 | offView("offline-sealed", frame{"waiting": proc}), |
| 1344 | offView("offline-sealed", frame{"next": "flip"}), |
| 1345 | }, |
| 1346 | } |
| 1347 | code, out := runOfflineAgainst(t, c, "seal") |
| 1348 | if want := oCopying + "\n" + proc + "\n" + oCopied + "\n"; code != 0 || out != want { |
| 1349 | t.Fatalf("exit %d, output:\n%s\nwant:\n%s", code, out, want) |
| 1350 | } |
| 1351 | if got := c.requests()[0]; !strings.HasPrefix(got, "POST /offline/seal") { |
| 1352 | t.Errorf("first request %q", got) |
| 1353 | } |
| 1354 | } |
| 1355 | |
| 1356 | func TestMoveOfflineSealFails(t *testing.T) { |
| 1357 | item := func(status, detail string) frame { |
| 1358 | return frame{"id": "files", "owner": "seal", "status": status, "detail": detail} |
| 1359 | } |
| 1360 | for _, c := range []struct { |
| 1361 | name string |
| 1362 | extra frame |
| 1363 | say string |
| 1364 | }{ |
| 1365 | {"item", frame{"items": []frame{item("ok", ""), item("failed", "a file over 256 MiB")}}, "The move could not finish a step: a file over 256 MiB. Run carlos move offline status --host acme.shop.example.com for each item."}, |
| 1366 | {"item without detail", frame{"items": []frame{item("failed", "")}}, "The move could not finish a step: files. Run carlos move offline status --host acme.shop.example.com for each item."}, |
| 1367 | {"failed flag and report", frame{"failed": true, "reports": []frame{{"box": "ie-box", "role": "sealer", "phase": "failed", "fresh": true, |
| 1368 | "items": frame{"db": frame{"ok": true, "reason": ""}, "files": frame{"ok": false, "reason": "special file in the archive"}}}}}, |
| 1369 | "The move could not finish a step: special file in the archive. Run carlos move offline status --host acme.shop.example.com for each item."}, |
| 1370 | } { |
| 1371 | t.Run(c.name, func(t *testing.T) { |
| 1372 | f := &fakeOfflineConsole{ |
| 1373 | posts: map[string][]offPost{"/seal": {{202, offView("offline-sealing", nil)}}}, |
| 1374 | gets: []frame{offView("offline-sealing", c.extra)}, |
| 1375 | } |
| 1376 | code, out := runOfflineAgainst(t, f, "seal") |
| 1377 | if want := oCopying + "\n" + c.say + "\n"; code != 4 || out != want { |
| 1378 | t.Fatalf("exit %d, output:\n%s\nwant 4:\n%s", code, out, want) |
| 1379 | } |
| 1380 | }) |
| 1381 | } |
| 1382 | } |
| 1383 | |
| 1384 | func TestMoveOfflineFlip(t *testing.T) { |
| 1385 | c := &fakeOfflineConsole{ |
| 1386 | posts: map[string][]offPost{"/flip": {{202, offView("offline-landing", nil)}}}, |
| 1387 | gets: []frame{ |
| 1388 | offView("offline-landing", nil), |
| 1389 | offView("offline-landed", frame{"waiting": "this move is offline-landed. run carlos move offline status --host acme.shop.example.com to see what it waits for"}), |
| 1390 | offView("offline-landed", frame{"next": "open"}), |
| 1391 | }, |
| 1392 | } |
| 1393 | code, out := runOfflineAgainst(t, c, "flip") |
| 1394 | want := oFlipped + "\nthis move is offline-landed. run carlos move offline status --host acme.shop.example.com to see what it waits for\n" + oLanded + "\n" |
| 1395 | if code != 0 || out != want { |
| 1396 | t.Fatalf("exit %d, output:\n%s\nwant:\n%s", code, out, want) |
| 1397 | } |
| 1398 | } |
| 1399 | |
| 1400 | func TestMoveOfflineFlipFails(t *testing.T) { |
| 1401 | failedItem := frame{"id": "db", "owner": "landing", "status": "failed", "detail": "the listing differs"} |
| 1402 | for _, c := range []struct { |
| 1403 | name string |
| 1404 | extra frame |
| 1405 | say string |
| 1406 | }{ |
| 1407 | {"item", frame{"items": []frame{failedItem}}, "uk could not take acme.shop.example.com: the listing differs. Run carlos move offline abort --host acme.shop.example.com to move it back."}, |
| 1408 | {"failed flag, check in the report", frame{"failed": true, "reports": []frame{{"box": "uk-box", "role": "target", "phase": "failed", "fresh": true, |
| 1409 | "items": frame{"env-pin": frame{"ok": false, "reason": "the app settings differ"}}}}}, |
| 1410 | "uk could not take acme.shop.example.com: the app settings differ. Run carlos move offline abort --host acme.shop.example.com to move it back."}, |
| 1411 | {"failed flag only", frame{"failed": true}, "uk could not take acme.shop.example.com: failed. Run carlos move offline abort --host acme.shop.example.com to move it back."}, |
| 1412 | } { |
| 1413 | t.Run(c.name, func(t *testing.T) { |
| 1414 | f := &fakeOfflineConsole{ |
| 1415 | posts: map[string][]offPost{"/flip": {{202, offView("offline-landing", nil)}}}, |
| 1416 | gets: []frame{offView("offline-landing", nil), offView("offline-landing", c.extra)}, |
| 1417 | } |
| 1418 | code, out := runOfflineAgainst(t, f, "flip") |
| 1419 | if want := oFlipped + "\n" + c.say + "\n"; code != 4 || out != want { |
| 1420 | t.Fatalf("exit %d, output:\n%s\nwant 4:\n%s", code, out, want) |
| 1421 | } |
| 1422 | if f.posted("/flip") != 1 { |
| 1423 | t.Error("flip posted more than once") |
| 1424 | } |
| 1425 | }) |
| 1426 | } |
| 1427 | } |
| 1428 | |
| 1429 | func TestMoveOfflineOpen(t *testing.T) { |
| 1430 | src := func(phase string) frame { return offReport("ie-box", "source", phase, true) } |
| 1431 | tgt := func(phase string) frame { return offReport("uk-box", "target", phase, true) } |
| 1432 | c := &fakeOfflineConsole{ |
| 1433 | posts: map[string][]offPost{"/open": {{202, offView("offline-open", nil)}}}, |
| 1434 | gets: []frame{ |
| 1435 | offView("offline-open", frame{"reports": []frame{src("stopped"), tgt("landed")}}), |
| 1436 | offView("offline-open", frame{"reports": []frame{src("stopped"), tgt("landed")}}), |
| 1437 | offView("offline-open", frame{"reports": []frame{src("stopped"), tgt("opened")}}), |
| 1438 | offView("offline-open", frame{"next": "finish", "reports": []frame{src("parked"), tgt("opened")}}), |
| 1439 | }, |
| 1440 | } |
| 1441 | code, out := runOfflineAgainst(t, c, "open") |
| 1442 | // The served line waits for the target's opened report. |
| 1443 | want := "Waiting for ie-box to clean up acme.shop.example.com.\n" + oServed + "\n" |
| 1444 | if code != 0 || out != want { |
| 1445 | t.Fatalf("exit %d, output:\n%s\nwant:\n%s", code, out, want) |
| 1446 | } |
| 1447 | // Or for finish to be next, when the target's report is not in the view. |
| 1448 | c = &fakeOfflineConsole{ |
| 1449 | posts: map[string][]offPost{"/open": {{202, offView("offline-open", nil)}}}, |
| 1450 | gets: []frame{offView("offline-open", frame{"next": "finish"})}, |
| 1451 | } |
| 1452 | if code, out := runOfflineAgainst(t, c, "open"); code != 0 || out != "Waiting for ie-box to clean up acme.shop.example.com.\n"+oServed+"\n" { |
| 1453 | t.Fatalf("exit %d, output:\n%s", code, out) |
| 1454 | } |
| 1455 | } |
| 1456 | |
| 1457 | // finish posts twice for a move that was open: the first starts the clean-up |
| 1458 | // and answers offline-finishing; once the boxes are gone and the tombstone is |
| 1459 | // lifted, the status shows finish again and the second post ends the move. |
| 1460 | func TestMoveOfflineFinishPostsTwice(t *testing.T) { |
| 1461 | c := &fakeOfflineConsole{ |
| 1462 | posts: map[string][]offPost{"/finish": { |
| 1463 | {202, offView("offline-finishing", frame{"reports": []frame{offReport("ie-box", "source", "parked", true)}})}, |
| 1464 | {202, offDone("done", 1800)}, |
| 1465 | }}, |
| 1466 | gets: []frame{ |
| 1467 | offView("offline-open", frame{"next": "finish"}), |
| 1468 | offView("offline-finishing", frame{"reports": []frame{offReport("ie-box", "source", "parked", true)}}), |
| 1469 | offView("offline-finishing", frame{"reports": []frame{offReport("ie-box", "source", "gone", true), offReport("ie-box", "sealer", "sealed", true)}}), |
| 1470 | offView("offline-finishing", frame{"next": "finish", "reports": []frame{offReport("ie-box", "source", "gone", true), offReport("ie-box", "sealer", "lifted", true)}}), |
| 1471 | }, |
| 1472 | } |
| 1473 | code, out := runOfflineAgainst(t, c, "finish") |
| 1474 | want := "Waiting for ie-box to clean up acme.shop.example.com.\nMoved. acme.shop.example.com is served from uk. It was offline for 1800 seconds.\n" |
| 1475 | if code != 0 || out != want { |
| 1476 | t.Fatalf("exit %d, output:\n%s\nwant:\n%s", code, out, want) |
| 1477 | } |
| 1478 | if n := c.posted("/finish"); n != 2 { |
| 1479 | t.Fatalf("finish posted %d times, want 2:\n%v", n, c.requests()) |
| 1480 | } |
| 1481 | } |
| 1482 | |
| 1483 | // A finish before the boxes have parked does not ask the door early. |
| 1484 | func TestMoveOfflineFinishWaitsBeforeThePost(t *testing.T) { |
| 1485 | c := &fakeOfflineConsole{ |
| 1486 | posts: map[string][]offPost{"/finish": {{202, offDone("done", 12)}}}, |
| 1487 | gets: []frame{ |
| 1488 | offView("offline-open", frame{"reports": []frame{offReport("ie-box", "source", "stopped", true)}}), |
| 1489 | offView("offline-open", frame{"next": "finish"}), |
| 1490 | }, |
| 1491 | } |
| 1492 | code, out := runOfflineAgainst(t, c, "finish") |
| 1493 | if code != 0 || !strings.HasSuffix(out, "It was offline for 12 seconds.\n") || !strings.HasPrefix(out, "Waiting for ie-box to clean up acme.shop.example.com.\n") { |
| 1494 | t.Fatalf("exit %d:\n%s", code, out) |
| 1495 | } |
| 1496 | reqs := c.requests() |
| 1497 | post := -1 |
| 1498 | for i, r := range reqs { |
| 1499 | if strings.HasPrefix(r, "POST") { |
| 1500 | post = i |
| 1501 | } |
| 1502 | } |
| 1503 | if post < 2 { |
| 1504 | t.Fatalf("the door was posted before the status showed finish:\n%v", reqs) |
| 1505 | } |
| 1506 | } |
| 1507 | |
| 1508 | func TestMoveOfflineFinishAfterAbort(t *testing.T) { |
| 1509 | c := &fakeOfflineConsole{ |
| 1510 | posts: map[string][]offPost{"/finish": {{202, offDone("aborted", 0)}}}, |
| 1511 | gets: []frame{offView("offline-reverting", frame{"next": "finish"})}, |
| 1512 | } |
| 1513 | code, out := runOfflineAgainst(t, c, "finish") |
| 1514 | if want := "acme.shop.example.com is back in ie and serving. Nothing was lost.\n"; code != 3 || out != want { |
| 1515 | t.Fatalf("exit %d, output:\n%s\nwant 3:\n%s", code, out, want) |
| 1516 | } |
| 1517 | if c.posted("/finish") != 1 { |
| 1518 | t.Error("finish after an abort posted more than once") |
| 1519 | } |
| 1520 | } |
| 1521 | |
| 1522 | func TestMoveOfflineFinishRefusedAndOutcomeNotRecorded(t *testing.T) { |
| 1523 | // Nothing in flight: the door's own sentence, exit 5. |
| 1524 | c := &fakeOfflineConsole{ |
| 1525 | posts: map[string][]offPost{"/finish": {{409, frame{"error": "acme.shop.example.com has no offline move in flight"}}}}, |
| 1526 | gets: []frame{{"host": oHost, "state": "none"}}, |
| 1527 | } |
| 1528 | if code, out := runOfflineAgainst(t, c, "finish"); code != 5 || out != "acme.shop.example.com has no offline move in flight\n" { |
| 1529 | t.Fatalf("exit %d:\n%s", code, out) |
| 1530 | } |
| 1531 | // An outcome that is another move's is not this move's. |
| 1532 | c = &fakeOfflineConsole{ |
| 1533 | posts: map[string][]offPost{"/finish": {{202, frame{"host": oHost, "state": "none", "last": frame{"id": "mv_other", "outcome": "done"}}}}}, |
| 1534 | gets: []frame{offView("offline-reverting", frame{"next": "finish"})}, |
| 1535 | } |
| 1536 | if code, out := runOfflineAgainst(t, c, "finish"); code != 4 || out != "This move's outcome is not recorded.\n" { |
| 1537 | t.Fatalf("exit %d:\n%s", code, out) |
| 1538 | } |
| 1539 | } |
| 1540 | |
| 1541 | func TestMoveOfflineAbort(t *testing.T) { |
| 1542 | c := &fakeOfflineConsole{ |
| 1543 | posts: map[string][]offPost{"/abort": {{202, offView("offline-reverting", nil)}}}, |
| 1544 | gets: []frame{ |
| 1545 | offView("offline-reverting", frame{"reports": []frame{offReport("ie-box", "source", "stopped", true)}}), |
| 1546 | offView("offline-reverting", frame{"next": "finish", "reports": []frame{offReport("ie-box", "source", "reverted", true)}}), |
| 1547 | }, |
| 1548 | } |
| 1549 | code, out := runOfflineAgainst(t, c, "abort") |
| 1550 | want := oBack + "\nWaiting for ie-box to clean up acme.shop.example.com.\n" + oPutBack + "\n" |
| 1551 | if code != 3 || out != want { |
| 1552 | t.Fatalf("exit %d, output:\n%s\nwant 3:\n%s", code, out, want) |
| 1553 | } |
| 1554 | if c.posted("/finish") != 0 { |
| 1555 | t.Error("abort ran finish by itself") |
| 1556 | } |
| 1557 | // Once open, the door refuses and the CLI says so as sent. |
| 1558 | c = &fakeOfflineConsole{posts: map[string][]offPost{"/abort": {{409, frame{"error": "acme.shop.example.com is already served from uk. move it back with a new move"}}}}} |
| 1559 | if code, out := runOfflineAgainst(t, c, "abort"); code != 5 || out != "acme.shop.example.com is already served from uk. move it back with a new move\n" { |
| 1560 | t.Fatalf("exit %d:\n%s", code, out) |
| 1561 | } |
| 1562 | } |
| 1563 | |
| 1564 | // A verb whose step is overtaken (another terminal aborted) says where the |
| 1565 | // move stands and leaves the decision to the operator. |
| 1566 | func TestMoveOfflineOvertaken(t *testing.T) { |
| 1567 | c := &fakeOfflineConsole{ |
| 1568 | posts: map[string][]offPost{"/seal": {{202, offView("offline-sealing", nil)}}}, |
| 1569 | gets: []frame{offView("offline-reverting", nil)}, |
| 1570 | } |
| 1571 | code, out := runOfflineAgainst(t, c, "seal") |
| 1572 | if want := oCopying + "\n" + oStarting + "\n" + oBack + "\nWaiting for ie-box to clean up acme.shop.example.com.\n"; code != 4 || out != want { |
| 1573 | t.Fatalf("exit %d, output:\n%s\nwant 4:\n%s", code, out, want) |
| 1574 | } |
| 1575 | } |
| 1576 | |
| 1577 | func TestMoveOfflineStatus(t *testing.T) { |
| 1578 | c := &fakeOfflineConsole{gets: []frame{offView("offline-sealed", frame{ |
| 1579 | "next": "flip", |
| 1580 | "items": []frame{ |
| 1581 | {"id": "db", "owner": "landing", "status": "waiting"}, |
| 1582 | {"id": "files", "owner": "seal", "status": "ok", "detail": "12 files"}, |
| 1583 | }, |
| 1584 | "reports": []frame{offReport("ie-box", "source", "stopped", true), offReport("ie-box", "sealer", "sealed", false)}, |
| 1585 | })}} |
| 1586 | code, out := runOfflineAgainst(t, c, "status") |
| 1587 | want := oStarting + "\n" + oCopied + "\n" + |
| 1588 | "Item db is waiting.\nItem files is ok: 12 files.\n" + |
| 1589 | "ie-box is stopped as the source.\nie-box is sealed as the sealer. Its report is not current.\n" |
| 1590 | if code != 0 || out != want { |
| 1591 | t.Fatalf("exit %d, output:\n%s\nwant:\n%s", code, out, want) |
| 1592 | } |
| 1593 | if reqs := c.requests(); len(reqs) != 1 || !strings.HasPrefix(reqs[0], "GET /offline") { |
| 1594 | t.Fatalf("status sent %v", reqs) |
| 1595 | } |
| 1596 | |
| 1597 | // A failed item is the operator's: exit 4. |
| 1598 | c = &fakeOfflineConsole{gets: []frame{offView("offline-landing", frame{"items": []frame{{"id": "db", "owner": "landing", "status": "failed", "detail": "digest differs"}}})}} |
| 1599 | code, out = runOfflineAgainst(t, c, "status") |
| 1600 | wantF := oStarting + "\nuk could not take acme.shop.example.com: digest differs. Run carlos move offline abort --host acme.shop.example.com to move it back.\nItem db is failed: digest differs.\n" |
| 1601 | if code != 4 || out != wantF { |
| 1602 | t.Fatalf("exit %d, output:\n%s\nwant 4:\n%s", code, out, wantF) |
| 1603 | } |
| 1604 | |
| 1605 | // Waiting states say the console's sentence. |
| 1606 | c = &fakeOfflineConsole{gets: []frame{offView("offline-stopping", frame{"waiting": oWaitBox})}} |
| 1607 | if code, out := runOfflineAgainst(t, c, "status"); code != 0 || out != oStarting+"\n"+oWaitBox+"\n" { |
| 1608 | t.Fatalf("exit %d:\n%s", code, out) |
| 1609 | } |
| 1610 | |
| 1611 | // Nothing in flight. |
| 1612 | c = &fakeOfflineConsole{gets: []frame{{"host": oHost, "state": "none"}}} |
| 1613 | if code, out := runOfflineAgainst(t, c, "status"); code != 0 || out != "No move is in flight for acme.shop.example.com.\n" { |
| 1614 | t.Fatalf("exit %d:\n%s", code, out) |
| 1615 | } |
| 1616 | c = &fakeOfflineConsole{gets: []frame{offDone("done", 5)}} |
| 1617 | if code, out := runOfflineAgainst(t, c, "status"); code != 0 || out != "Move "+tMoveID+" of acme.shop.example.com is done.\n" { |
| 1618 | t.Fatalf("exit %d:\n%s", code, out) |
| 1619 | } |
| 1620 | } |
| 1621 | |
| 1622 | func TestMoveOfflineWaitingAfterOpenAndAbort(t *testing.T) { |
| 1623 | // Status of an open move names the boxes still cleaning up. |
| 1624 | c := &fakeOfflineConsole{gets: []frame{offView("offline-finishing", frame{ |
| 1625 | "reports": []frame{offReport("ie-box", "source", "gone", true), offReport("ie-box", "sealer", "sealed", true)}})}} |
| 1626 | code, out := runOfflineAgainst(t, c, "status") |
| 1627 | want := oStarting + "\n" + oServed + "\nWaiting for ie-box to clean up acme.shop.example.com.\n" + |
| 1628 | "ie-box is gone as the source.\nie-box is sealed as the sealer.\n" |
| 1629 | if code != 0 || out != want { |
| 1630 | t.Fatalf("exit %d, output:\n%s\nwant:\n%s", code, out, want) |
| 1631 | } |
| 1632 | } |
| 1633 | |
| 1634 | func TestMoveOfflineNeedsNoPhase1AndRefusesRoot(t *testing.T) { |
| 1635 | // The Phase 1 door is never read: the fake 404s everything outside |
| 1636 | // /move/offline, and the verb still works (the other tests). Root is |
| 1637 | // refused as for every carlos move. |
| 1638 | c := &fakeOfflineConsole{gets: []frame{{"host": oHost, "state": "none"}}} |
| 1639 | ts := httptest.NewServer(c) |
| 1640 | defer ts.Close() |
| 1641 | var out bytes.Buffer |
| 1642 | deps := moveDeps{session: func(string) (*apiSession, error) { t.Error("signed in as root"); return nil, nil }, euid: func() int { return 0 }, poll: time.Millisecond} |
| 1643 | if code := moveMain(context.Background(), []string{"offline", "status", "--host", oHost, "--app", "shop"}, &out, deps); code != 5 { |
| 1644 | t.Fatalf("exit %d:\n%s", code, out.String()) |
| 1645 | } |
| 1646 | wantLines(t, out.String(), "Run carlos move as yourself, not as root. It uses your own credentials.") |
| 1647 | if len(c.requests()) != 0 { |
| 1648 | t.Error("a root run reached the console") |
| 1649 | } |
| 1650 | } |
| 1651 | |
| 1652 | func TestMoveOfflineRidesOutARateLimit(t *testing.T) { |
| 1653 | c := &fakeOfflineConsole{ |
| 1654 | posts: map[string][]offPost{"/seal": {{429, frame{"error": "too many instance requests. wait a minute and try again"}}, {202, offView("offline-sealing", nil)}}}, |
| 1655 | gets: []frame{offView("offline-sealed", frame{"next": "flip"})}, |
| 1656 | } |
| 1657 | code, out := runOfflineAgainst(t, c, "seal") |
| 1658 | if want := "The console is limiting requests. Checking every 10 seconds.\n" + oCopying + "\n" + oCopied + "\n"; code != 0 || out != want { |
| 1659 | t.Fatalf("exit %d, output:\n%s\nwant:\n%s", code, out, want) |
| 1660 | } |
| 1661 | if c.posted("/seal") != 2 { |
| 1662 | t.Errorf("seal posted %d times, want 2", c.posted("/seal")) |
| 1663 | } |
| 1664 | } |
| 1665 | |
| 1666 | // The page's paragraph is the copy record's rows MV-O32 and MV-O33, word for |
| 1667 | // word, and its examples parse as the seven offline verbs. |
| 1668 | func TestMoveDocsOfflineParagraph(t *testing.T) { |
| 1669 | raw, err := os.ReadFile("docs/site/cli.md") |
| 1670 | if err != nil { |
| 1671 | t.Fatal(err) |
| 1672 | } |
| 1673 | page := string(raw) |
| 1674 | rec, err := os.ReadFile("docs/superpowers/plans/2026-10-06-moves-phase1-copy.md") |
| 1675 | if err != nil { |
| 1676 | t.Fatal(err) |
| 1677 | } |
| 1678 | for _, id := range []string{"MV-O32", "MV-O33"} { |
| 1679 | text := "" |
| 1680 | for _, line := range strings.Split(string(rec), "\n") { |
| 1681 | if cells := strings.Split(line, "|"); len(cells) >= 5 && strings.TrimSpace(cells[1]) == id { |
| 1682 | text = strings.TrimSpace(cells[3]) |
| 1683 | } |
| 1684 | } |
| 1685 | if text == "" { |
| 1686 | t.Fatalf("the copy record has no row %s", id) |
| 1687 | } |
| 1688 | if !strings.Contains(page, "\n"+text+"\n") { |
| 1689 | t.Errorf("the page does not carry %s word for word: %q", id, text) |
| 1690 | } |
| 1691 | } |
| 1692 | seen := map[string]bool{} |
| 1693 | for _, line := range strings.Split(page, "\n") { |
| 1694 | fields := strings.Fields(line) |
| 1695 | if len(fields) < 4 || fields[0] != "carlos" || fields[1] != "move" || fields[2] != "offline" { |
| 1696 | continue |
| 1697 | } |
| 1698 | args := fields[2:] |
| 1699 | for i, a := range args { |
| 1700 | if strings.HasPrefix(a, "<") { |
| 1701 | args[i] = "x" |
| 1702 | } |
| 1703 | } |
| 1704 | got, err := parseMoveArgs(args, io.Discard) |
| 1705 | if err != nil { |
| 1706 | t.Errorf("the page's example %q does not parse: %v", line, err) |
| 1707 | continue |
| 1708 | } |
| 1709 | seen[got.offline] = true |
| 1710 | } |
| 1711 | for _, v := range offlineVerbs { |
| 1712 | if !seen[v] { |
| 1713 | t.Errorf("the page has no example of carlos move offline %s", v) |
| 1714 | } |
| 1715 | } |
| 1716 | } |
| 1717 | |
| 1718 | // status of a failed move: exit 4 on the flag alone, a failed seal is not |
| 1719 | // headlined as progress, and the role the view marks decides MV-O6 or MV-O9. |
| 1720 | func TestMoveOfflineStatusFailed(t *testing.T) { |
| 1721 | sealerFailed := frame{"box": "ie-box", "role": "sealer", "phase": "failed", "fresh": true, |
| 1722 | "items": frame{"files": frame{"ok": false, "reason": "special file in the archive"}}} |
| 1723 | targetFailed := frame{"box": "uk-box", "role": "target", "phase": "failed", "fresh": true} |
| 1724 | for _, c := range []struct { |
| 1725 | name string |
| 1726 | state string |
| 1727 | extra frame |
| 1728 | say string |
| 1729 | }{ |
| 1730 | {"failed seal, flag only", "offline-sealing", frame{"failed": true}, |
| 1731 | "The move could not finish a step: failed. Run carlos move offline status --host acme.shop.example.com for each item."}, |
| 1732 | {"failed seal, sealer report", "offline-sealing", frame{"failed": true, "reports": []frame{sealerFailed}}, |
| 1733 | "The move could not finish a step: special file in the archive. Run carlos move offline status --host acme.shop.example.com for each item."}, |
| 1734 | {"failed landing, target report", "offline-landed", frame{"failed": true, "reports": []frame{targetFailed}}, |
| 1735 | "uk could not take acme.shop.example.com: failed. Run carlos move offline abort --host acme.shop.example.com to move it back."}, |
| 1736 | {"failed target before the flip", "offline-stopping", frame{"failed": true, "reports": []frame{targetFailed}}, |
| 1737 | "uk could not take acme.shop.example.com: failed. Run carlos move offline abort --host acme.shop.example.com to move it back."}, |
| 1738 | } { |
| 1739 | t.Run(c.name, func(t *testing.T) { |
| 1740 | f := &fakeOfflineConsole{gets: []frame{offView(c.state, c.extra)}} |
| 1741 | code, out := runOfflineAgainst(t, f, "status") |
| 1742 | if code != 4 { |
| 1743 | t.Errorf("exit %d, want 4:\n%s", code, out) |
| 1744 | } |
| 1745 | if !strings.HasPrefix(out, oStarting+"\n"+c.say+"\n") { |
| 1746 | t.Errorf("output:\n%s\nwant the failure line after the headline: %s", out, c.say) |
| 1747 | } |
| 1748 | wantNoLine(t, out, "Copying") |
| 1749 | }) |
| 1750 | } |
| 1751 | } |
| 1752 | |
| 1753 | func TestMoveOfflineStatusShowsPathsLeft(t *testing.T) { |
| 1754 | c := &fakeOfflineConsole{gets: []frame{offView("offline-finishing", frame{ |
| 1755 | "next": "finish", |
| 1756 | "left": []frame{{"box": "ie-box", "path": "/opt/carlos/live/acme.shop.example.com", "reason": "a root-owned directory"}}, |
| 1757 | })}} |
| 1758 | code, out := runOfflineAgainst(t, c, "status") |
| 1759 | if code != 0 { |
| 1760 | t.Fatalf("exit %d:\n%s", code, out) |
| 1761 | } |
| 1762 | wantLines(t, out, "ie-box left /opt/carlos/live/acme.shop.example.com in place: a root-owned directory.") |
| 1763 | } |
| 1764 | |
| 1765 | // A failed report seen after the open is not silent: status says it and exits |
| 1766 | // 4, and open stops polling with the same line. |
| 1767 | func TestMoveOfflineFailedReportAfterTheOpen(t *testing.T) { |
| 1768 | failedRep := func(box, role, check string) frame { |
| 1769 | f := offReport(box, role, "failed", true) |
| 1770 | f["items"] = frame{check: frame{"ok": false, "reason": "the " + check + " check failed"}} |
| 1771 | return f |
| 1772 | } |
| 1773 | for _, c := range []struct { |
| 1774 | name string |
| 1775 | state string |
| 1776 | rep frame |
| 1777 | say string |
| 1778 | }{ |
| 1779 | // Abort refuses once open, so a target failure there says MV-O9. |
| 1780 | {"target, open", "offline-open", failedRep("uk-box", "target", "config"), |
| 1781 | "The move could not finish a step: the config check failed. Run carlos move offline status --host acme.shop.example.com for each item."}, |
| 1782 | {"sealer, finishing", "offline-finishing", failedRep("ie-box", "sealer", "lease"), |
| 1783 | "The move could not finish a step: the lease check failed. Run carlos move offline status --host acme.shop.example.com for each item."}, |
| 1784 | {"source, open", "offline-open", failedRep("ie-box", "source", "warm-cache"), |
| 1785 | "ie-box could not clean up acme.shop.example.com: the warm-cache check failed. Run carlos move offline status --host acme.shop.example.com for each item."}, |
| 1786 | {"source, reverting", "offline-reverting", failedRep("ie-box", "source", "warm-cache"), |
| 1787 | "ie-box could not clean up acme.shop.example.com: the warm-cache check failed. Run carlos move offline status --host acme.shop.example.com for each item."}, |
| 1788 | } { |
| 1789 | t.Run(c.name, func(t *testing.T) { |
| 1790 | f := &fakeOfflineConsole{gets: []frame{offView(c.state, frame{"reports": []frame{c.rep}})}} |
| 1791 | code, out := runOfflineAgainst(t, f, "status") |
| 1792 | if code != 4 || !strings.HasPrefix(out, oStarting+"\n"+c.say+"\n") { |
| 1793 | t.Errorf("status exit %d, output:\n%s\nwant 4 and %q after the headline", code, out, c.say) |
| 1794 | } |
| 1795 | if c.state == "offline-open" { |
| 1796 | f := &fakeOfflineConsole{ |
| 1797 | posts: map[string][]offPost{"/open": {{202, offView("offline-open", nil)}}}, |
| 1798 | gets: []frame{offView(c.state, frame{"reports": []frame{c.rep}})}, |
| 1799 | } |
| 1800 | code, out := runOfflineAgainst(t, f, "open") |
| 1801 | if code != 4 || !strings.HasSuffix(out, c.say+"\n") { |
| 1802 | t.Errorf("open exit %d, output:\n%s\nwant 4 ending in %q", code, out, c.say) |
| 1803 | } |
| 1804 | } |
| 1805 | }) |
| 1806 | } |
| 1807 | // A target or sealer that failed before an abort is being put back: a |
| 1808 | // reverting move shows only a failed source box. |
| 1809 | stale := offReport("uk-box", "target", "failed", true) |
| 1810 | f := &fakeOfflineConsole{gets: []frame{offView("offline-reverting", frame{"next": "finish", "reports": []frame{stale}})}} |
| 1811 | if code, out := runOfflineAgainst(t, f, "status"); code != 0 || strings.Contains(out, "could not") { |
| 1812 | t.Errorf("a reverting move with an old failed target: exit %d:\n%s", code, out) |
| 1813 | } |
| 1814 | } |
| 1815 | |
| 1816 | // ---- carlos move offline stage ---- |
| 1817 | |
| 1818 | const ( |
| 1819 | oStaging = "Staging acme.shop.example.com in uk. acme.shop.example.com keeps serving while the data is copied." |
| 1820 | oReady = "acme.shop.example.com keeps serving. Run carlos move offline start --host acme.shop.example.com --to uk when you are ready to take it offline." |
| 1821 | oUnstage = "Removing the staged copy of acme.shop.example.com from uk." |
| 1822 | oUnstaged = "The staged copy of acme.shop.example.com is removed. Run carlos move offline finish --host acme.shop.example.com to end the move." |
| 1823 | ) |
| 1824 | |
| 1825 | // stageView is a staging move's view with the stager's last round, which |
| 1826 | // answered the request at answered (0: no round yet). |
| 1827 | func stageView(stagedAt, answered int64, phase string, extra frame) frame { |
| 1828 | f := frame{"staged_at": stagedAt, "never_offline": true, "sealer": "", "boxes": nil, "can_abort": true} |
| 1829 | if answered != 0 { |
| 1830 | f["stage"] = frame{"box": "ie-box", "phase": phase, "at": int64(1792573200), "replica_objects": 305, "replica_bytes": 1_580_000_000, |
| 1831 | "staged_objects": 305, "staged_bytes": 1_580_000_000, "round_objects": 305, "round_bytes": 1_580_000_000, "request": answered, |
| 1832 | "error": "mirror: copy instances/x: boom"} |
| 1833 | } |
| 1834 | for k, v := range extra { |
| 1835 | f[k] = v |
| 1836 | } |
| 1837 | return offView("offline-staging", f) |
| 1838 | } |
| 1839 | |
| 1840 | // stage posts and then waits for the round that answers its own request: |
| 1841 | // not the earlier round the answer already showed. |
| 1842 | func TestMoveOfflineStage(t *testing.T) { |
| 1843 | c := &fakeOfflineConsole{ |
| 1844 | posts: map[string][]offPost{"/stage": {{202, stageView(200, 100, "staged", nil)}}}, |
| 1845 | gets: []frame{ |
| 1846 | stageView(200, 100, "staged", nil), |
| 1847 | stageView(200, 200, "staged", frame{"next": "start"}), |
| 1848 | }, |
| 1849 | } |
| 1850 | code, out := runOfflineAgainst(t, c, "stage", "--to", "uk") |
| 1851 | want := oStaging + "\nStaged 305 of 305 objects for acme.shop.example.com in uk (1580.0 of 1580.0 MB) at 2026-10-21 09:00 UTC.\n" + oReady + "\n" |
| 1852 | if code != 0 || out != want { |
| 1853 | t.Fatalf("exit %d, output:\n%s\nwant 0:\n%s", code, out, want) |
| 1854 | } |
| 1855 | if got := c.requests(); len(got) == 0 || got[0] != `POST /offline/stage {"region":"uk"}` { |
| 1856 | t.Fatalf("requests %q", got) |
| 1857 | } |
| 1858 | if c.posted("") != 0 { |
| 1859 | t.Error("stage started the move by itself") |
| 1860 | } |
| 1861 | // A failed round: the box's reason, the way on, exit 4. |
| 1862 | c = &fakeOfflineConsole{ |
| 1863 | posts: map[string][]offPost{"/stage": {{202, stageView(300, 0, "", nil)}}}, |
| 1864 | gets: []frame{stageView(300, 300, "failed", nil)}, |
| 1865 | } |
| 1866 | code, out = runOfflineAgainst(t, c, "stage", "--to", "uk") |
| 1867 | want = oStaging + "\nThe last staging round failed: mirror: copy instances/x: boom. Run carlos move offline stage --host acme.shop.example.com --to uk to try again.\n" |
| 1868 | if code != 4 || out != want { |
| 1869 | t.Fatalf("exit %d, output:\n%s\nwant 4:\n%s", code, out, want) |
| 1870 | } |
| 1871 | // A refusal is the console's sentence, as sent. |
| 1872 | refusal := "acme.shop.example.com is staged for uk. abort the move to move it to another region" |
| 1873 | c = &fakeOfflineConsole{posts: map[string][]offPost{"/stage": {{409, frame{"error": refusal}}}}} |
| 1874 | if code, out := runOfflineAgainst(t, c, "stage", "--to", "us"); code != 5 || out != refusal+"\n" { |
| 1875 | t.Fatalf("exit %d:\n%s", code, out) |
| 1876 | } |
| 1877 | } |
| 1878 | |
| 1879 | // status while staging: the stage's line instead of the manifest's items. |
| 1880 | func TestMoveOfflineStatusWhileStaging(t *testing.T) { |
| 1881 | c := &fakeOfflineConsole{gets: []frame{stageView(200, 0, "", nil)}} |
| 1882 | code, out := runOfflineAgainst(t, c, "status") |
| 1883 | if want := oStaging + "\nWaiting for the first staging round of acme.shop.example.com.\n"; code != 0 || out != want { |
| 1884 | t.Fatalf("exit %d, output:\n%s\nwant 0:\n%s", code, out, want) |
| 1885 | } |
| 1886 | c = &fakeOfflineConsole{gets: []frame{stageView(200, 200, "staged", frame{"next": "start"})}} |
| 1887 | code, out = runOfflineAgainst(t, c, "status") |
| 1888 | want := oStaging + "\nStaged 305 of 305 objects for acme.shop.example.com in uk (1580.0 of 1580.0 MB) at 2026-10-21 09:00 UTC.\n" + oReady + "\n" |
| 1889 | if code != 0 || out != want { |
| 1890 | t.Fatalf("exit %d, output:\n%s\nwant 0:\n%s", code, out, want) |
| 1891 | } |
| 1892 | c = &fakeOfflineConsole{gets: []frame{stageView(200, 200, "failed", nil)}} |
| 1893 | if code, _ := runOfflineAgainst(t, c, "status"); code != 4 { |
| 1894 | t.Fatalf("status of a failed round exit %d, want 4", code) |
| 1895 | } |
| 1896 | } |
| 1897 | |
| 1898 | // abort of a move still staging: nothing to move back, only the staged copy. |
| 1899 | func TestMoveOfflineAbortStaged(t *testing.T) { |
| 1900 | c := &fakeOfflineConsole{ |
| 1901 | posts: map[string][]offPost{"/abort": {{202, offView("offline-reverting", frame{"never_offline": true, "sealer": ""})}}}, |
| 1902 | gets: []frame{ |
| 1903 | offView("offline-reverting", frame{"never_offline": true, "sealer": ""}), |
| 1904 | offView("offline-reverting", frame{"never_offline": true, "sealer": "", "next": "finish"}), |
| 1905 | }, |
| 1906 | } |
| 1907 | code, out := runOfflineAgainst(t, c, "abort") |
| 1908 | if want := oUnstage + "\n" + oUnstaged + "\n"; code != 3 || out != want { |
| 1909 | t.Fatalf("exit %d, output:\n%s\nwant 3:\n%s", code, out, want) |
| 1910 | } |
| 1911 | if c.posted("/finish") != 0 { |
| 1912 | t.Error("abort ran finish by itself") |
| 1913 | } |
| 1914 | } |
| 1915 | |
amadan