| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "crypto/ecdh" |
| 7 | "encoding/base64" |
| 8 | "encoding/json" |
| 9 | "net/http" |
| 10 | "net/http/httptest" |
| 11 | "path/filepath" |
| 12 | "strings" |
| 13 | "testing" |
| 14 | "time" |
| 15 | |
| 16 | "github.com/carlosframework/platform/internal/appconfig" |
| 17 | "github.com/carlosframework/platform/internal/console" |
| 18 | "github.com/carlosframework/platform/internal/moves" |
| 19 | "github.com/carlosframework/platform/internal/provision" |
| 20 | "github.com/carlosframework/platform/internal/release" |
| 21 | "github.com/carlosframework/platform/internal/status" |
| 22 | ) |
| 23 | |
| 24 | // offlineE2E is a real console.Server with one hibernating instance of the |
| 25 | // app shop in ie, an operator's CLI login, and the ie and uk boxes |
| 26 | // published with caps. run is moveMain for `carlos move offline <args>` |
| 27 | // against it; put writes a box's report where the real box writes it. |
| 28 | type offlineE2E struct { |
| 29 | host string |
| 30 | rel release.Store |
| 31 | run func(c context.Context, args ...string) (int, string) |
| 32 | put func(id, box, role, phase string, edit func(*moves.OfflineReport)) |
| 33 | } |
| 34 | |
| 35 | func newOfflineE2E(t *testing.T, caps []string) *offlineE2E { |
| 36 | const ( |
| 37 | host = "acme.shop.example.com" |
| 38 | addr = "paul@keymail.dev" |
| 39 | ) |
| 40 | tmp := t.TempDir() |
| 41 | rel := release.NewDirStore(filepath.Join(tmp, "store")) |
| 42 | seed, pubB64, err := release.GenerateKey() |
| 43 | if err != nil { |
| 44 | t.Fatal(err) |
| 45 | } |
| 46 | priv, err := release.ParseSigningKey(seed, "e2e") |
| 47 | if err != nil { |
| 48 | t.Fatal(err) |
| 49 | } |
| 50 | pub, err := release.ParsePublicKey(pubB64, "e2e") |
| 51 | if err != nil { |
| 52 | t.Fatal(err) |
| 53 | } |
| 54 | cst, err := console.Open(filepath.Join(tmp, "console.db")) |
| 55 | if err != nil { |
| 56 | t.Fatal(err) |
| 57 | } |
| 58 | t.Cleanup(func() { cst.Close() }) |
| 59 | if err := cst.SetOperator(addr, true); err != nil { |
| 60 | t.Fatal(err) |
| 61 | } |
| 62 | secrets, err := ecdh.P256().NewPrivateKey(bytes.Repeat([]byte{1}, 32)) |
| 63 | if err != nil { |
| 64 | t.Fatal(err) |
| 65 | } |
| 66 | fp := appconfig.KeyFingerprint(secrets.PublicKey()) |
| 67 | |
| 68 | var srv *console.Server |
| 69 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv.ServeHTTP(w, r) })) |
| 70 | t.Cleanup(ts.Close) |
| 71 | srv, err = console.NewServer(console.Config{ |
| 72 | Origin: ts.URL, |
| 73 | Keymail: "http://keymail.invalid", |
| 74 | Registry: filepath.Join(tmp, "registry.db"), |
| 75 | Releases: rel, |
| 76 | ReleasePub: pub, |
| 77 | Key: priv, |
| 78 | AppScheme: "http", |
| 79 | SecretsPub: base64.StdEncoding.EncodeToString(secrets.PublicKey().Bytes()), |
| 80 | }, cst) |
| 81 | if err != nil { |
| 82 | t.Fatal(err) |
| 83 | } |
| 84 | if err := cst.AddMember(console.OpsAccountID, addr, console.RoleOwner, time.Now()); err != nil { |
| 85 | t.Fatal(err) |
| 86 | } |
| 87 | if _, err := cst.CreateAppIn(console.OpsAccountID, "shop", addr, time.Now()); err != nil { |
| 88 | t.Fatal(err) |
| 89 | } |
| 90 | token, _ := e2eDeviceLogin(t, ts.URL, cst, addr, "offline@laptop") |
| 91 | ctx := context.Background() |
| 92 | |
| 93 | // A hibernating instance in ie, and the two regions' boxes published. |
| 94 | if err := provision.PutManifest(ctx, rel, release.OpsAccount, "shop", |
| 95 | provision.Manifest{Schema: 1, App: "shop", Domain: "shop.example.com", Channel: "stable"}); err != nil { |
| 96 | t.Fatal(err) |
| 97 | } |
| 98 | req, _ := http.NewRequest(http.MethodPost, ts.URL+"/api/cli/apps/"+release.OpsAccount+"/shop/instances", |
| 99 | strings.NewReader(`{"host":"`+host+`","region":"ie"}`)) |
| 100 | req.Header.Set("Authorization", "Bearer "+token) |
| 101 | req.Header.Set("Content-Type", "application/json") |
| 102 | res, err := http.DefaultClient.Do(req) |
| 103 | if err != nil { |
| 104 | t.Fatal(err) |
| 105 | } |
| 106 | res.Body.Close() |
| 107 | if res.StatusCode != http.StatusCreated { |
| 108 | t.Fatalf("create the instance = %d", res.StatusCode) |
| 109 | } |
| 110 | for _, b := range []struct{ box, placement, aws, bucket string }{ |
| 111 | {"ie-box", "ie", "eu-west-1", "data-ie"}, |
| 112 | {"uk-box", "uk", "eu-west-2", "data-uk"}, |
| 113 | } { |
| 114 | body, _ := json.Marshal(map[string]any{ |
| 115 | "v": 1, "box": b.box, "box_id": b.box, "eip": "1.2.3.4", "region": b.aws, "placement": b.placement, |
| 116 | "data_bucket": b.bucket, "hibernation_configured": true, "serves_shared_pool": true, |
| 117 | "accepts_inbound": true, "secrets_key_fp": fp, |
| 118 | "capabilities": caps, "hosts": map[string]any{}, |
| 119 | }) |
| 120 | if _, err := rel.Put(ctx, "control/edges/"+b.box+".json", body, ""); err != nil { |
| 121 | t.Fatal(err) |
| 122 | } |
| 123 | if err := status.WriteBox(ctx, rel, status.Box{Label: b.box, GeneratedAt: time.Now(), Metrics: &status.BoxMetrics{}}); err != nil { |
| 124 | t.Fatal(err) |
| 125 | } |
| 126 | } |
| 127 | |
| 128 | run := func(c context.Context, args ...string) (int, string) { |
| 129 | var out bytes.Buffer |
| 130 | deps := moveDeps{ |
| 131 | session: func(string) (*apiSession, error) { |
| 132 | return &apiSession{c: newConsoleClient(ts.URL, token), console: ts.URL, address: addr}, nil |
| 133 | }, |
| 134 | euid: func() int { return 1000 }, |
| 135 | poll: 20 * time.Millisecond, |
| 136 | } |
| 137 | args = append(append([]string{"offline"}, args...), "--host", host, "--account", release.OpsAccount, "--app", "shop") |
| 138 | code := moveMain(c, args, &out, deps) |
| 139 | return code, out.String() |
| 140 | } |
| 141 | put := func(id, box, role, phase string, edit func(*moves.OfflineReport)) { |
| 142 | t.Helper() |
| 143 | r := moves.OfflineReport{V: moves.OfflineReportVersion, Op: id, Host: host, Box: box, Role: role, Phase: phase, |
| 144 | At: time.Now().UTC(), Held: true, Fenced: true, Quiesced: true} |
| 145 | if edit != nil { |
| 146 | edit(&r) |
| 147 | } |
| 148 | body, err := json.Marshal(r) |
| 149 | if err != nil { |
| 150 | t.Fatal(err) |
| 151 | } |
| 152 | key := moves.OfflineReportKey(host, id, role, box) |
| 153 | _, etag, _ := rel.Get(ctx, key) |
| 154 | if _, err := rel.Put(ctx, key, body, etag); err != nil { |
| 155 | t.Fatal(err) |
| 156 | } |
| 157 | } |
| 158 | |
| 159 | return &offlineE2E{host: host, rel: rel, run: run, put: put} |
| 160 | } |
| 161 | |
| 162 | // TestMoveOfflineAgainstTheRealDoors drives `carlos move offline start` and |
| 163 | // `status` through moveMain against a real console.Server: the stub tests |
| 164 | // type the status view by hand, and this one ties the CLI's view structs to |
| 165 | // the JSON the console writes (next, waiting, failed, items, reports, left). |
| 166 | // The boxes are played by reports written where the real boxes write them. |
| 167 | func TestMoveOfflineAgainstTheRealDoors(t *testing.T) { |
| 168 | e := newOfflineE2E(t, []string{"region-ha-1", "offline-move-1"}) |
| 169 | host, rel, run, put := e.host, e.rel, e.run, e.put |
| 170 | ctx := context.Background() |
| 171 | |
| 172 | // start waits for the source box; it runs on while the test plays it. |
| 173 | ctxStart, cancel := context.WithTimeout(ctx, 30*time.Second) |
| 174 | defer cancel() |
| 175 | type result struct { |
| 176 | code int |
| 177 | out string |
| 178 | } |
| 179 | started := make(chan result, 1) |
| 180 | go func() { |
| 181 | code, out := run(ctxStart, "start", "--to", "uk") |
| 182 | started <- result{code, out} |
| 183 | }() |
| 184 | var id string |
| 185 | for deadline := time.Now().Add(20 * time.Second); id == "" && time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) { |
| 186 | if rec, err := provision.GetRecord(ctx, rel, release.OpsAccount, "shop", host); err == nil && rec.Move != nil { |
| 187 | id = rec.Move.ID |
| 188 | } |
| 189 | } |
| 190 | if id == "" { |
| 191 | t.Fatal("start never moved the record") |
| 192 | } |
| 193 | |
| 194 | // status, before the box has reported: the console's own waiting sentence. |
| 195 | code, out := run(ctx, "status") |
| 196 | wantWait := "Waiting for ie-box in ie to confirm it has stopped " + host + ". " + host + " is offline." |
| 197 | wantLines(t, out, "Moving "+host+" from ie to uk.", wantWait, "Item class is ok.", "Item db is waiting.") |
| 198 | if code != 0 { |
| 199 | t.Fatalf("status exit %d:\n%s", code, out) |
| 200 | } |
| 201 | |
| 202 | // The box reports; start's next poll sees next == seal. |
| 203 | put(id, "ie-box", moves.OfflineRoleSource, moves.OfflinePhaseStopped, nil) |
| 204 | select { |
| 205 | case r := <-started: |
| 206 | if r.code != 0 { |
| 207 | t.Fatalf("start exit %d:\n%s", r.code, r.out) |
| 208 | } |
| 209 | wantLines(t, r.out, "Moving "+host+" from ie to uk.", host+" is offline from now until the move opens or is aborted.", |
| 210 | "Every box in ie has stopped "+host+". Run carlos move offline seal --host "+host+" to copy it.") |
| 211 | case <-time.After(20 * time.Second): |
| 212 | t.Fatal("start never finished") |
| 213 | } |
| 214 | code, out = run(ctx, "status") |
| 215 | wantLines(t, out, "Every box in ie has stopped "+host+". Run carlos move offline seal --host "+host+" to copy it.", |
| 216 | "ie-box is stopped as the source.") |
| 217 | if code != 0 { |
| 218 | t.Fatalf("status exit %d:\n%s", code, out) |
| 219 | } |
| 220 | |
| 221 | // A target that says it failed sets the view's failed flag and its check. |
| 222 | put(id, "uk-box", moves.OfflineRoleTarget, moves.OfflinePhaseFailed, func(r *moves.OfflineReport) { |
| 223 | r.Items = map[string]moves.Check{"config": {OK: false, Reason: "the app settings differ"}} |
| 224 | }) |
| 225 | code, out = run(ctx, "status") |
| 226 | if code != 4 { |
| 227 | t.Fatalf("status of a failed move exit %d, want 4:\n%s", code, out) |
| 228 | } |
| 229 | wantLines(t, out, "uk could not take "+host+": the app settings differ. Run carlos move offline abort --host "+host+" to move it back.") |
| 230 | |
| 231 | // A source box that left a path in place says so, with its reason. |
| 232 | put(id, "ie-box", moves.OfflineRoleSource, moves.OfflinePhaseParked, func(r *moves.OfflineReport) { |
| 233 | r.Retired = []moves.RetiredPath{{Path: "/opt/carlos/live/" + host, Action: moves.RetiredLeft, Reason: "a root-owned directory"}} |
| 234 | }) |
| 235 | _, out = run(ctx, "status") |
| 236 | wantLines(t, out, "ie-box left /opt/carlos/live/"+host+" in place: a root-owned directory.") |
| 237 | } |
| 238 | |
| 239 | // TestMoveOfflineStageAgainstTheRealDoors drives stage, status, start and |
| 240 | // abort of a staged move through moveMain against a real console.Server, |
| 241 | // tying the CLI's stage view to the JSON the console writes. The stager is |
| 242 | // played by its report, written for the stage request the record carries. |
| 243 | func TestMoveOfflineStageAgainstTheRealDoors(t *testing.T) { |
| 244 | e := newOfflineE2E(t, []string{"region-ha-1", "offline-move-1", "offline-stage-1"}) |
| 245 | host, ctx := e.host, context.Background() |
| 246 | record := func() *provision.Record { |
| 247 | t.Helper() |
| 248 | rec, err := provision.GetRecord(ctx, e.rel, release.OpsAccount, "shop", host) |
| 249 | if err != nil { |
| 250 | t.Fatal(err) |
| 251 | } |
| 252 | return rec |
| 253 | } |
| 254 | // round plays the stager: one round for the record's current request. |
| 255 | round := func(id string) { |
| 256 | t.Helper() |
| 257 | e.put(id, "ie-box", moves.OfflineRoleStager, moves.OfflinePhaseStaged, func(r *moves.OfflineReport) { |
| 258 | r.Held, r.Fenced, r.Quiesced = false, false, false |
| 259 | r.StageRequest = *record().Move.StagedAt |
| 260 | r.ReplicaObjects, r.ReplicaBytes, r.StagedObjects, r.StagedBytes = 288, 72_000_000, 288, 72_000_000 |
| 261 | }) |
| 262 | } |
| 263 | type result struct { |
| 264 | code int |
| 265 | out string |
| 266 | } |
| 267 | // stage waits for its round; the test plays the stager once the record |
| 268 | // carries the request. |
| 269 | stage := func(wantRequestAfter time.Time) (string, result) { |
| 270 | t.Helper() |
| 271 | sctx, cancel := context.WithTimeout(ctx, 30*time.Second) |
| 272 | defer cancel() |
| 273 | done := make(chan result, 1) |
| 274 | go func() { |
| 275 | code, out := e.run(sctx, "stage", "--to", "uk") |
| 276 | done <- result{code, out} |
| 277 | }() |
| 278 | var id string |
| 279 | for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) { |
| 280 | if rec, err := provision.GetRecord(ctx, e.rel, release.OpsAccount, "shop", host); err == nil && rec.Move != nil && rec.Move.StagedAt != nil && rec.Move.StagedAt.After(wantRequestAfter) { |
| 281 | id = rec.Move.ID |
| 282 | break |
| 283 | } |
| 284 | } |
| 285 | if id == "" { |
| 286 | t.Fatal("stage never wrote its request") |
| 287 | } |
| 288 | round(id) |
| 289 | select { |
| 290 | case r := <-done: |
| 291 | return id, r |
| 292 | case <-time.After(20 * time.Second): |
| 293 | t.Fatal("stage never finished") |
| 294 | } |
| 295 | return "", result{} |
| 296 | } |
| 297 | staging := "Staging " + host + " in uk. " + host + " keeps serving while the data is copied." |
| 298 | ready := host + " keeps serving. Run carlos move offline start --host " + host + " --to uk when you are ready to take it offline." |
| 299 | id, r := stage(time.Time{}) |
| 300 | if r.code != 0 { |
| 301 | t.Fatalf("stage exit %d:\n%s", r.code, r.out) |
| 302 | } |
| 303 | wantLines(t, r.out, staging, ready) |
| 304 | if !strings.Contains(r.out, "Staged 288 of 288 objects for "+host+" in uk (72.0 of 72.0 MB) at ") { |
| 305 | t.Fatalf("stage output lacks the progress line:\n%s", r.out) |
| 306 | } |
| 307 | // Again: a new request, answered by a new round. |
| 308 | first := *record().Move.StagedAt |
| 309 | if id2, r := stage(first); r.code != 0 || id2 != id { |
| 310 | t.Fatalf("second stage exit %d (move %s, want %s):\n%s", r.code, id2, id, r.out) |
| 311 | } |
| 312 | code, out := e.run(ctx, "status") |
| 313 | if code != 0 { |
| 314 | t.Fatalf("status exit %d:\n%s", code, out) |
| 315 | } |
| 316 | wantLines(t, out, staging, ready) |
| 317 | |
| 318 | // abort from staging: the staged copy goes, then finish. |
| 319 | actx, cancel := context.WithTimeout(ctx, 30*time.Second) |
| 320 | defer cancel() |
| 321 | aborted := make(chan result, 1) |
| 322 | go func() { |
| 323 | code, out := e.run(actx, "abort") |
| 324 | aborted <- result{code, out} |
| 325 | }() |
| 326 | for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && record().Move.State != provision.OfflineReverting; time.Sleep(10 * time.Millisecond) { |
| 327 | } |
| 328 | e.put(id, "ie-box", moves.OfflineRoleSource, moves.OfflinePhaseReverted, func(r *moves.OfflineReport) { r.Held = false }) |
| 329 | // The pinned stager says the staged copy is gone: the finish waits for it. |
| 330 | e.put(id, "ie-box", moves.OfflineRoleStager, moves.OfflinePhaseReverted, func(r *moves.OfflineReport) { r.Held, r.Fenced, r.Quiesced = false, false, false }) |
| 331 | select { |
| 332 | case r := <-aborted: |
| 333 | if r.code != 3 { |
| 334 | t.Fatalf("abort exit %d:\n%s", r.code, r.out) |
| 335 | } |
| 336 | wantLines(t, r.out, "Removing the staged copy of "+host+" from uk.", |
| 337 | "The staged copy of "+host+" is removed. Run carlos move offline finish --host "+host+" to end the move.") |
| 338 | case <-time.After(20 * time.Second): |
| 339 | t.Fatal("abort never finished") |
| 340 | } |
| 341 | if code, out := e.run(ctx, "finish"); code != 3 { |
| 342 | t.Fatalf("finish exit %d:\n%s", code, out) |
| 343 | } |
| 344 | if rec := record(); rec.Move != nil || rec.LastMove == nil || rec.LastMove.Outcome != provision.OfflineOutcomeAborted || rec.LastMove.OfflineS != 0 { |
| 345 | t.Fatalf("after finish: %+v %+v", rec.Move, rec.LastMove) |
| 346 | } |
| 347 | } |
| 348 | |
amadan