amadan
e2e: the operator's doors unchanged on the bridge with the switch off, an operator return after a self-serve move, and the claims-off box's published instance id
TestSelfServeOperatorDoorsUnchangedOnTheBridge replays the console's answers from main (operator-door-answers.golden) for the bridge's world and checks the sentinel reservation and the legacy move door; TestSelfServeThenOperatorReturn runs a self-serve forward and then the operator's return through every operator door to home; TestClaimsOffInstanceIDIsPublished checks the status block's box id and instance id, read on each tick, and the participants pinned by them.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1 file changed,
+358
−0
internal/e2e/selfserve/selfserve_operator_test.go+358 −0
diff --git a/internal/e2e/selfserve/selfserve_operator_test.go b/internal/e2e/selfserve/selfserve_operator_test.go| new file mode 100644 |
| index 000000000..8a3297d93 |
| --- /dev/null |
| +++ b/internal/e2e/selfserve/selfserve_operator_test.go |
| @@ -0,0 +1,358 @@ |
| +//go:build e2e |
| + |
| +package selfserve |
| + |
| +import ( |
| + "encoding/json" |
| + "fmt" |
| + "io" |
| + "net/http" |
| + "os" |
| + "path/filepath" |
| + "strings" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/activator" |
| + "github.com/carlosframework/platform/internal/host" |
| + "github.com/carlosframework/platform/internal/lease" |
| + "github.com/carlosframework/platform/internal/moves" |
| + "github.com/carlosframework/platform/internal/provision" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// The operator's doors beside the self-serve move, and the box's identity |
| +// in its status (self-serve moves plan 2 Task 10). |
| + |
| +// operatorLine renders an operator door's answer as the console's replay |
| +// from main keeps it (operatorAnswer in |
| +// internal/console/apiinstancemove_offline_snapshot_test.go): the status |
| +// and the error sentence, or for a 200 or 202 the view's stable fields. |
| +func operatorLine(code int, out string) string { |
| + if code == http.StatusAccepted || code == http.StatusOK { |
| + var v struct { |
| + State string `json:"state"` |
| + From string `json:"from"` |
| + To string `json:"to"` |
| + Stager string `json:"stager"` |
| + Next string `json:"next"` |
| + Waiting string `json:"waiting"` |
| + CanAbort bool `json:"can_abort"` |
| + NeverOffline bool `json:"never_offline"` |
| + } |
| + if err := json.Unmarshal([]byte(out), &v); err != nil { |
| + return fmt.Sprintf("%d unparsed %q", code, out) |
| + } |
| + return fmt.Sprintf("%d state=%s from=%s to=%s stager=%s next=%s waiting=%q can_abort=%v never_offline=%v", |
| + code, v.State, v.From, v.To, v.Stager, v.Next, v.Waiting, v.CanAbort, v.NeverOffline) |
| + } |
| + return fmt.Sprintf("%d %q", code, errorOf(out)) |
| +} |
| + |
| +// goldenOperatorLines is the console's replay from main, by "case | what", |
| +// with its host named as the bridge names its instance. |
| +func goldenOperatorLines(t *testing.T) map[string]string { |
| + t.Helper() |
| + raw, err := os.ReadFile("../../console/testdata/operator-door-answers.golden") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + out := map[string]string{} |
| + for _, line := range strings.Split(strings.ReplaceAll(string(raw), "acme.shop.example.com", host.E2EHost), "\n") { |
| + parts := strings.SplitN(line, " | ", 3) |
| + if len(parts) == 3 { |
| + out[parts[0]+" | "+parts[1]] = parts[2] |
| + } |
| + } |
| + return out |
| +} |
| + |
| +// legacyMoveDoor calls the move door itself (POST .../move, the request |
| +// door that self-serve mode makes the member's) with the operator's token. |
| +func (w *jobWorld) legacyMoveDoor(body string) (int, string) { |
| + w.t.Helper() |
| + tok, base := w.operator() |
| + req, err := http.NewRequest(http.MethodPost, strings.TrimSuffix(base, "/offline"), strings.NewReader(body)) |
| + if err != nil { |
| + w.t.Fatal(err) |
| + } |
| + req.Header.Set("Authorization", "Bearer "+tok) |
| + req.Header.Set("Content-Type", "application/json") |
| + resp, err := http.DefaultClient.Do(req) |
| + if err != nil { |
| + w.t.Fatal(err) |
| + } |
| + defer resp.Body.Close() |
| + out, _ := io.ReadAll(resp.Body) |
| + return resp.StatusCode, string(out) |
| +} |
| + |
| +// TestSelfServeOperatorDoorsUnchangedOnTheBridge: on the bridge's real |
| +// boxes, with CARLOS_SELF_SERVE_MOVES off, the operator's start and stage |
| +// doors answer every line the console's replay from main keeps for this |
| +// world (the good body, a malformed one, the current region and an invalid |
| +// one, each door's view after it, and a staged move's status, restage and |
| +// start), byte for byte; the operator's reservation is the sentinel "*" |
| +// and nothing else; and the move door itself is the legacy door (its own |
| +// refusals), where in self-serve mode it is the member's door, which |
| +// refuses the operator's token with the family's 404. |
| +func TestSelfServeOperatorDoorsUnchangedOnTheBridge(t *testing.T) { |
| + golden := goldenOperatorLines(t) |
| + switchOff := func(t *testing.T) *jobWorld { |
| + w := newJobWorld(t) |
| + w.switchOff = true |
| + w.restart() |
| + return w |
| + } |
| + check := func(t *testing.T, key string, code int, out string) { |
| + t.Helper() |
| + want, ok := golden[key] |
| + if !ok { |
| + t.Fatalf("the replay has no line %q", key) |
| + } |
| + if got := operatorLine(code, out); got != want { |
| + t.Errorf("%s:\n got %s\nwant %s", key, got, want) |
| + } |
| + } |
| + sentinel := func(t *testing.T, w *jobWorld) { |
| + t.Helper() |
| + table := w.reservations() |
| + if _, ok := table[moves.OfflineSentinel]; !ok || len(table) != 1 { |
| + t.Fatalf("the operator's reservation %v, want the sentinel alone", table) |
| + } |
| + } |
| + t.Run("the refusals", func(t *testing.T) { |
| + w := switchOff(t) |
| + for _, door := range []struct{ name, path string }{{"start", ""}, {"stage", "/stage"}} { |
| + for _, b := range []struct{ name, body string }{{"malformed", `{"region":`}, {"current", `{"region":"ie"}`}, {"invalid", `{"region":"UK!"}`}} { |
| + code, out := w.door(http.MethodPost, door.path, b.body) |
| + check(t, "ok | "+door.name+" "+b.name, code, out) |
| + } |
| + } |
| + if rec := w.record(); rec.Move != nil || len(w.reservations()) != 0 { |
| + t.Fatalf("a refusal wrote: %+v, %v", rec.Move, w.reservations()) |
| + } |
| + // The move door: the legacy door's refusals with the switch off. |
| + for body, want := range map[string]string{ |
| + `{"region":"UK!"}`: `400 "invalid region"`, |
| + `{"region":"ie"}`: `400 "moves to the home region are not supported yet — the home publishes no staging lane"`, |
| + } { |
| + if got := operatorLine(w.legacyMoveDoor(body)); got != want { |
| + t.Errorf("the move door with %s: %s, want %s", body, got, want) |
| + } |
| + } |
| + // In self-serve mode the move door is the member's: the operator's |
| + // token is not a move-capable token, and gets the 404. |
| + w.switchOff = false |
| + w.restart() |
| + for _, body := range []string{`{"region":"UK!"}`, `{"region":"uk"}`} { |
| + if code, out := w.legacyMoveDoor(body); code != http.StatusNotFound || out != "404 page not found\n" { |
| + t.Errorf("the move door in self-serve mode with %s: %d %q", body, code, out) |
| + } |
| + } |
| + }) |
| + t.Run("start", func(t *testing.T) { |
| + w := switchOff(t) |
| + code, out := w.door(http.MethodPost, "", `{"region":"uk"}`) |
| + check(t, "ok | start good", code, out) |
| + code, out = w.door(http.MethodGet, "", "") |
| + check(t, "ok | start good then status", code, out) |
| + sentinel(t, w) |
| + }) |
| + t.Run("stage, then the staged move", func(t *testing.T) { |
| + w := switchOff(t) |
| + code, out := w.door(http.MethodPost, "/stage", `{"region":"uk"}`) |
| + check(t, "ok | stage good", code, out) |
| + code, out = w.door(http.MethodGet, "", "") |
| + check(t, "ok | stage good then status", code, out) |
| + sentinel(t, w) |
| + code, out = w.door(http.MethodGet, "", "") |
| + check(t, "staged | status", code, out) |
| + code, out = w.door(http.MethodPost, "/stage", `{"region":"uk"}`) |
| + check(t, "staged | restage", code, out) |
| + code, out = w.door(http.MethodPost, "", `{"region":"uk"}`) |
| + check(t, "staged | start", code, out) |
| + sentinel(t, w) |
| + }) |
| +} |
| + |
| +// operatorRun drives the operator's offline doors from the record's state |
| +// until the move is cleared: seal once stopping, flip once sealed, open |
| +// once landed, finish once open and while finishing; the boxes and the |
| +// console's pass run between the doors, and the source's held config is |
| +// rendered as config sync would. It fails after max rounds. |
| +func (w *jobWorld) operatorRun(max int, source string) *provision.Record { |
| + w.t.Helper() |
| + rendered := false |
| + for i := 0; i < max; i++ { |
| + rec := w.record() |
| + if rec.Move == nil { |
| + return rec |
| + } |
| + w.boxes() |
| + if !rendered { |
| + ok, err := w.e.Box(source).RenderHeldConfig(w.ctx, rec.Move.ID) |
| + if err != nil { |
| + w.t.Logf("render: %v", err) |
| + } |
| + rendered = ok |
| + } |
| + w.tick() |
| + verb := map[string]string{provision.OfflineStopping: "/seal", provision.OfflineSealed: "/flip", |
| + provision.OfflineLanded: "/open", provision.OfflineOpen: "/finish", provision.OfflineFinishing: "/finish"}[w.record().Move.State] |
| + if verb != "" { |
| + if code, out := w.door(http.MethodPost, verb, ""); code != http.StatusAccepted { |
| + w.t.Logf("%s: %d %s", verb, code, errorOf(out)) |
| + } |
| + } |
| + } |
| + w.t.Fatalf("after %d rounds the operator's move is %+v", max, w.record().Move) |
| + return nil |
| +} |
| + |
| +// TestSelfServeThenOperatorReturn: the member's self-serve move to uk is |
| +// done; the operator then returns the instance to ie through the operator |
| +// doors: the return request (its box report of the leftovers), the stage |
| +// and the start with return_of (the staged return names the forward), the |
| +// seal, the flip, the open and both finishes. The record is home with no |
| +// move and last_move done for the return; nothing stays reserved; and the |
| +// member's poll reads no move in flight, its last self-serve move still |
| +// the forward. The handshake's refusals and restarts are |
| +// TestOperatorReturnHandshake. |
| +func TestSelfServeThenOperatorReturn(t *testing.T) { |
| + w := newMemberWorld(t) |
| + build := w.liveBuild(host.E2ESourceBox) |
| + fwd := w.cutByDoor() |
| + if r := w.done(fwd); r.Outcome != "done" { |
| + t.Fatalf("the forward finished %s", describe(r)) |
| + } |
| + w.wakeTarget(build) |
| + // The forward leaves the source's timer rows (not among the leftovers |
| + // a return accepts): the operator clears them, as the runbook does. |
| + w.clearTimers(host.E2ESourceBox) |
| + key := w.returnRequest(fwd) |
| + w.boxes() |
| + if r, ok := w.returnReport(fwd, w.e.Box(host.E2ESourceBox).BoxID(), key); !ok || r.Ready == nil || r.Ready.Leftovers == nil { |
| + t.Fatalf("no return report: %v %+v", ok, r) |
| + } |
| + if code, out := w.door(http.MethodPost, "/stage", `{"region":"ie","return_of":"`+fwd+`"}`); code != http.StatusAccepted { |
| + t.Fatalf("the return's stage = %d %s", code, out) |
| + } |
| + ret := w.record().Move.ID |
| + if m := w.record().Move; m.ReturnOf != fwd || m.State != provision.OfflineStaging { |
| + t.Fatalf("the staged return %+v", m) |
| + } |
| + w.until(60, "the return staged", func() bool { |
| + st, ok := w.report(ret, moves.OfflineRoleStager, host.E2ETargetBox) |
| + return ok && st.Phase == moves.OfflinePhaseStaged |
| + }) |
| + if code, out := w.door(http.MethodPost, "", `{"region":"ie","return_of":"`+fwd+`"}`); code != http.StatusAccepted { |
| + t.Fatalf("the return's start = %d %s", code, out) |
| + } |
| + rec := w.operatorRun(200, host.E2ETargetBox) |
| + switch { |
| + case rec.Region != "" || rec.LastMove == nil || rec.LastMove.ID != ret || rec.LastMove.Outcome != provision.OfflineOutcomeDone: |
| + t.Fatalf("after the return: region %q last %+v", rec.Region, rec.LastMove) |
| + case len(w.reservations()) != 0: |
| + t.Fatalf("reservations left: %v", w.reservations()) |
| + } |
| + v := w.view() |
| + if v.State != "none" || v.ID != "" || v.Last == nil || v.Last.ID != fwd || v.Last.Outcome != "done" { |
| + t.Fatalf("the member's poll after the return: %+v %+v", v, v.Last) |
| + } |
| + if open, err := w.cst.OpenSelfServeMoves(); err != nil || len(open) != 0 { |
| + t.Fatalf("self-serve rows open: %v %v", open, err) |
| + } |
| +} |
| + |
| +// TestClaimsOffInstanceIDIsPublished: on a claims-off box the agent's |
| +// status tick publishes the box's id and its cloud instance id in its |
| +// status block, the instance id read when the tick runs (an instance id |
| +// the box learns later is published by the next tick, not held at the one |
| +// the worker was armed with), and the console pins a self-serve move's |
| +// participants by them. |
| +func TestClaimsOffInstanceIDIsPublished(t *testing.T) { |
| + w := newMemberWorld(t) |
| + src, tgt := w.e.Box(host.E2ESourceBox), w.e.Box(host.E2ETargetBox) |
| + published := func(label string) (boxID, instanceID string) { |
| + t.Helper() |
| + _, metrics, _ := loadBoxes(t, w.raw) |
| + m := metrics[label] |
| + if m == nil { |
| + t.Fatalf("no status block for %s", label) |
| + } |
| + return m.BoxID, m.InstanceID |
| + } |
| + if os.Getenv("CARLOS_HOST_CLAIM") != "" { |
| + t.Fatal("the bridge's boxes are not claims-off") |
| + } |
| + for _, b := range []struct { |
| + label string |
| + h interface{ BoxID() string } |
| + }{{src.Label(), src}, {tgt.Label(), tgt}} { |
| + if id, inst := published(b.label); id == "" || id != b.h.BoxID() || inst != "i-e2e-"+b.label { |
| + t.Fatalf("%s publishes box %q instance %q", b.label, id, inst) |
| + } |
| + } |
| + // The source learns a new instance id (a stop and start of its VM): |
| + // the next status tick publishes it. |
| + src.SetInstanceID("i-e2e-ie-box-2") |
| + if err := w.e.Status(w.ctx); err != nil { |
| + t.Fatal(err) |
| + } |
| + if id, inst := published(src.Label()); id != src.BoxID() || inst != "i-e2e-ie-box-2" { |
| + t.Fatalf("after the new instance id: box %q instance %q", id, inst) |
| + } |
| + // The member's move pins both boxes by what they publish. |
| + id := w.prepare() |
| + r := w.row(id) |
| + if len(r.Participants.Source) != 1 || r.Participants.Source[0].BoxID != src.BoxID() || r.Participants.Source[0].InstanceID != "i-e2e-ie-box-2" || |
| + r.Participants.Target.BoxID != tgt.BoxID() || r.Participants.Target.InstanceID != "i-e2e-uk-box" { |
| + t.Fatalf("the move pinned %+v", r.Participants) |
| + } |
| +} |
| + |
| +// liveBuild is the build live for the instance on box label: the version |
| +// directory its live link names (<opt>/versions/<build>/<acct>/<app>). |
| +func (w *jobWorld) liveBuild(label string) string { |
| + w.t.Helper() |
| + opt := w.e.Box(label).OptDir() |
| + target, err := os.Readlink(release.LiveLink(opt, host.E2EHost)) |
| + if err != nil { |
| + w.t.Fatalf("no live build on %s: %v", label, err) |
| + } |
| + rel, err := filepath.Rel(filepath.Join(opt, "versions"), target) |
| + if err != nil { |
| + w.t.Fatal(err) |
| + } |
| + return strings.SplitN(rel, string(filepath.Separator), 2)[0] |
| +} |
| + |
| +// wakeTarget stands in for what the target box does after the open that |
| +// the bridge does not run: the build adoption puts the moved build live on |
| +// the box, and the first wake promotes the staged replica (moves/<host>/) |
| +// over the instance's replica (instances/<host>) in the box's own bucket |
| +// (the activator's promoteStagedReplica), which the served instance |
| +// replicates to from then on. |
| +func (w *jobWorld) wakeTarget(build string) { |
| + w.t.Helper() |
| + opt := w.e.Box(host.E2ETargetBox).OptDir() |
| + versions := filepath.Join(opt, "versions", build, w.acct, host.E2EApp) |
| + link := release.LiveLink(opt, host.E2EHost) |
| + for _, dir := range []string{versions, filepath.Dir(link)} { |
| + if err := os.MkdirAll(dir, 0o755); err != nil { |
| + w.t.Fatal(err) |
| + } |
| + } |
| + if err := os.Symlink(versions, link); err != nil && !os.IsExist(err) { |
| + w.t.Fatal(err) |
| + } |
| + bucket := w.e.Box(host.E2ETargetBox).DataBucket() |
| + m, err := lease.NewS3Mirror(w.ctx, "eu-west-2", "eu-west-2") |
| + if err != nil { |
| + w.t.Fatal(err) |
| + } |
| + round, err := lease.MirrorPrefix(w.ctx, m, bucket, activator.MovesPrefix(host.E2EHost), bucket, activator.HibernatePrefix("instances", host.E2EHost), lease.Receipts{}) |
| + if err != nil || round.Deferred > 0 { |
| + w.t.Fatalf("the promotion: %+v %v", round, err) |
| + } |
| +} |