amadan

carlos / platform Public

e2e: the self-serve move through the member's doors: forward, cancel in ready, the four aborts and the post-open warning

TestSelfServeForwardMove, TestSelfServeCancelInReady,
TestSelfServeAbortByFailedPrecheck, TestSelfServeAbortByStopDeadline,
TestSelfServeAbortByLandingFailure, TestSelfServeAbortByOpenCheckRefusal and
TestSelfServePostOpenFailureIsDoneWithAWarning: the owner's move-capable
token at the request, cut and cancel doors and the poll, over HTTP, on a
console with lifetimes on, the job on its pass and the bridge's real boxes.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
claude 1a8955b155c2e353c005d3e793dcb8aadd256ac8 parent 1676697
3 files changed, +564 −0
  • internal/e2e/selfserve/job_world_test.go +9 −0
  • internal/e2e/selfserve/member_test.go +203 −0
  • internal/e2e/selfserve/selfserve_forward_test.go +352 −0
diff --git a/internal/e2e/selfserve/job_world_test.go b/internal/e2e/selfserve/job_world_test.go
index fcc85f6bd..2a42b0a36 100644
--- a/internal/e2e/selfserve/job_world_test.go
+++ b/internal/e2e/selfserve/job_world_test.go
@@ -215,6 +215,12 @@ type jobWorld struct {
opToken, opBase string
// switchOff opens the console with CARLOS_SELF_SERVE_MOVES unset.
switchOff bool
+ // lifetimes turns lifetimes on in every console the world opens (the
+ // member's request door admits only then; member_test.go).
+ lifetimes bool
+ // The member's move-capable instance token and the console's HTTP
+ // front (member_test.go).
+ memberToken, memberFront string
}
// switchOf is the console's CARLOS_SELF_SERVE_MOVES.
@@ -274,6 +280,9 @@ func (w *jobWorld) open() {
}
console.TestHooks.NameConsole(srv, jobConsole)
w.cst, w.srv = cst, srv
+ if w.lifetimes && !console.TestHooks.LifetimesOn(w.ctx, srv) {
+ w.t.Fatal("lifetimes did not turn on in the console")
+ }
}
// restart is a console crash and restart: the same database file opened
diff --git a/internal/e2e/selfserve/member_test.go b/internal/e2e/selfserve/member_test.go
new file mode 100644
index 000000000..0679fb166
--- /dev/null
+++ b/internal/e2e/selfserve/member_test.go
@@ -0,0 +1,203 @@
+//go:build e2e
+
+package selfserve
+
+import (
+ "encoding/json"
+ "errors"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/console"
+ "github.com/carlosframework/platform/internal/host"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// The member's doors in the job's world (self-serve moves plan 2 Task 10):
+// an account owner's move-capable instance token on the console, over
+// HTTP, as the member's own code would call the three self-serve doors and
+// the status poll. The request door admits only on a console that has
+// turned lifetimes on, so a member world writes the cutover marker and
+// turns lifetimes on in every console it opens.
+
+const ownerAddr = "owner@example.com"
+
+// newMemberWorld is newJobWorld with lifetimes on and the member's token.
+func newMemberWorld(t *testing.T) *jobWorld {
+ t.Helper()
+ w := newJobWorld(t)
+ w.lifetimesOn()
+ w.member()
+ return w
+}
+
+// lifetimesOn writes the cutover marker and turns lifetimes on in this
+// console and every console the world opens after it.
+func (w *jobWorld) lifetimesOn() {
+ w.t.Helper()
+ const key = "control/lifetimes-cutover.json"
+ if _, _, err := w.raw.Get(w.ctx, key); err != nil {
+ body, _ := json.Marshal(struct {
+ At time.Time `json:"at"`
+ By string `json:"by"`
+ }{time.Now().UTC(), ownerAddr})
+ if _, err := w.raw.Put(w.ctx, key, body, ""); err != nil && !errors.Is(err, release.ErrCASConflict) {
+ w.t.Fatal(err)
+ }
+ }
+ w.lifetimes = true
+ if !console.TestHooks.LifetimesOn(w.ctx, w.srv) {
+ w.t.Fatal("lifetimes did not turn on")
+ }
+}
+
+// member is the owner's move-capable instance token for the app and the
+// console's HTTP front, made once per world (the console behind it is
+// whichever w.srv is current, so a restart keeps it).
+func (w *jobWorld) member() (token, front string) {
+ w.t.Helper()
+ if w.memberToken != "" {
+ return w.memberToken, w.memberFront
+ }
+ ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { w.srv.ServeHTTP(rw, r) }))
+ w.t.Cleanup(ts.Close)
+ now := time.Now()
+ acct, err := w.cst.AccountBySqid(w.acct)
+ if err != nil || acct == nil {
+ w.t.Fatalf("account %s: %v %v", w.acct, acct, err)
+ }
+ // The owner and the app as the console knows them (an error is either
+ // already there).
+ _ = w.cst.AddMember(acct.ID, ownerAddr, console.RoleOwner, now)
+ _, _ = w.cst.CreateAppIn(acct.ID, host.E2EApp, ownerAddr, now)
+ tok, err := w.cst.NewAppTokenContext(w.ctx, acct.ID, host.E2EApp, ownerAddr, "shop mover", true, now)
+ if err != nil || tok == "" {
+ w.t.Fatalf("no move-capable token: %v", err)
+ }
+ w.memberToken, w.memberFront = tok, ts.URL
+ return w.memberToken, w.memberFront
+}
+
+// memberDoor calls a member door of hostname's move ("" is the request,
+// the cancel or the poll, "/cut" the cut) and answers the status and body.
+func (w *jobWorld) memberDoor(method, hostname, verb, body string) (int, string) {
+ w.t.Helper()
+ tok, front := w.member()
+ u := front + "/api/cli/apps/" + w.acct + "/" + host.E2EApp + "/instances/" + hostname + "/move" + verb
+ req, err := http.NewRequest(method, u, strings.NewReader(body))
+ if err != nil {
+ w.t.Fatal(err)
+ }
+ req.Header.Set("Authorization", "Bearer "+tok)
+ if body != "" {
+ req.Header.Set("Content-Type", "application/json")
+ }
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ w.t.Fatal(err)
+ }
+ defer resp.Body.Close()
+ out, _ := io.ReadAll(resp.Body)
+ return resp.StatusCode, string(out)
+}
+
+// memberView is the self-serve view of GET .../move, as the member reads
+// it (console.selfServeMoveView's wire form).
+type memberView struct {
+ State string `json:"state"`
+ Target string `json:"target"`
+ Kind string `json:"kind"`
+ ID string `json:"id"`
+ EstimateSeconds int `json:"estimate_seconds"`
+ OfflineAt int64 `json:"offline_at"`
+ Reason string `json:"reason"`
+ Abort *struct {
+ RequestedAt int64 `json:"requested_at"`
+ SourceReopenedAt int64 `json:"source_reopened_at"`
+ FinishedAt int64 `json:"finished_at"`
+ } `json:"abort"`
+ Last *struct {
+ ID string `json:"id"`
+ Outcome string `json:"outcome"`
+ Reason string `json:"reason"`
+ Warning string `json:"warning"`
+ From string `json:"from"`
+ To string `json:"to"`
+ At int64 `json:"at"`
+ } `json:"last"`
+}
+
+func parseView(t *testing.T, out string) memberView {
+ t.Helper()
+ var v memberView
+ if err := json.Unmarshal([]byte(out), &v); err != nil {
+ t.Fatalf("the view %q: %v", out, err)
+ }
+ return v
+}
+
+// view is the member's poll of hostname.
+func (w *jobWorld) viewOf(hostname string) memberView {
+ w.t.Helper()
+ code, out := w.memberDoor(http.MethodGet, hostname, "", "")
+ if code != http.StatusOK {
+ w.t.Fatalf("the poll = %d %s", code, out)
+ }
+ return parseView(w.t, out)
+}
+
+func (w *jobWorld) view() memberView { return w.viewOf(host.E2EHost) }
+
+// prepare asks for the move to uk through the member's request door and
+// answers the move's id.
+func (w *jobWorld) prepare() string {
+ w.t.Helper()
+ code, out := w.memberDoor(http.MethodPost, host.E2EHost, "", `{"region":"uk","display_name":"Shop's account"}`)
+ if code != http.StatusAccepted {
+ w.t.Fatalf("the request = %d %s", code, out)
+ }
+ v := parseView(w.t, out)
+ if v.State != "preparing" || v.ID == "" || v.Kind != "offline" {
+ w.t.Fatalf("the request answered %s", out)
+ }
+ return v.ID
+}
+
+// cut and cancelDoor name id at the member's cut and cancel doors.
+func (w *jobWorld) cut(id string) (int, string) {
+ w.t.Helper()
+ return w.memberDoor(http.MethodPost, host.E2EHost, "/cut", `{"id":"`+id+`"}`)
+}
+
+func (w *jobWorld) cancelDoor(id string) (int, string) {
+ w.t.Helper()
+ return w.memberDoor(http.MethodDelete, host.E2EHost, "", `{"id":"`+id+`"}`)
+}
+
+// readyByDoor prepares a move through the member's door and runs it to
+// ready, the member's poll saying so.
+func (w *jobWorld) readyByDoor() string {
+ w.t.Helper()
+ id := w.prepare()
+ w.watch = id
+ w.phase(id, "ready", 60)
+ if v := w.view(); v.State != "ready" || v.ID != id {
+ w.t.Fatalf("the poll at ready: %+v", v)
+ }
+ return id
+}
+
+// cutByDoor runs a move prepared through the member's door to ready and
+// cuts it there.
+func (w *jobWorld) cutByDoor() string {
+ w.t.Helper()
+ id := w.readyByDoor()
+ if code, out := w.cut(id); code != http.StatusAccepted || parseView(w.t, out).State != "cutting" {
+ w.t.Fatalf("the cut = %d %s", code, out)
+ }
+ return id
+}
diff --git a/internal/e2e/selfserve/selfserve_forward_test.go b/internal/e2e/selfserve/selfserve_forward_test.go
new file mode 100644
index 000000000..1a85ecec4
--- /dev/null
+++ b/internal/e2e/selfserve/selfserve_forward_test.go
@@ -0,0 +1,352 @@
+//go:build e2e
+
+package selfserve
+
+import (
+ "encoding/json"
+ "net/http"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/console"
+ "github.com/carlosframework/platform/internal/host"
+ "github.com/carlosframework/platform/internal/moves"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// The self-serve move through the member's doors (self-serve moves plan 2
+// Task 10): the request, the poll, the cut and the cancel over HTTP with
+// the owner's move-capable token, the job on the console's pass and the
+// bridge's two real boxes doing the work.
+
+// TestSelfServeForwardMove: the member asks for the move to uk, polls it
+// preparing then ready, and cuts it; the job runs it through every phase
+// and the poll reads cutting with the offline time, then none with the
+// move's outcome: done from ie to uk. The record is in uk with no move and
+// last_move done with the job's receipt, the open record names the move,
+// the reservations are released, and the move's events end in exactly one
+// terminal event, every one delivered.
+func TestSelfServeForwardMove(t *testing.T) {
+ w := newMemberWorld(t)
+ id := w.prepare()
+ if v := w.view(); v.State != "preparing" || v.ID != id || v.EstimateSeconds <= 0 || v.Target != host.E2ETargetRegion {
+ t.Fatalf("the poll after the request: %+v", v)
+ }
+ w.watch = id
+ w.phase(id, "ready", 60)
+ if v := w.view(); v.State != "ready" || v.ID != id {
+ t.Fatalf("the poll at ready: %+v", v)
+ }
+ if code, out := w.cut(id); code != http.StatusAccepted || parseView(t, out).State != "cutting" {
+ t.Fatalf("the cut = %d %s", code, out)
+ }
+ sawOffline := false
+ r := w.run(300, "done", id, func(r *console.SelfServeMove) bool {
+ if r.Outcome == "" {
+ if v := w.view(); v.State == "cutting" && v.OfflineAt > 0 {
+ sawOffline = true
+ }
+ }
+ return r.Outcome != "" && !r.ReleasePending && r.EffectsDone
+ })
+ rec := w.record()
+ switch {
+ case r.Outcome != "done" || r.Warning != "":
+ t.Fatalf("finished %s warning %q", describe(r), r.Warning)
+ case !sawOffline:
+ t.Fatal("the poll never read cutting with the offline time")
+ case rec.Move != nil || rec.Region != host.E2ETargetRegion || rec.LastMove == nil || rec.LastMove.ID != id ||
+ rec.LastMove.Outcome != provision.OfflineOutcomeDone || rec.LastMove.Request == "":
+ t.Fatalf("the record: move %+v region %q last %+v", rec.Move, rec.Region, rec.LastMove)
+ case len(w.reservations()) != 0:
+ t.Fatalf("reservations left: %v", w.reservations())
+ }
+ if o := w.openRecord(id); o.MoveID != id || o.To != host.E2ETargetRegion || o.Incarnation != rec.Incarnation {
+ t.Fatalf("the open record %+v", o)
+ }
+ v := w.view()
+ if v.State != "none" || v.Last == nil || v.Last.ID != id || v.Last.Outcome != "done" || v.Last.From != host.E2EHomeRegion || v.Last.To != host.E2ETargetRegion || v.Last.Warning != "" {
+ t.Fatalf("the poll after the move: %+v %+v", v, v.Last)
+ }
+ w.deliverEvents()
+ evs, err := w.cst.SelfServeEvents(id)
+ if err != nil {
+ t.Fatal(err)
+ }
+ terminal := 0
+ for _, e := range evs {
+ if !e.Emitted {
+ t.Fatalf("event %+v not delivered", e)
+ }
+ if e.Phase == "done" {
+ terminal++
+ }
+ }
+ if terminal != 1 {
+ t.Fatalf("%d terminal events in %d", terminal, len(evs))
+ }
+}
+
+// deliverEvents ticks until every event is delivered (the terminal event
+// is the last tick's, so the next delivers it).
+func (w *jobWorld) deliverEvents() {
+ w.t.Helper()
+ for i := 0; i < 3; i++ {
+ if left, err := w.cst.UnemittedSelfServeEvents(10); err == nil && len(left) == 0 {
+ return
+ }
+ w.tick()
+ }
+}
+
+// TestSelfServeCancelInReady: the member cancels a ready move through the
+// cancel door: 202 with the cancellation's request time; a cut naming the
+// move after it is refused "the move was cancelled"; the job finishes it
+// aborted "the move was cancelled", lossless (the record home with no
+// move, last_move aborted with no offline time, the reservations released,
+// the source reopened at the cancel), and the poll reads none with that
+// outcome. The cancel door's own table and races are Task 5's
+// (TestMutationsNameTheMoveAndRace); the abort in every phase before the
+// cut is TestAbortBeforeOfflineIsLossless.
+func TestSelfServeCancelInReady(t *testing.T) {
+ w := newMemberWorld(t)
+ region := w.record().Region
+ id := w.readyByDoor()
+ code, out := w.cancelDoor(id)
+ if code != http.StatusAccepted {
+ t.Fatalf("the cancel = %d %s", code, out)
+ }
+ if v := parseView(t, out); v.ID != id || v.Abort == nil || v.Abort.RequestedAt == 0 {
+ t.Fatalf("the cancel answered %s", out)
+ }
+ if code, out := w.cut(id); code != http.StatusConflict || errorOf(out) != "the move was cancelled" {
+ t.Fatalf("the cut after the cancel = %d %s", code, out)
+ }
+ r := w.finished(id, 60)
+ w.requireLossless(r, region)
+ rec := w.record()
+ switch {
+ case r.OutcomeReason != "the move was cancelled" || !r.SourceReopenedAt.Equal(r.AbortRequestedAt):
+ t.Fatalf("finished %s, reopened %v, asked %v", describe(r), r.SourceReopenedAt, r.AbortRequestedAt)
+ case rec.LastMove.ID != id || rec.LastMove.OfflineS != 0 || rec.LastMove.Request == "":
+ t.Fatalf("last_move %+v", rec.LastMove)
+ }
+ v := w.view()
+ if v.State != "none" || v.Last == nil || v.Last.ID != id || v.Last.Outcome != "aborted" || v.Last.Reason != "the move was cancelled" {
+ t.Fatalf("the poll after the cancel: %+v %+v", v, v.Last)
+ }
+}
+
+// requireAbortedView is the member's poll after an abort: none, with the
+// move's outcome aborted and its reason.
+func (w *jobWorld) requireAbortedView(id, reason string) {
+ w.t.Helper()
+ v := w.view()
+ if v.State != "none" || v.Last == nil || v.Last.ID != id || v.Last.Outcome != "aborted" || v.Last.Reason != reason {
+ w.t.Fatalf("the poll after the abort: %+v %+v; want aborted %q", v, v.Last, reason)
+ }
+}
+
+// TestSelfServeAbortByFailedPrecheck: the move is staged and in its
+// prechecks when the real target box comes to hold data for the instance
+// (an env file in its data directory); the target's precheck report says
+// so and the job aborts with the target's fact, before the cut: the record
+// never goes offline (last_move aborted, no offline time), the source
+// never stopped, and the poll reads the reason. The precheck's book
+// comparison is TestPrechecksCompareTheBook.
+func TestSelfServeAbortByFailedPrecheck(t *testing.T) {
+ w := newMemberWorld(t)
+ region := w.record().Region
+ id := w.prepare()
+ w.watch = id
+ w.phase(id, "prechecks", 60)
+ env := filepath.Join(w.e.Box(host.E2ETargetBox).DataDir(), host.E2EHost+".env")
+ if err := os.WriteFile(env, []byte("A=1\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ r := w.finished(id, 40)
+ want := "the target box already holds data for shop.example.com: " + env
+ if r.OutcomeReason != want {
+ t.Fatalf("finished %s; want %q", describe(r), want)
+ }
+ w.requireLossless(r, region)
+ if rec := w.record(); rec.LastMove.OfflineS != 0 || !r.SourceReopenedAt.Equal(r.AbortRequestedAt) || w.hits["start:write"] != 0 {
+ t.Fatalf("the move went offline: last %+v, reopened %v asked %v, %d starts", rec.LastMove, r.SourceReopenedAt, r.AbortRequestedAt, w.hits["start:write"])
+ }
+ w.requireAbortedView(id, want)
+}
+
+// TestSelfServeAbortByStopDeadline: after the cut the source box stops
+// answering, so its stop is never confirmed; past the stop's deadline the
+// job aborts naming it ("could not determine the stop on ie-box"), and
+// once the box is back the abort completes lossless and the poll reads it.
+func TestSelfServeAbortByStopDeadline(t *testing.T) {
+ w := newMemberWorld(t)
+ region := w.record().Region
+ id := w.cutByDoor()
+ w.run(100, "stopping", id, func(r *console.SelfServeMove) bool { return r.Phase == "stopping" || r.Outcome != "" })
+ src := w.e.Box(host.E2ESourceBox)
+ src.Down()
+ w.tick()
+ r := w.row(id)
+ if r.Phase != "stopping" {
+ t.Fatalf("the row left stopping with the source down: %s", describe(r))
+ }
+ w.pastDeadline(r)
+ w.consoleUntil(4, "the abort", func() bool { return !w.row(id).AbortRequestedAt.IsZero() })
+ const want = "could not determine the stop on ie-box"
+ if r = w.row(id); r.OutcomeReason != want {
+ t.Fatalf("aborted %s; want %q", describe(r), want)
+ }
+ src.Up()
+ w.requireLossless(w.done(id), region)
+ w.requireAbortedView(id, want)
+}
+
+// TestSelfServeAbortByLandingFailure: the sealed archive in the target's
+// bucket is damaged after the flip and before the target lands it; the
+// real target box reports its landing failed ("the archive does not
+// verify") and the job aborts with the box's own detail: the sealer lifts
+// the tombstone, the source reopens, the abort ends lossless and the poll
+// reads the detail.
+func TestSelfServeAbortByLandingFailure(t *testing.T) {
+ w := newMemberWorld(t)
+ region := w.record().Region
+ id := w.cutByDoor()
+ damaged := false
+ w.e.Fault(host.FaultPass+host.E2ETargetBox, func() {
+ if rec := w.record(); damaged || rec.Move == nil || rec.Move.State != provision.OfflineLanding {
+ return
+ }
+ damaged = true
+ w.damageArchive(id)
+ })
+ r := w.run(200, "the abort", id, func(r *console.SelfServeMove) bool { return !r.AbortRequestedAt.IsZero() || r.Outcome != "" })
+ w.e.Fault(host.FaultPass+host.E2ETargetBox, nil)
+ if !damaged {
+ t.Fatalf("the move never reached landing: %s", describe(r))
+ }
+ // The target's report as the job decided on it (the box rewrites it
+ // once it reverts).
+ tgt, ok := w.report(id, moves.OfflineRoleTarget, host.E2ETargetBox)
+ if !ok || tgt.Phase != moves.OfflinePhaseFailed {
+ t.Fatalf("the target's report %+v", tgt)
+ }
+ const want = "the archive does not verify"
+ if r.OutcomeReason != want || moves.FailedDetail(tgt) != want {
+ t.Fatalf("aborted %s; the target's detail %q", describe(r), moves.FailedDetail(tgt))
+ }
+ r = w.done(id)
+ if r.OutcomeReason != want || w.hits["open:write"] != 0 {
+ t.Fatalf("finished %s, %d opens", describe(r), w.hits["open:write"])
+ }
+ w.requireLossless(r, region)
+ w.requireAbortedView(id, want)
+}
+
+// damageArchive flips a byte of the move's sealed archive in the target's
+// data bucket, over the bridge's S3 endpoint.
+func (w *jobWorld) damageArchive(id string) {
+ w.t.Helper()
+ st, err := release.NewS3StoreIn(w.ctx, w.e.Box(host.E2ETargetBox).DataBucket(), "eu-west-2")
+ if err != nil {
+ w.t.Fatal(err)
+ }
+ key := moves.OfflineArchivePrefix(host.E2EHost, id) + "files.sealed"
+ body, etag, err := st.Get(w.ctx, key)
+ if err != nil || len(body) == 0 {
+ w.t.Fatalf("the sealed archive %s: %d bytes, %v", key, len(body), err)
+ }
+ body[len(body)/2] ^= 0xff
+ if _, err := st.Put(w.ctx, key, body, etag); err != nil {
+ w.t.Fatal(err)
+ }
+}
+
+// TestSelfServeAbortByOpenCheckRefusal: the target has landed and the job
+// is to open it, but the open is refused. Another move's open record in
+// the way is refused at once: the job aborts "another move's open record
+// is in the way". A landing report the open check does not accept (an
+// item not OK, the target then silent) is waited on and never opened, and
+// past the open's deadline aborts "could not determine the open on
+// uk-box". Either way the open is never written, the abort ends lossless
+// and the poll reads the reason.
+func TestSelfServeAbortByOpenCheckRefusal(t *testing.T) {
+ for _, c := range []struct {
+ name, want string
+ refuse func(w *jobWorld, id string)
+ }{
+ {"another move's open record", "another move's open record is in the way", func(w *jobWorld, id string) {
+ body, _ := json.Marshal(moves.OpenRecord{MoveID: id, Host: host.E2EHost, Account: w.acct, App: host.E2EApp,
+ Incarnation: w.record().Incarnation + 1, From: "", To: host.E2ETargetRegion, OpenedAt: time.Now().UTC()})
+ if _, err := w.raw.Put(w.ctx, moves.OpenRecordKey(host.E2EHost, id), body, ""); err != nil {
+ w.t.Fatal(err)
+ }
+ }},
+ {"a landing the open check refuses", "could not determine the open on uk-box", func(w *jobWorld, id string) {
+ w.e.Box(host.E2ETargetBox).Down()
+ w.doctor(id, moves.OfflineRoleTarget, host.E2ETargetBox, func(r *moves.OfflineReport) {
+ for k, c := range r.Items {
+ c.OK, c.Reason = false, "waits"
+ r.Items[k] = c
+ break
+ }
+ })
+ w.tick()
+ if r := w.row(id); r.Phase != "landed" || w.hits["open:write"] != 0 {
+ w.t.Fatalf("the refused open moved: %s, %d writes", describe(r), w.hits["open:write"])
+ }
+ w.pastDeadline(w.row(id))
+ }},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ w := newMemberWorld(t)
+ region := w.record().Region
+ id := w.cutByDoor()
+ // At landed the job's open is the next tick's.
+ if r := w.run(200, "landed", id, func(r *console.SelfServeMove) bool { return r.Phase == "landed" || r.Outcome != "" }); r.Phase != "landed" {
+ t.Fatalf("the move is %s", describe(r))
+ }
+ c.refuse(w, id)
+ r := w.done(id)
+ if r.OutcomeReason != c.want || w.hits["open:write"] != 0 {
+ t.Fatalf("finished %s, %d opens written; want %q", describe(r), w.hits["open:write"], c.want)
+ }
+ w.requireLossless(r, region)
+ w.requireAbortedView(id, c.want)
+ })
+ }
+}
+
+// TestSelfServePostOpenFailureIsDoneWithAWarning: after the open the
+// account never answers its health probes; the health window runs out and
+// the move is done all the same (the instance is open in uk; nothing is
+// undone after the open), with the warning, which the poll reads with the
+// outcome. The paging and the window's precedence are
+// TestHealthFailureFinishesWithAWarningAndPages and
+// TestHealthExpiryTakesPrecedence.
+func TestSelfServePostOpenFailureIsDoneWithAWarning(t *testing.T) {
+ w := newMemberWorld(t)
+ w.probe = func() (bool, string) { return false, "the probe did not get an answer" }
+ id := w.cutByDoor()
+ r := w.run(200, "health", id, func(r *console.SelfServeMove) bool { return r.Phase == "health" || r.Outcome != "" })
+ w.pastDeadline(r)
+ r = w.done(id)
+ const warning = "the account did not answer on uk within 4 minutes"
+ rec := w.record()
+ switch {
+ case r.Outcome != "done" || r.Warning != warning || !r.AbortRequestedAt.IsZero():
+ t.Fatalf("finished %s warning %q", describe(r), r.Warning)
+ case rec.Move != nil || rec.Region != host.E2ETargetRegion || rec.LastMove == nil || rec.LastMove.Outcome != provision.OfflineOutcomeDone:
+ t.Fatalf("the record: move %+v region %q last %+v", rec.Move, rec.Region, rec.LastMove)
+ case len(w.reservations()) != 0:
+ t.Fatalf("reservations left: %v", w.reservations())
+ }
+ v := w.view()
+ if v.State != "none" || v.Last == nil || v.Last.ID != id || v.Last.Outcome != "done" || v.Last.Warning != warning {
+ t.Fatalf("the poll after the move: %+v %+v", v, v.Last)
+ }
+}