amadan

carlos / platform Public

e2e: a cut naming a stale id, a host recreated under its row, two moves on shared and disjoint boxes, and the reclaim only on evidence

TestSelfServeCutNamingAStaleID, TestSelfServeRecreatedBetweenRowAndRecord,
TestSelfServeTwoMovesDisjointAndQueued and
TestSelfServeReservationReclaimOnlyOnEvidence, through the member's doors on
a console with lifetimes on, over the bridge's real boxes.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
claude 6d7d4aa1decf2c4f332a91d70a9099ef785a75df parent 1a8955b
1 file changed, +309 −0
  • internal/e2e/selfserve/selfserve_rows_test.go +309 −0
diff --git a/internal/e2e/selfserve/selfserve_rows_test.go b/internal/e2e/selfserve/selfserve_rows_test.go
new file mode 100644
index 000000000..ec945e62d
--- /dev/null
+++ b/internal/e2e/selfserve/selfserve_rows_test.go
@@ -0,0 +1,309 @@
+//go:build e2e
+
+package selfserve
+
+import (
+ "encoding/json"
+ "net/http"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/console"
+ "github.com/carlosframework/platform/internal/host"
+ "github.com/carlosframework/platform/internal/moves"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/status"
+)
+
+// The self-serve rows against the records and the reservations
+// (self-serve moves plan 2 Task 10): a cut naming another move, a host
+// recreated under its row, moves sharing boxes and moves on other boxes,
+// and the reclaim of a reservation no move owns.
+
+// TestSelfServeCutNamingAStaleID: the member's first move is cancelled and
+// finished; the second is ready. A cut naming the first move's id, or an
+// id no move has, is refused "this is not the move you confirmed" and
+// writes nothing (the ready row keeps no cut request, the serial is not
+// bumped, the row stays ready over ticks); a cancel naming the stale id
+// likewise. The cut naming the ready move's own id is accepted and the
+// move goes on into the cut.
+func TestSelfServeCutNamingAStaleID(t *testing.T) {
+ w := newMemberWorld(t)
+ first := w.prepare()
+ if code, out := w.cancelDoor(first); code != http.StatusAccepted {
+ t.Fatalf("the first's cancel = %d %s", code, out)
+ }
+ if r := w.finished(first, 30); r.Outcome != "aborted" {
+ t.Fatalf("the first finished %s", describe(r))
+ }
+ second := w.readyByDoor()
+ bumps := w.fs.bumpCount()
+ for _, stale := range []string{first, "mv_nosuchmovenosuchmovenosuch", ""} {
+ if code, out := w.cut(stale); code != http.StatusConflict || errorOf(out) != "this is not the move you confirmed" {
+ t.Fatalf("the cut naming %q = %d %s", stale, code, out)
+ }
+ if code, out := w.cancelDoor(stale); code != http.StatusConflict || errorOf(out) != "this is not the move you confirmed" {
+ t.Fatalf("the cancel naming %q = %d %s", stale, code, out)
+ }
+ }
+ if got := w.fs.bumpCount(); got != bumps {
+ t.Fatalf("the refused doors bumped the serial %d times", got-bumps)
+ }
+ for i := 0; i < 3; i++ {
+ w.boxes()
+ w.tick()
+ }
+ if r := w.row(second); r.Phase != "ready" || !r.CutRequestedAt.IsZero() || !r.AbortRequestedAt.IsZero() {
+ t.Fatalf("after the refused doors: %s cut %v abort %v", describe(r), r.CutRequestedAt, r.AbortRequestedAt)
+ }
+ if v := w.view(); v.State != "ready" || v.ID != second {
+ t.Fatalf("the poll after the refused doors: %+v", v)
+ }
+ if code, out := w.cut(second); code != http.StatusAccepted || parseView(t, out).State != "cutting" {
+ t.Fatalf("the cut naming the move = %d %s", code, out)
+ }
+ w.consoleUntil(5, "the cut taken", func() bool { return jobPhaseOrder[w.row(second).Phase] > jobPhaseOrder["ready"] })
+}
+
+// recreate deletes hostname's record and creates it again in a new
+// lifetime, as a delete and a create would: the host's incarnation slot at
+// inc, live, then the record at inc.
+func (w *jobWorld) recreate(hostname string, inc uint64) {
+ w.t.Helper()
+ if err := w.raw.Delete(w.ctx, provision.RecordKey(w.acct, host.E2EApp, hostname)); err != nil {
+ w.t.Fatal(err)
+ }
+ w.lifetimeHost(hostname, inc)
+}
+
+// lifetimeHost writes hostname's incarnation slot at inc, live, and creates
+// its record at inc (no replica on the boxes: its moves are followed only
+// so far as the console takes them).
+func (w *jobWorld) lifetimeHost(hostname string, inc uint64) {
+ w.t.Helper()
+ _, etag, err := provision.ReadHostIncarnation(w.ctx, w.raw, hostname)
+ if err != nil {
+ w.t.Fatal(err)
+ }
+ if _, err := provision.WriteHostIncarnation(w.ctx, w.raw, hostname, provision.HostIncarnation{Account: w.acct, App: host.E2EApp,
+ Incarnation: inc, State: provision.SlotLive, Intent: "create-e2e"}, etag); err != nil {
+ w.t.Fatal(err)
+ }
+ if err := provision.CreateRecord(w.ctx, w.raw, provision.Record{Schema: 1, Host: hostname, Slug: strings.SplitN(hostname, ".", 2)[0],
+ App: host.E2EApp, Account: w.acct, Channel: "stable", CreatedAt: time.Now().UTC(), Incarnation: inc}); err != nil {
+ w.t.Fatal(err)
+ }
+}
+
+// TestSelfServeRecreatedBetweenRowAndRecord: the member's request is
+// admitted (the row bound to the record's lifetime), and before the job
+// queues the move the instance is deleted and created again. The job
+// finishes the row aborted "the instance was recreated" and never writes
+// the new lifetime's record (no move, its sequence unchanged); nothing
+// stays reserved; and the member's poll of the new instance shows nothing
+// of the old lifetime's move.
+func TestSelfServeRecreatedBetweenRowAndRecord(t *testing.T) {
+ w := newMemberWorld(t)
+ id := w.prepare()
+ old := w.record().Incarnation
+ w.recreate(host.E2EHost, old+1)
+ fresh := w.record()
+ r := w.finished(id, 20)
+ after := w.record()
+ switch {
+ case r.Outcome != "aborted" || r.OutcomeReason != "the instance was recreated":
+ t.Fatalf("finished %s", describe(r))
+ case after.Incarnation != old+1 || after.Move != nil || after.LastMove != nil || after.RecordSeq != fresh.RecordSeq:
+ t.Fatalf("the new lifetime's record was written: %+v (seq %d, was %d)", after, after.RecordSeq, fresh.RecordSeq)
+ case len(w.reservations()) != 0:
+ t.Fatalf("reservations left: %v", w.reservations())
+ }
+ if v := w.view(); v.State != "none" || v.ID != "" || v.Last != nil {
+ t.Fatalf("the poll of the new instance shows the old move: %+v %+v", v, v.Last)
+ }
+}
+
+// TestSelfServeTwoMovesDisjointAndQueued: the member's move is mid-cut on
+// the two boxes. A second instance's move on the same boxes waits before
+// it reserves, its poll saying why; a third instance's move on two other
+// boxes reserves at once, beside the first. Once the first is done and its
+// boxes released, the second takes them. The orderings on their own are
+// TestTwoMovesOnOneBoxQueueInOrder, TestTwoMovesOnDisjointBoxesRunAtOnce
+// and TestQueueOrderAndDisjointProgress.
+func TestSelfServeTwoMovesDisjointAndQueued(t *testing.T) {
+ w := newMemberWorld(t)
+ first := w.cutByDoor()
+ w.run(200, "sealing", first, func(r *console.SelfServeMove) bool { return r.Phase == "sealing" || r.Outcome != "" })
+ const queued, disjoint = "b.shop.example.com", "c.shop.example.com"
+ w.lifetimeHost(queued, 1)
+ w.lifetimeHost(disjoint, 1)
+ other := provision.Participants{
+ Source: []provision.Participant{{BoxID: "x-box-id", Label: "x-box", InstanceID: "i-x"}},
+ Target: provision.Participant{BoxID: "y-box-id", Label: "y-box", InstanceID: "i-y"},
+ }
+ // The other boxes' reports are an hour old: a box reporting now with
+ // no edges file would make the fleet incomplete evidence for every
+ // move (readOfflineFleet), and only the reservation is followed here.
+ for _, p := range []provision.Participant{other.Source[0], other.Target} {
+ if err := status.WriteBox(w.ctx, w.raw, status.Box{Label: p.Label, GeneratedAt: time.Now().Add(-time.Hour).UTC(),
+ Metrics: &status.BoxMetrics{BoxID: p.BoxID, InstanceID: p.InstanceID}}); err != nil {
+ t.Fatal(err)
+ }
+ }
+ second := w.admitFor(queued, w.participants())
+ third := w.admitFor(disjoint, other)
+ w.consoleUntil(5, "the disjoint move reserved", func() bool { return w.reservedBy(third.ID) })
+ if !w.reservedBy(first) || w.reservedBy(second.ID) {
+ t.Fatalf("with the disjoint move reserved: %v", w.reservations())
+ }
+ s := w.row(second.ID)
+ v := w.viewOf(queued)
+ switch {
+ case s.Phase != "requested" && s.Phase != "reserving":
+ t.Fatalf("the queued move moved: %s", describe(s))
+ case s.Reason != "waiting for another move on this box":
+ t.Fatalf("the queued move's reason %q", s.Reason)
+ case v.State != "preparing" || v.ID != second.ID || v.Reason != s.Reason:
+ t.Fatalf("the queued move's poll %+v, its row's reason %q", v, s.Reason)
+ }
+ r := w.run(300, "the first done", first, func(r *console.SelfServeMove) bool {
+ if r.Outcome == "" && w.reservedBy(second.ID) {
+ t.Fatalf("the queued move reserved while the first is %s", r.Phase)
+ }
+ return r.Outcome != "" && !r.ReleasePending
+ })
+ if r.Outcome != "done" {
+ t.Fatalf("the first finished %s", describe(r))
+ }
+ w.consoleUntil(10, "the queued move reserved", func() bool {
+ return w.reservedBy(second.ID) || jobPhaseOrder[w.row(second.ID).Phase] > jobPhaseOrder["reserving"]
+ })
+}
+
+// putEntry writes active.json with e as its one entry, under key.
+func (w *jobWorld) putEntry(key string, e moves.OfflineReservation) {
+ w.t.Helper()
+ body, err := json.Marshal(moves.OfflineActiveBody(map[string]moves.OfflineReservation{key: e}))
+ if err != nil {
+ w.t.Fatal(err)
+ }
+ _, etag, _ := w.raw.Get(w.ctx, moves.OfflineActiveKey)
+ if _, err := w.raw.Put(w.ctx, moves.OfflineActiveKey, body, etag); err != nil {
+ w.t.Fatal(err)
+ }
+}
+
+// TestSelfServeReservationReclaimOnlyOnEvidence: a reservation of the
+// source box held by an operation the member's move cannot see. Older than
+// the grace period with a clean negative (its holder's record, fenced,
+// carries no move of it, and no row is open for it), the job reclaims it
+// after the fence (the holder's record_seq advanced) and the member's move
+// goes on. Young, it is waited on and kept, and reclaimed once old. Old
+// but with its holder's record carrying the move, it is kept with nothing
+// written to the holder, and reclaimed once the record no longer does.
+// Old with its holder's record unreadable, it is kept, and reclaimed once
+// the record reads. Every reclaim refusal of the rule on its own is
+// TestReclaimOnlyOnEvidence.
+func TestSelfServeReservationReclaimOnlyOnEvidence(t *testing.T) {
+ const holder = "b.shop.example.com"
+ const orphan = "mv_holderholderholderholderho"
+ old := time.Now().Add(-time.Hour).UTC()
+ entry := func(w *jobWorld, at time.Time) moves.OfflineReservation {
+ return moves.OfflineReservation{Kind: moves.OfflineKindMove, ID: orphan, Host: holder, Account: w.acct, App: host.E2EApp, ReservedAt: at}
+ }
+ // waits is the member's move kept in reserving over ticks, the entry
+ // kept, its poll saying why.
+ waits := func(w *jobWorld, id string) {
+ w.t.Helper()
+ for i := 0; i < 3; i++ {
+ w.boxes()
+ w.tick()
+ }
+ r := w.row(id)
+ switch {
+ case r.Phase != "reserving" || !w.reservedBy(orphan) || w.reservedBy(id):
+ w.t.Fatalf("the holder's entry was not kept: %s, reservations %v", describe(r), w.reservations())
+ case r.Reason != "waiting for another move on this box":
+ // The unreadable holder is retried without a reason.
+ if r.Reason != "" {
+ w.t.Fatalf("the waiting move's reason %q", r.Reason)
+ }
+ }
+ }
+ // reclaimed is the member's move past reserving on its own entry.
+ reclaimed := func(w *jobWorld, id string) {
+ w.t.Helper()
+ w.consoleUntil(5, "the move queued", func() bool { return jobPhaseOrder[w.row(id).Phase] > jobPhaseOrder["reserving"] })
+ if w.reservedBy(orphan) || !w.reservedBy(id) {
+ w.t.Fatalf("after the reclaim: %v", w.reservations())
+ }
+ }
+ for _, c := range []struct {
+ name string
+ setup func(w *jobWorld)
+ hold func(w *jobWorld, id string) // the kept half, then the evidence cleared
+ }{
+ {"no evidence: reclaimed after the fence", func(w *jobWorld) {}, nil},
+ {"young: kept, then reclaimed once old", func(w *jobWorld) {}, func(w *jobWorld, id string) {
+ w.putEntry(w.e.Box(host.E2ESourceBox).BoxID(), entry(w, time.Now().UTC()))
+ waits(w, id)
+ w.putEntry(w.e.Box(host.E2ESourceBox).BoxID(), entry(w, old))
+ }},
+ {"the holder's record carries the move: kept, nothing written", func(w *jobWorld) {
+ if _, err := provision.SetRecordMove(w.ctx, w.raw, w.acct, host.E2EApp, holder, func(rec *provision.Record) error {
+ rec.Move = &provision.MoveState{ID: orphan, Kind: provision.MoveKindOffline, State: provision.OfflineQueued, To: host.E2ETargetRegion}
+ return nil
+ }); err != nil {
+ w.t.Fatal(err)
+ }
+ }, func(w *jobWorld, id string) {
+ seq := w.recordOf(holder).RecordSeq
+ waits(w, id)
+ if got := w.recordOf(holder).RecordSeq; got != seq {
+ w.t.Fatalf("the live holder's record was written: seq %d, was %d", got, seq)
+ }
+ if _, err := provision.SetRecordMove(w.ctx, w.raw, w.acct, host.E2EApp, holder, func(rec *provision.Record) error {
+ rec.Move = nil
+ return nil
+ }); err != nil {
+ w.t.Fatal(err)
+ }
+ }},
+ {"the holder's record unreadable: kept, then reclaimed once it reads", func(w *jobWorld) {}, func(w *jobWorld, id string) {
+ key := provision.RecordKey(w.acct, host.E2EApp, holder)
+ good, etag, err := w.raw.Get(w.ctx, key)
+ if err != nil {
+ w.t.Fatal(err)
+ }
+ etag, err = w.raw.Put(w.ctx, key, []byte("{"), etag)
+ if err != nil {
+ w.t.Fatal(err)
+ }
+ waits(w, id)
+ if _, err := w.raw.Put(w.ctx, key, good, etag); err != nil {
+ w.t.Fatal(err)
+ }
+ }},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ w := newMemberWorld(t)
+ w.lifetimeHost(holder, 1)
+ c.setup(w)
+ w.putEntry(w.e.Box(host.E2ESourceBox).BoxID(), entry(w, old))
+ seq := w.recordOf(holder).RecordSeq
+ id := w.prepare()
+ if c.hold != nil {
+ c.hold(w, id)
+ seq = w.recordOf(holder).RecordSeq
+ }
+ reclaimed(w, id)
+ if got := w.recordOf(holder).RecordSeq; got <= seq {
+ t.Fatalf("reclaimed without fencing the holder: seq %d, was %d", got, seq)
+ }
+ if r := w.phase(id, "ready", 60); r.Outcome != "" {
+ t.Fatalf("the move after the reclaim: %s", describe(r))
+ }
+ })
+ }
+}