amadan

carlos / platform Public

console: lifetimes end to end across a landed create's recoveries, a late legacy create and a replayed create interleaved with a move and a retirement

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
claude 44438709d03461b1950f990d972a65f6917df57c parent 8e38e78
1 file changed, +452 −0
  • internal/console/lifetimes_e2e_test.go +452 −0
diff --git a/internal/console/lifetimes_e2e_test.go b/internal/console/lifetimes_e2e_test.go
new file mode 100644
index 000000000..c04e20ffd
--- /dev/null
+++ b/internal/console/lifetimes_e2e_test.go
@@ -0,0 +1,452 @@
+package console
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "net/http"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// Lifetimes end to end on the console (self-serve moves plan 1, Task 9):
+// the interleavings of design §11 that cross a create, its recovery, a move
+// and a retirement, driven through the lifetimes-on console of
+// lifetimes_create_test.go (newLtOn: the real server, doors, SQLite and a
+// bucket that can fail, pause or record any call) and restarted consoles
+// over the same SQLite and bucket.
+//
+// A move here is the offline doors' own record write (offlineApply, the
+// function every offline door ends in) applied in the doors' order, start
+// to finish: the doors' box-evidence checks are the move's business, which
+// TestOfflineMoveE2E drives against real boxes; what these witnesses need
+// is the record the move leaves. A box's view of the bucket is one
+// claims-off box reconcile pass (provision.Reconcile with a lifetime judge,
+// as the agent's reconcileSerializedWith runs it) into a registry of its
+// own.
+
+// moveThrough runs one whole offline move of host, from its region to to,
+// through srv's offline record write in a unit admitted on.
+func (l ltOn) moveThrough(t *testing.T, srv *Server, host, id, to string) provision.Record {
+ t.Helper()
+ ctx := onCtx(t, srv)
+ d := offlineDoor{acct: l.acct(t), app: ltApp, host: host}
+ raw, _ := rawOf(t, l.fx.rel, ltApp, host)
+ from := decodeRec(t, raw).Region
+ now := l.clock.Now()
+ var rec *provision.Record
+ for _, mut := range []func(*provision.Record) error{
+ provision.OfflineStart(id, from, to, "paul@keymail.dev", now),
+ provision.OfflineSeal(id, "ie-a", []string{"ie-a"}),
+ provision.OfflineSealedBy(id, "ie-a", provision.MoveSource{}),
+ provision.OfflineFlip(id),
+ provision.OfflineLandedBy(id, to),
+ provision.OfflineOpenTransition(id, now),
+ provision.OfflineFinishingTransition(id),
+ provision.OfflineFinish(id, now),
+ } {
+ var err error
+ if rec, err = srv.offlineApply(ctx, d, mut); err != nil {
+ t.Fatalf("the move of %s: %v", host, err)
+ }
+ }
+ if rec.Move != nil || rec.MoveSeq < 3 || rec.LastMove == nil || rec.LastMove.ID != id {
+ t.Fatalf("after the move the record is %+v", rec)
+ }
+ return *rec
+}
+
+// ltBox is one claims-off box over the console's bucket: its own registry
+// and data directory.
+type ltBox struct {
+ reg *registry.Registry
+ dd string
+ // claimsOn: the box judges nothing (a claims-on box, the control).
+ claimsOn bool
+}
+
+func newLtBox(t *testing.T) ltBox {
+ t.Helper()
+ dd := t.TempDir()
+ reg, err := registry.Open(filepath.Join(dd, "registry.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { reg.Close() })
+ return ltBox{reg: reg, dd: dd}
+}
+
+// pass is one reconcile pass of the box in region over l's bucket, judged
+// as a claims-off agent judges it, and the box's row for host afterwards
+// (nil: none).
+func (b ltBox) pass(t *testing.T, l ltOn, region, host string) *registry.Route {
+ t.Helper()
+ cfg := provision.ReconcileConfig{Region: region, DataDir: b.dd}
+ if !b.claimsOn {
+ cfg.Judge = provision.NewLifetimeJudge()
+ }
+ if err := provision.Reconcile(context.Background(), l.fx.rel, b.reg, cfg); err != nil {
+ t.Fatalf("the box's reconcile: %v", err)
+ }
+ rt, err := b.reg.Get(host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return rt
+}
+
+// requireValid: srv's judged readers in a unit admitted on read host's
+// record, at incarnation n with ETag etag.
+func requireValid(t *testing.T, srv *Server, host string, n uint64, etag, when string) {
+ t.Helper()
+ ctx := onCtx(t, srv)
+ rec, got, err := srv.readRecord(ctx, "bab", ltApp, host)
+ if err != nil || rec.Incarnation != n || got != etag {
+ t.Fatalf("%s: readRecord = %+v %q %v, want incarnation %d at %q", when, rec, got, err, n, etag)
+ }
+ if obs := srv.observeRecord(ctx, "bab", ltApp, host); !obs.Present || obs.Verdict != provision.LifetimeValid {
+ t.Fatalf("%s: observeRecord = present %v, %v; want valid", when, obs.Present, obs.Verdict)
+ }
+ recs, _, err := srv.listRecords(ctx, "bab", ltApp, instanceListMax)
+ if err != nil || !strings.Contains(","+recordHostsOf(recs)+",", ","+host+",") {
+ t.Fatalf("%s: listRecords = %s %v, want %s in it", when, recordHostsOf(recs), err, host)
+ }
+}
+
+// TestLandedCreateThenSecondRecoveryCrash is §11's "a create whose record
+// landed and is served, a crash before the allocation commits, and
+// recovery marking it committed with the account still serving, including
+// a second crash during recovery".
+func TestLandedCreateThenSecondRecoveryCrash(t *testing.T) {
+ l := newLtOn(t)
+ box := newLtBox(t)
+ a := l.pendingCreate(t, ltOne, "commit") // the record landed; the commit did not
+ raw, etag := rawOf(t, l.fx.rel, ltApp, ltOne)
+ if raw == nil {
+ t.Fatal("fixture: the record did not land before the commit failed")
+ }
+ _, slotETag := slotOf(t, l.fx.rel, ltOne)
+ // served: every judged reader of the console, the instance door and a
+ // box all see the record.
+ unchanged := func(when string) {
+ t.Helper()
+ if now, tag := rawOf(t, l.fx.rel, ltApp, ltOne); tag != etag || !bytes.Equal(now, raw) {
+ t.Fatalf("%s: the record moved: %s", when, now)
+ }
+ if _, tag := slotOf(t, l.fx.rel, ltOne); tag != slotETag {
+ t.Fatalf("%s: the slot moved", when)
+ }
+ if rt := box.pass(t, l, "", ltOne); rt == nil || rt.Status != "ready" || rt.Incarnation != 1 {
+ t.Fatalf("%s: the box's row is %+v, want ready at incarnation 1", when, rt)
+ }
+ }
+ requireValid(t, l.fx.srv, ltOne, 1, etag, "after the crash")
+ if code, _, body := rawReq(t, http.MethodGet, instancesURL(l.fx, "bab", ltApp)+"/"+ltOne, l.member, ""); code != http.StatusOK {
+ t.Fatalf("GET after the crash = %d %s", code, body)
+ }
+ unchanged("after the crash")
+
+ // The first recovery, on a restarted console, crashes at the same
+ // point: the landed branch's commit fails.
+ second := l.restart(t)
+ drop := sqlFailCommit(t, l.fx.st)
+ recoverOn(t, second)
+ drop()
+ oneAlloc(t, l.fx.st, ltOne, lifetimePending)
+ requireValid(t, second, ltOne, 1, etag, "after the second crash")
+ unchanged("after the second crash")
+ // The hostname's creates wait on the pending allocation meanwhile.
+ if code, body := l.create(t, ltOne); code != http.StatusConflict || ltErr(body) != msgLifetimeSettling {
+ t.Fatalf("a create while pending = %d %s, want 409 settling", code, body)
+ }
+
+ // A third console's recovery commits it; nothing in the bucket moves.
+ third := l.restart(t)
+ recoverOn(t, third)
+ got := oneAlloc(t, l.fx.st, ltOne, lifetimeCommitted)
+ if got.OpID != a.OpID {
+ t.Fatalf("committed %s, want %s", got.OpID, a.OpID)
+ }
+ requireServing(t, l.fx.rel, ltApp, ltOne, 1, a.OpID)
+ requireValid(t, third, ltOne, 1, etag, "after recovery")
+ unchanged("after recovery")
+}
+
+// TestLateLegacyCreateIsInert is §11's "an old console's fieldless create
+// landing after an upgrade, a retry, a move and a retirement, ignored by
+// every consumer and removed by the next create".
+func TestLateLegacyCreateIsInert(t *testing.T) {
+ l := newLtOn(t) // the upgrade: lifetimes on
+ srv := l.fx.srv
+ // The old console's create was issued before the upgrade and is still
+ // in flight. The user retries on the new console, the instance moves
+ // away and back, and it is deleted.
+ if rec := l.mustCreate(t, ltOne); rec.Incarnation != 1 {
+ t.Fatalf("the retry created incarnation %d", rec.Incarnation)
+ }
+ l.moveThrough(t, srv, ltOne, "mv_"+strings.Repeat("a", 26), "uk")
+ l.moveThrough(t, srv, ltOne, "mv_"+strings.Repeat("d", 26), "")
+ // A home box serves the instance back home from the record, minted at
+ // the second move's (1, 6); it is down from here until the stale body
+ // has landed.
+ box, control := newLtBox(t), newLtBox(t)
+ control.claimsOn = true
+ if rt := box.pass(t, l, "", ltOne); rt == nil || rt.Incarnation != 1 || rt.ServingSeq != 6 {
+ t.Fatalf("the home box's row after the moves = %+v, want (1, 6)", rt)
+ }
+ if rt := control.pass(t, l, "", ltOne); rt == nil {
+ t.Fatal("fixture: the control box has no row")
+ }
+ if code, body := l.del(t, ltOne); code != http.StatusOK {
+ t.Fatalf("delete = %d %s", code, body)
+ }
+ requireRetired(t, l.fx.rel, ltApp, ltOne, 1)
+
+ // Now the old console's create-only write lands on the absent key.
+ staleETag := legacyBody(t, l.fx.rel, ltApp, ltOne, "edge")
+ staleRaw, _ := rawOf(t, l.fx.rel, ltApp, ltOne)
+ slot, slotETag := slotOf(t, l.fx.rel, ltOne)
+
+ // Ignored by every consumer.
+ ctx := onCtx(t, srv)
+ if _, _, err := provision.ReadRecordJudged(context.Background(), srv.cfg.Releases, srv.judge, "bab", ltApp, ltOne); !errors.Is(err, os.ErrNotExist) {
+ t.Errorf("ReadRecordJudged = %v, want not found", err)
+ }
+ if _, _, err := srv.readRecord(ctx, "bab", ltApp, ltOne); !errors.Is(err, os.ErrNotExist) {
+ t.Errorf("readRecord = %v, want not found", err)
+ }
+ if obs := srv.observeRecord(ctx, "bab", ltApp, ltOne); !obs.Present || obs.Verdict != provision.LifetimeStale {
+ t.Errorf("observeRecord = present %v, %v; want a present stale body", obs.Present, obs.Verdict)
+ }
+ if recs, _, err := srv.listRecords(ctx, "bab", ltApp, instanceListMax); err != nil || len(recs) != 0 {
+ t.Errorf("listRecords = %s %v, want none", recordHostsOf(recs), err)
+ }
+ if recs, unreadable, err := srv.listAllRecordsStrict(ctx); err != nil || len(unreadable) != 0 || len(recs) != 0 {
+ t.Errorf("listAllRecordsStrict = %s %v %v, want none", recordHostsOf(recs), unreadable, err)
+ }
+ requireListing(t, l.lifetimesFx)
+ if code, _, body := rawReq(t, http.MethodGet, instancesURL(l.fx, "bab", ltApp)+"/"+ltOne, l.member, ""); code != http.StatusNotFound {
+ t.Errorf("GET = %d %s, want 404", code, body)
+ }
+ rec := decodeRec(t, staleRaw)
+ rec.Gate = "maintenance"
+ if err := provision.UpdateRecord(ctx, srv.cfg.Releases, rec, staleETag); !errors.Is(err, os.ErrNotExist) {
+ t.Errorf("UpdateRecord = %v, want not found", err)
+ }
+ if _, err := srv.offlineApply(ctx, offlineDoor{acct: l.acct(t), app: ltApp, host: ltOne},
+ provision.OfflineStart("mv_"+strings.Repeat("b", 26), "", "uk", "paul@keymail.dev", l.clock.Now())); !errors.Is(err, os.ErrNotExist) {
+ t.Errorf("the offline start's write = %v, want not found", err)
+ }
+ srv.cfg.LifetimesMigrate = true
+ if !srv.incarnationMigrationIteration(context.Background()) {
+ t.Fatal("the migration pass did not run with lifetimes on")
+ }
+ if rows := migRows(t, l.fx.st, ltOne); len(rows) != 0 {
+ t.Errorf("migrations = %+v, want none", rows)
+ }
+ recoverOn(t, srv)
+ // The box, back from its outage, removes the row of the lifetime that
+ // ended rather than keeping it for the body now under the key (the
+ // control, a claims-on box, keeps it: unjudged the body is a valid
+ // legacy record of its region), and a fresh box mints nothing from it.
+ if rt := box.pass(t, l, "", ltOne); rt != nil {
+ t.Errorf("the box kept lifetime 1's row beside the stale body: %+v", rt)
+ }
+ if rt := newLtBox(t).pass(t, l, "", ltOne); rt != nil {
+ t.Errorf("a fresh claims-off box minted the stale body: %+v", rt)
+ }
+ if rt := control.pass(t, l, "", ltOne); rt == nil {
+ t.Fatal("fixture: unjudged, the box drops the row too, so the judged pass proves nothing")
+ }
+ if now, tag := rawOf(t, l.fx.rel, ltApp, ltOne); tag != staleETag || !bytes.Equal(now, staleRaw) {
+ t.Fatalf("a consumer wrote the stale body:\n%s", now)
+ }
+ if now, tag := slotOf(t, l.fx.rel, ltOne); tag != slotETag || *now != *slot {
+ t.Fatalf("a consumer wrote the slot: %+v", now)
+ }
+
+ // Removed by the next create, on its own ETag.
+ if code, body := l.create(t, ltOne); code != http.StatusCreated {
+ t.Fatalf("the next create = %d %s", code, body)
+ }
+ var next allocRow
+ for _, a := range allocRows(t, l.fx.st, ltOne) {
+ if a.Incarnation == 2 {
+ next = a
+ }
+ }
+ if next.State != lifetimeCommitted || next.Stale != staleETag {
+ t.Fatalf("the next create's allocation = %+v, want committed at 2 over the stale %q", next, staleETag)
+ }
+ if !contains(l.bucket.opsOf(recKey(ltApp, ltOne)), "delete-if-match") {
+ t.Fatal("the stale body was not deleted on its ETag")
+ }
+ created := requireServing(t, l.fx.rel, ltApp, ltOne, 2, next.OpID)
+ if created.Channel == "edge" {
+ t.Fatalf("the record is the stale body's: %+v", created)
+ }
+ if rt := box.pass(t, l, "", ltOne); rt == nil || rt.Incarnation != 2 || rt.ServingSeq != 0 {
+ t.Fatalf("the box's row after the next create = %+v, want (2, 0)", rt)
+ }
+}
+
+// lateAnswer is what the first console's create door answered, late.
+type lateAnswer struct {
+ code int
+ body string
+}
+
+// replayedAndMoved is the interleave's start: the first console's
+// create-only record write held in flight, a second console replaying the
+// create from SQLite, and a completed move. It returns the second console,
+// the held write's gate and the first console's answer to come.
+func replayedAndMoved(t *testing.T, l ltOn) (*Server, *pauseGate, <-chan lateAnswer) {
+ t.Helper()
+ gate, hook := pauseNth(t, "put", recKey(ltApp, ltOne), 1)
+ l.bucket.setHook(hook)
+ answered := make(chan lateAnswer, 1)
+ go func() { code, body := l.create(t, ltOne); answered <- lateAnswer{code, body} }()
+ waitFor(t, gate.entered, "the first console's record write")
+
+ // A second console replays the create from SQLite.
+ second := l.restart(t)
+ recoverOn(t, second)
+ a := oneAlloc(t, l.fx.st, ltOne, lifetimeCommitted)
+ requireServing(t, l.fx.rel, ltApp, ltOne, 1, a.OpID)
+
+ // A completed move.
+ if rec := l.moveThrough(t, second, ltOne, "mv_"+strings.Repeat("c", 26), "uk"); rec.MoveSeq != 3 {
+ t.Fatalf("after the move move_seq is %d", rec.MoveSeq)
+ }
+ return second, gate, answered
+}
+
+// TestDelayedCreateReplayMoveAndRetirementInterleave is §11's "a replayed
+// create, a completed move, a delete that crashes between the tombstone and
+// the slot, and the late original create failing on the tombstone, then the
+// slot retired and the tombstone removed". The second subtest is the other
+// half of §6.4's sentence on it: once the tombstone is gone the slot is
+// already retired, so the late write that lands then is stale to every
+// consumer, and the next create replaces it.
+func TestDelayedCreateReplayMoveAndRetirementInterleave(t *testing.T) {
+ t.Run("the late create fails on the tombstone", func(t *testing.T) {
+ l := newLtOn(t)
+ key, slotKey := recKey(ltApp, ltOne), provision.HostIncarnationKey(ltOne)
+ second, gate, answered := replayedAndMoved(t, l)
+ // A box in uk serves the moved instance.
+ box := newLtBox(t)
+ if rt := box.pass(t, l, "uk", ltOne); rt == nil || rt.Incarnation != 1 || rt.ServingSeq != 3 {
+ t.Fatalf("the uk box's row after the move = %+v, want (1, 3)", rt)
+ }
+
+ // The delete crashes between the tombstone and the slot.
+ l.bucket.setHook(failNth("put", slotKey, 1, false))
+ if err := retireOn(t, second, ltOne, retireAny); !errors.Is(err, errLifetimePending) {
+ t.Fatalf("the delete = %v, want pending", err)
+ }
+ l.bucket.setHook(nil)
+ r := lastRetirement(t, l.fx.st, ltOne)
+ if r.State != lifetimePending {
+ t.Fatalf("retirement = %+v, want pending", r)
+ }
+ tombRaw, tombETag := rawOf(t, l.fx.rel, ltApp, ltOne)
+ if tomb := decodeRec(t, tombRaw); !tomb.Retired || tomb.Incarnation != 1 {
+ t.Fatalf("under the key after the crash: %s, want the tombstone of 1", tombRaw)
+ }
+ if slot, _ := slotOf(t, l.fx.rel, ltOne); slot == nil || slot.State != provision.SlotLive || slot.Incarnation != 1 {
+ t.Fatalf("slot after the crash = %+v, want {1, live}", slot)
+ }
+ // The tombstone is absent to the box under the live slot.
+ if rt := box.pass(t, l, "uk", ltOne); rt != nil {
+ t.Fatalf("the uk box kept its row beside the tombstone: %+v", rt)
+ }
+
+ // The late original create arrives: its create-only write fails on
+ // the tombstone, and the first console answers that it could not
+ // confirm.
+ gate.open()
+ late := <-answered
+ if late.code != http.StatusBadGateway || ltErr(late.body) != msgLifetimeCreatePending {
+ t.Fatalf("the late create answered %d %s, want 502 %q", late.code, late.body, msgLifetimeCreatePending)
+ }
+ if now, tag := rawOf(t, l.fx.rel, ltApp, ltOne); tag != tombETag || !bytes.Equal(now, tombRaw) {
+ t.Fatalf("the late create changed the tombstone: %s", now)
+ }
+ if rows := allocRows(t, l.fx.st, ltOne); len(rows) != 1 || rows[0].State != lifetimeCommitted {
+ t.Fatalf("allocations after the late create = %+v, want the one, committed", rows)
+ }
+
+ if rt := box.pass(t, l, "uk", ltOne); rt != nil {
+ t.Fatalf("the uk box minted a row after the late create: %+v", rt)
+ }
+
+ // Recovery: the slot retired, then the tombstone removed.
+ l.recoveredTo(t, r.OpID, lifetimeCommitted)
+ requireRetired(t, l.fx.rel, ltApp, ltOne, 1)
+ if ops := l.bucket.opsOf(key); !contains(ops, "delete-if-match") {
+ t.Fatalf("record ops = %v, want the tombstone deleted on its ETag", ops)
+ }
+ // The hostname's next lifetime is 2.
+ if rec := l.mustCreate(t, ltOne); rec.Incarnation != 2 {
+ t.Fatalf("the next create is at %d, want 2", rec.Incarnation)
+ }
+ })
+
+ t.Run("the late create lands after the tombstone went", func(t *testing.T) {
+ l := newLtOn(t)
+ second, gate, answered := replayedAndMoved(t, l)
+ l.bucket.setHook(nil)
+ if err := retireOn(t, second, ltOne, retireAny); err != nil {
+ t.Fatalf("the delete = %v", err)
+ }
+ requireRetired(t, l.fx.rel, ltApp, ltOne, 1)
+
+ // The late original create's write lands on the absent key: the
+ // record of lifetime 1 is back under the slot {1, retired}.
+ gate.open()
+ late := <-answered
+ raw, etag := rawOf(t, l.fx.rel, ltApp, ltOne)
+ if raw == nil || decodeRec(t, raw).Incarnation != 1 {
+ t.Fatalf("fixture: the late write did not land (answered %d %s): %s", late.code, late.body, raw)
+ }
+ // The first console's commit meets the allocation recovery already
+ // committed, so its door cannot confirm the create.
+ if late.code != http.StatusBadGateway || ltErr(late.body) != msgLifetimeCreatePending {
+ t.Fatalf("the late create answered %d %s, want 502 %q", late.code, late.body, msgLifetimeCreatePending)
+ }
+ if rows := allocRows(t, l.fx.st, ltOne); len(rows) != 1 || rows[0].State != lifetimeCommitted {
+ t.Fatalf("allocations after the late create = %+v, want the one, committed", rows)
+ }
+ // Stale to every consumer.
+ ctx := onCtx(t, second)
+ if _, _, err := second.readRecord(ctx, "bab", ltApp, ltOne); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("readRecord = %v, want not found", err)
+ }
+ if code, _, body := rawReq(t, http.MethodGet, instancesURL(l.fx, "bab", ltApp)+"/"+ltOne, l.member, ""); code != http.StatusNotFound {
+ t.Fatalf("GET = %d %s, want 404", code, body)
+ }
+ if rt := newLtBox(t).pass(t, l, "uk", ltOne); rt != nil {
+ t.Fatalf("a box minted the late record: %+v", rt)
+ }
+ // The next create replaces it, on its ETag.
+ code, body := l.create(t, ltOne)
+ if code != http.StatusCreated {
+ t.Fatalf("the next create = %d %s", code, body)
+ }
+ var next allocRow
+ for _, a := range allocRows(t, l.fx.st, ltOne) {
+ if a.Incarnation == 2 {
+ next = a
+ }
+ }
+ if next.State != lifetimeCommitted || next.Stale != etag {
+ t.Fatalf("the next create's allocation = %+v, want committed at 2 over the late body's %q", next, etag)
+ }
+ requireServing(t, l.fx.rel, ltApp, ltOne, 2, next.OpID)
+ })
+}