amadan

carlos / platform Public

moves: port wRollbackBaselinePreserved and wReturnLostRecord (§5.1 step 9, §6.4)

The rollback baseline: a landing's probe publication leaves the source's
generation unqueued and the target's record uncovered until Commit; the
real rollback, release and clean return find the generation untouched in
the ie bucket fake. The lost record: a dirty return whose source loses its
guard record at claimed re-initialises at S and installs under S, never F.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
claude 30972b9f2ba5c35c767f89a10802f38732b4ce75 parent 2ef8400
2 files changed, +378 −0
  • internal/host/movesrollback_witness_test.go +243 −0
  • internal/hostclaim/move_witness_test.go +135 −0
diff --git a/internal/host/movesrollback_witness_test.go b/internal/host/movesrollback_witness_test.go
new file mode 100644
index 000000000..361e1e6e0
--- /dev/null
+++ b/internal/host/movesrollback_witness_test.go
@@ -0,0 +1,243 @@
+package host
+
+import (
+ "context"
+ "errors"
+ "os"
+ "path/filepath"
+ "reflect"
+ "strconv"
+ "strings"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/hostguard"
+ "github.com/carlosframework/platform/internal/provision"
+)
+
+// Two model witnesses ported onto Task 4's target-side rollback harness and
+// Task 5's source-side recovery harness (moves spec §5.1 step 9 and §6.4;
+// Plan 4 Task 8). The claim half of the first is in hostclaim's
+// move_witness_test.go.
+
+// sourceGenerationObjects are the objects of the source's generation "cur"
+// in the source region's bucket, by key.
+func sourceGenerationObjects(h *recoverHarness) map[string]fakeMoveObject {
+ out := map[string]fakeMoveObject{}
+ for k, o := range h.f.buckets["data-ie"] {
+ if strings.Contains(k, "/g-cur/") {
+ out[k] = o
+ }
+ }
+ return out
+}
+
+// TestRollbackBaselineSurvivesTheCleanReturn is wRollbackBaselinePreserved
+// (claim rev 6.14 item 5, R12-3; spec §5.1 step 9 and §6.4): a landing's
+// probe publishes the target's generation in uk, and the source's
+// generation, the target's predecessor in ie, is not queued for a park by
+// PublishGeneration. The target's admission record stays uncovered, no
+// obligation reaches ie, and the real sequence RollBack, TargetReleased and
+// ReturnToSource(clean) then finds the source's generation exactly as the
+// cut left it in the ie bucket fake, so the source restores from it. Only
+// Commit covers the record (hostclaim's half of the witness).
+func TestRollbackBaselineSurvivesTheCleanReturn(t *testing.T) {
+ ctx := context.Background()
+ h := newRecoverHarness(t, false)
+ baseline := sourceGenerationObjects(h)
+ if len(baseline) == 0 {
+ t.Fatal("fixture: the source's generation holds no objects in ie")
+ }
+
+ // The landing's probe admission, with the claim library's own
+ // transition: cold, in uk, its predecessor the source's generation in ie.
+ srcGen := h.claim(t).Op.Journal[hostclaim.JournalSrcGen]
+ if srcGen != "cur" {
+ t.Fatalf("fixture: src_gen %q", srcGen)
+ }
+ obs, err := hostclaim.Read(ctx, h.s, p1Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ op, holder := obs.Claim.Op, obs.Claim.Holder
+ if _, err := hostclaim.Update(ctx, h.s, obs, hostclaim.AdmitWriter(hostclaim.Admission{
+ ObservedOpID: op.ID, ObservedPhase: op.Phase, ObservedSession: holder.Session, ObservedGeneration: holder.Generation,
+ Box: "uk-box", Mode: "cold", NewGeneration: p1ID, Coverage: "sealed", Region: "uk", PredecessorRegion: "ie", KeepStamp: true,
+ })); err != nil {
+ t.Fatal(err)
+ }
+
+ // kept is the witness at one point of the flow.
+ kept := func(where string) {
+ t.Helper()
+ c := h.claim(t)
+ var target *hostclaim.AdmissionRecord
+ for i := range c.Holder.Admissions {
+ if a := &c.Holder.Admissions[i]; a.Box == "uk-box" && a.Mode == "cold" {
+ target = a
+ }
+ }
+ if target == nil || target.Region != "uk" || target.PredecessorRegion != "ie" || target.Predecessor != srcGen {
+ t.Fatalf("%s: the target's record does not name the source's generation in ie: %+v", where, c.Holder.Admissions)
+ }
+ if target.Covered {
+ t.Fatalf("%s: the target's record is covered: %+v", where, target)
+ }
+ for _, o := range c.Obligations {
+ if o.Region == "ie" || o.Box == "ie-box" {
+ t.Fatalf("%s: an obligation reaches the source's region: %+v", where, o)
+ }
+ }
+ if got := sourceGenerationObjects(h); !reflect.DeepEqual(got, baseline) {
+ t.Fatalf("%s: the source's generation changed in ie: %d objects, was %d", where, len(got), len(baseline))
+ }
+ }
+
+ // The probe's publication, as the claim library has it on main.
+ obs, err = hostclaim.Read(ctx, h.s, p1Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ op, holder = obs.Claim.Op, obs.Claim.Holder
+ if holder.Generation != p1ID {
+ t.Fatalf("fixture: holder.generation %q", holder.Generation)
+ }
+ publish := hostclaim.PublishGeneration(op.ID, op.Phase, holder.Session, holder.Generation, "uk", func() string {
+ t.Error("the target's publication allocated an obligation id")
+ return "id"
+ })
+ if _, err := hostclaim.Update(ctx, h.s, obs, publish); err != nil {
+ t.Fatal(err)
+ }
+ kept(t.Name() + " after the target's publication")
+
+ // The real sequence: the rollback, the target's release, the source's
+ // clean return, through to opened.
+ rb := h.rollBack(t)
+ kept("after the rollback decision")
+ h.rollbackPass(t)
+ if got := h.rb(t); got.TargetPhase != provision.RbTargetReleased || got.Dirty {
+ t.Fatalf("the target did not release clean: %+v", got)
+ }
+ kept("after the target's release")
+ h.srcPass(t)
+ h.opened(t)
+ kept("after the clean return")
+
+ c := h.claim(t)
+ if c.Holder.Generation != srcGen || c.Op.Journal[hostclaim.JournalSrcGen] != srcGen {
+ t.Fatalf("the return names generation %q, want the source's %q", c.Holder.Generation, srcGen)
+ }
+ if len(h.probes) != 1 || !h.probes[0].Clean || h.probes[0].SourceGeneration != srcGen || h.probes[0].RollbackID != rb.ID {
+ t.Fatalf("the recovery probe was not asked to restore from the source's generation: %+v", h.probes)
+ }
+}
+
+// loseGuardRecords is the lost-record fault: the source's installation
+// record is gone, either the host's file or the whole hostguard directory
+// (re-provisioned empty, as the box's boot does).
+func loseGuardRecords(t *testing.T, dataDir string, wholeDir bool) {
+ t.Helper()
+ dir := filepath.Join(dataDir, "hostguard")
+ if !wholeDir {
+ ents, err := os.ReadDir(dir)
+ if err != nil || len(ents) == 0 {
+ t.Fatalf("the guard directory holds no record to lose: %v %v", ents, err)
+ }
+ for _, e := range ents {
+ if err := os.RemoveAll(filepath.Join(dir, e.Name())); err != nil {
+ t.Fatal(err)
+ }
+ }
+ return
+ }
+ if err := os.RemoveAll(dir); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.MkdirAll(dir, 0o770); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestDirtyReturnAfterALostGuardRecord is wReturnLostRecord (claim rev 6.5,
+// R6-4; spec §5.1 step 9 and §6.4): a dirty rollback whose source loses its
+// installation record after the return's claim. The claimed phase had not
+// yet been left, so the source's recovery replays it: ReturnToSource wrote
+// S, the pre-move stamp, back into holder.stamp, so Guard.Activate
+// re-initialises the record from the claim at S, never at F (the target's
+// stamp), and Task 3's installer, entered through recoverSource, tags every
+// artifact it registers with S. The source ends opened at S with no path
+// tagged F.
+func TestDirtyReturnAfterALostGuardRecord(t *testing.T) {
+ for _, wholeDir := range []bool{false, true} {
+ for _, registered := range []bool{false, true} {
+ name := map[bool]string{false: "the record file", true: "the whole directory"}[wholeDir] + ", " + map[bool]string{false: "unregistered", true: "registered"}[registered]
+ t.Run(name, func(t *testing.T) {
+ h := newRecoverHarness(t, true)
+ if registered {
+ rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) {
+ c.Inventory = map[string]hostclaim.InventoryEntry{"ie-box": {Layout: "flat", Paths: map[string]uint64{p1Host + ".db": 1, p1Host + ".db.gen": 1, p1Host: 1, p1Host + ".env": 1}}}
+ })
+ }
+ rb := h.released(t)
+ c := h.claim(t)
+ s, err := strconv.ParseUint(c.Op.Journal[hostclaim.JournalSourceStamp], 10, 64)
+ if err != nil || s == 0 {
+ t.Fatalf("source_stamp %q: %v", c.Op.Journal[hostclaim.JournalSourceStamp], err)
+ }
+ f, err := strconv.ParseUint(c.Op.Journal[hostclaim.JournalTargetStamp], 10, 64)
+ if err != nil || f == 0 || f == s {
+ t.Fatalf("target_stamp %q, source_stamp %d: %v", c.Op.Journal[hostclaim.JournalTargetStamp], s, err)
+ }
+
+ // The record is lost right after the return's claim and the
+ // activation, before the record leaves claimed.
+ h.rcAt = "after-activate"
+ h.rcHook = func(point string) {
+ if point == "after-activate" {
+ loseGuardRecords(t, h.sdir, wholeDir)
+ }
+ }
+ h.srcPass(t)
+ h.sourceAt(t, provision.RbSourceClaimed)
+ if _, err := hostguard.Capture(h.sdir, p1Host); !errors.Is(err, hostguard.ErrNoRecord) && !errors.Is(err, hostguard.ErrNotProvisioned) {
+ t.Fatalf("the installation record survived the loss: %v", err)
+ }
+ if c := h.claim(t); c.Op.Phase != hostclaim.PhaseClaimed || c.Op.Exec != "ie-box" || c.Holder.Stamp != s {
+ t.Fatalf("claim after the loss: %+v holder stamp %d, want claimed by ie-box at %d", c.Op, c.Holder.Stamp, s)
+ }
+
+ // The replay re-initialises from the claim and recovers.
+ h.rcHook = nil
+ h.srcPass(t)
+ got := h.opened(t)
+ if got.ID != rb.ID {
+ t.Fatalf("opened %s, want %s", got.ID, rb.ID)
+ }
+ captured, err := hostguard.Capture(h.sdir, p1Host)
+ if err != nil || captured.Stamp != s || captured.Incarnation != h.claim(t).Holder.Incarnation {
+ t.Fatalf("the re-initialised record: %+v %v, want stamp %d (never %d)", captured, err, s, f)
+ }
+ c = h.claim(t)
+ if c.Holder.Stamp != s {
+ t.Fatalf("holder.stamp %d, want S = %d", c.Holder.Stamp, s)
+ }
+ // Everything the install registered on the source is tagged S,
+ // and none of it F (the target's own entries are its to park).
+ entry, ok := c.Inventory["ie-box"]
+ if !ok || len(entry.Paths) == 0 {
+ t.Fatalf("the install registered nothing: %+v", c.Inventory)
+ }
+ for p, stamp := range entry.Paths {
+ if stamp != s {
+ t.Fatalf("%s is tagged %d, want S = %d (F = %d)", p, stamp, s, f)
+ }
+ }
+ // The database is the target's rows, installed under S.
+ if n := countRowsForTest(t, provision.DefaultDBPath(h.sdir, p1Host)); n != h.rows+1 {
+ t.Fatalf("the returned database holds %d rows, the target wrote %d", n, h.rows+1)
+ }
+ })
+ }
+ }
+}
diff --git a/internal/hostclaim/move_witness_test.go b/internal/hostclaim/move_witness_test.go
new file mode 100644
index 000000000..beced5e28
--- /dev/null
+++ b/internal/hostclaim/move_witness_test.go
@@ -0,0 +1,135 @@
+package hostclaim
+
+import (
+ "reflect"
+ "testing"
+)
+
+// TestRollbackBaselineStaysUncoveredUntilCommit ports the claim half of the
+// model's wRollbackBaselinePreserved (claim rev 6.14 item 5, R12-3; moves
+// spec §5.1 step 9 and §6.4): a landing's probe publishes the target's
+// generation in the target's region, and PublishGeneration, as it stands on
+// main, does not queue the predecessor, which is the source's generation in
+// the source's region, for a park. The target's admission record stays
+// uncovered and no chore is queued into the source's region, at either
+// probing phase and when the publication is replayed, so a rollback still
+// has the generation it restores from. Only Commit covers the record, in
+// the CAS that queues the source region's whole-location park
+// (wCommitCoverage); a rollback, the target's release and a clean return
+// never do. The source's own claim sequence, RollBack then TargetReleased
+// then ReturnToSource(clean), ends with holder.generation naming the
+// source's generation again, which the host half
+// (TestRollbackBaselineSurvivesTheCleanReturn) finds still in the bucket.
+func TestRollbackBaselineStaysUncoveredUntilCommit(t *testing.T) {
+ failID := func(t *testing.T) func() string {
+ return func() string {
+ t.Error("the target's publication allocated an obligation id for the source's generation")
+ return "id"
+ }
+ }
+ publish := func(t *testing.T, c *Claim) {
+ t.Helper()
+ if err := PublishGeneration(c.Op.ID, c.Op.Phase, c.Holder.Session, c.Holder.Generation, "uk", failID(t))(c); err != nil {
+ t.Fatalf("publish at %s: %v", c.Op.Phase, err)
+ }
+ }
+ // baselineKept is the invariant: the admission record's predecessor is
+ // the source's generation in ie, nothing is queued into ie, and the
+ // record is uncovered.
+ baselineKept := func(t *testing.T, c *Claim, where string) {
+ t.Helper()
+ srcGen := c.Op.Journal[JournalSrcGen]
+ if srcGen == "" || len(c.Holder.Admissions) != 1 {
+ t.Fatalf("%s: fixture: src_gen %q, admissions %+v", where, srcGen, c.Holder.Admissions)
+ }
+ a := c.Holder.Admissions[0]
+ if a.Box != "uk-box" || a.PredecessorRegion != "ie" || a.Predecessor != srcGen {
+ t.Fatalf("%s: the target's record does not name the source's generation %q in ie: %+v", where, srcGen, a)
+ }
+ if a.Covered {
+ t.Fatalf("%s: the target's record is covered: %+v", where, a)
+ }
+ for _, o := range c.Obligations {
+ if o.Region == "ie" || o.Box == "ie-box" {
+ t.Fatalf("%s: an obligation reaches the source's region: %+v", where, o)
+ }
+ if o.Kind == "park-generation" {
+ t.Fatalf("%s: a generation park was queued: %+v", where, o)
+ }
+ }
+ }
+
+ for _, from := range []string{"probing", "probed"} {
+ t.Run("publication at "+from, func(t *testing.T) {
+ c := targetWroteLanding(t)
+ c.Op.Phase = from
+ publish(t, c)
+ baselineKept(t, c, "after the publication")
+ // A replay changes nothing.
+ before := mustJSON(t, c)
+ publish(t, c)
+ if string(mustJSON(t, c)) != string(before) {
+ t.Fatalf("the replayed publication changed the claim:\nbefore %s\nafter %s", before, mustJSON(t, c))
+ }
+ validAfterWrite(t, c)
+ })
+ }
+
+ t.Run("the rollback and the clean return", func(t *testing.T) {
+ c := targetWroteLanding(t)
+ publish(t, c)
+ baselineKept(t, c, "after the publication")
+ srcGen := c.Op.Journal[JournalSrcGen]
+ if err := RollBack(moveA, rbA)(c); err != nil {
+ t.Fatal(err)
+ }
+ baselineKept(t, c, "after the rollback")
+ if err := TargetReleased(moveA, "uk-box", "uk", stoppedTarget, releaseScope(c), counter())(c); err != nil {
+ t.Fatal(err)
+ }
+ // The release queues the target region's own park and covers only
+ // records whose predecessor lives there.
+ baselineKept(t, c, "after the target's release")
+ if k := obligationKinds(c); k["park-replica/uk"] != 1 || len(c.Obligations) != 1 {
+ t.Fatalf("the release owes %+v", c.Obligations)
+ }
+ if err := ReturnToSource(moveA, "ie-box", true, ReturnFence{})(c); err != nil {
+ t.Fatal(err)
+ }
+ baselineKept(t, c, "after the clean return")
+ if c.Op.Kind != KindRestoring || c.Op.Phase != PhaseClaimed || c.Op.Exec != "ie-box" {
+ t.Fatalf("op: %+v", c.Op)
+ }
+ if c.Holder.Generation != srcGen || c.Holder.Stamp != 3 {
+ t.Fatalf("the return names generation %q stamp %d, want the source's %q and 3", c.Holder.Generation, c.Holder.Stamp, srcGen)
+ }
+ validAfterWrite(t, c)
+ })
+
+ t.Run("only the commit covers the record", func(t *testing.T) {
+ c := targetWroteLanding(t)
+ publish(t, c)
+ baselineKept(t, c, "after the publication")
+ src, stg := commitScopes(c)
+ if err := Commit(moveA, "ie", "ie-box", "uk", src, stg, counter())(c); err != nil {
+ t.Fatal(err)
+ }
+ if !c.Holder.Admissions[0].Covered {
+ t.Fatalf("the commit left the record uncovered: %+v", c.Holder.Admissions[0])
+ }
+ if k := obligationKinds(c); k["park-replica/ie"] != 1 {
+ t.Fatalf("the commit queued %+v, want the source region's whole-location park", c.Obligations)
+ }
+ for _, o := range c.Obligations {
+ if o.Kind == "park-generation" {
+ t.Fatalf("the commit queued a generation park: %+v", o)
+ }
+ }
+ // A publication after the commit changes nothing more.
+ before := mustJSON(t, c)
+ publish2 := PublishGeneration(c.Op.ID, c.Op.Phase, c.Holder.Session, c.Holder.Generation, "uk", failID(t))
+ if err := publish2(c); err != nil || !reflect.DeepEqual(mustJSON(t, c), before) {
+ t.Fatalf("a publication after the commit: %v", err)
+ }
+ })
+}