amadan

carlos / platform Public

sim: Astra's round 6 on the two-box model: a release needs the session's own replicator and files, or the positions recorded for its stop, and is otherwise refused until the deadline; the operator stops a target that can never release

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude a2064475d41d32ec653f520a2faa3ba3b702094e parent 2b8f5c1
5 files changed, +138 −4
  • internal/sim/hostlife/console_test.go +11 −0
  • internal/sim/hostlife/regionha_round1_test.go +82 −0
  • internal/sim/hostlife/regionha_test.go +31 −1
  • internal/sim/hostlife/rules_test.go +1 −0
  • internal/sim/hostlife/world_test.go +13 −3
diff --git a/internal/sim/hostlife/console_test.go b/internal/sim/hostlife/console_test.go
index 806116598..ca3a2b689 100644
--- a/internal/sim/hostlife/console_test.go
+++ b/internal/sim/hostlife/console_test.go
@@ -1252,6 +1252,17 @@ func (w *world) operator() {
if dead != nil && !dead.Destroyed {
dead = nil
}
+ if ex := w.boxByID(op.Exec); ex != nil && !ex.Destroyed && op.Phase == "probe-failed" && !released && !contains(c.Lost, ex.ID) && on("reloc-release-needs-flush") {
+ // a live target that cannot release (its replicator and its
+ // files are gone and nothing recorded a flush, rev 9): no
+ // release will ever come, so the operator stops the box, and
+ // its declared loss (claim D2) lets the next resolve replay
+ // the operation from pinning
+ w.stop(ex)
+ w.stats["operator stopped a target that cannot release"]++
+ w.logf("operator stopped box %d: it cannot release relocation %s", ex.ID, op.ID)
+ return
+ }
var what string
if w.cas(c, func(n *claim) bool {
if !sameOp(n.Op.ID, op.ID) || n.Op.Kind != "relocating" || n.Op.Stuck == "" {
diff --git a/internal/sim/hostlife/regionha_round1_test.go b/internal/sim/hostlife/regionha_round1_test.go
index 28f7dd703..24937fab6 100644
--- a/internal/sim/hostlife/regionha_round1_test.go
+++ b/internal/sim/hostlife/regionha_round1_test.go
@@ -767,6 +767,88 @@ func TestPendingTargetFenceHoldsThePhase(t *testing.T) {
}
}
+// TestReleaseRefusedAfterACrashBeforeTheFlush (reloc-release-needs-flush;
+// rev 9, Astra round 6): a drain's source is in releasing, its last write
+// not yet uploaded, when its agent crashes: the session's replicator is
+// gone and nothing recorded a flush for this stop. No release is
+// journaled and no pin selected; at the release deadline the operation
+// is stuck, and the operator's resolve returns the host to its source.
+// The mutation journals the remote position with Flushed false.
+func TestReleaseRefusedAfterACrashBeforeTheFlush(t *testing.T) {
+ w := twoBoxWitness()
+ from, to := w.boxes[0], w.boxes[3]
+ w.serve(from, 1, app0)
+ w.running(t, from)
+ w.published(t, from)
+ w.drainTo(t, to)
+ w.appWrite(from)
+ w.holdKey(w.claim.Holder.Location) // the flush's upload does not land
+ if !w.stepUntil(t, 1500, "the source waiting for its flush", func() bool {
+ return w.claim.Op.Phase == "releasing" && w.stats["release: waits for the flush"] > 0
+ }) {
+ w.mustHold(t)
+ }
+ w.crash(from)
+ w.hold = nil
+ for i := 0; i < releaseDeadline+20; i++ {
+ w.tick(w.act(from, to))
+ c := w.claim
+ if c.Op.Release != nil || c.Op.Pin != nil {
+ t.Fatalf("a release or a pin without the source's flush (step %d): %s", i, w.state())
+ }
+ if c.Op.Stuck != "" {
+ break
+ }
+ }
+ if c := w.claim; c.Op.Stuck == "" || c.Op.Phase != "releasing" {
+ t.Fatalf("not stuck in releasing at the release deadline: %s", w.state())
+ }
+ w.finish(t)
+ w.mustHold(t)
+ if c := w.claim; c.Op.Kind != "serving" || c.Op.Exec != from.ID {
+ t.Fatalf("the operator's resolve did not return the host to its source: %s", w.state())
+ }
+}
+
+// TestReleaseReplaysTheRecordedFlush (reloc-release-needs-flush; rev 9,
+// Astra round 6): the source's release sequence completes and records
+// its flushed positions (ReleasePositions), and its agent crashes before
+// the release is journaled. The restarted agent has no replicator for
+// the session, but the record is durable: the release is replayed with
+// the recorded positions, Flushed, and the drain completes. The mutation
+// records nothing and journals the remote position with Flushed false.
+func TestReleaseReplaysTheRecordedFlush(t *testing.T) {
+ w := twoBoxWitness()
+ from, to := w.boxes[0], w.boxes[3]
+ w.serve(from, 1, app0)
+ w.running(t, from)
+ w.published(t, from)
+ w.drainTo(t, to)
+ w.appWrite(from)
+ if !w.stepUntil(t, 1500, "releasing", w.atPhase("releasing")) {
+ w.mustHold(t)
+ }
+ var r *releaseEntry
+ for i := 0; i < 60 && r == nil; i++ {
+ r = w.relocRelease(from, w.claim.clone()) // steps 1 to 3 and the record, not yet journaled
+ w.tick()
+ }
+ if r == nil || !r.Flushed {
+ t.Fatalf("the release sequence did not complete: %v %s", r, w.state())
+ }
+ w.crash(from)
+ if !w.reach(t, 60, "the release journaled", func() bool { return w.claim.Op.Release != nil }, w.act(from, to)) {
+ w.mustHold(t)
+ }
+ got := w.claim.Op.Release
+ if !got.Flushed || got.FinalTX != r.FinalTX || got.ManifestSeq != r.ManifestSeq {
+ t.Fatalf("the journaled release %+v is not the recorded one %+v", got, r)
+ }
+ w.servingOn(t, to)
+ w.finish(t)
+ w.mustHold(t)
+}
+
// TestPinNeverShortOfTheRelease (reloc-pin-reaches-release): the listing
// lags behind the journaled release (its newest file is not yet visible);
// no pin is journaled short of the release's final_txid.
diff --git a/internal/sim/hostlife/regionha_test.go b/internal/sim/hostlife/regionha_test.go
index 376e15129..17338b857 100644
--- a/internal/sim/hostlife/regionha_test.go
+++ b/internal/sim/hostlife/regionha_test.go
@@ -144,9 +144,27 @@ func (w *world) relocRelease(b *box, c *claim) *releaseEntry {
w.stats["release: a process survives the stop"]++
return nil
}
+ if r := b.relRecord; r != nil && on("reloc-release-needs-flush") && r.Op == c.Op.ID && r.Phase == c.Op.Phase && r.E.Session == h.Session && r.E.Gen == h.Gen {
+ // a flush this box recorded for this stop (ReleasePositions.Recorded):
+ // the release is replayed with the recorded positions, never
+ // flushed or guessed again (rev 9)
+ e := r.E
+ e.Files, e.Timers = r.E.Files.clone(), append([]int(nil), r.E.Timers...)
+ w.stats["release: replayed from the recorded positions"]++
+ return &e
+ }
prefix := dataKey(h.Location, h.Gen, "")
remote := w.remoteMax(w.buckets[b.Region], prefix)
n, flushed := remote, false
+ if on("reloc-release-needs-flush") && !(b.DB != nil && b.DB.Inc == h.Inc && b.DB.Gen == h.Gen && b.DB.Epoch == b.Epoch && b.DB.Session == h.Session) {
+ // relocationFlushDB needs the session's own replicator, and
+ // nothing recorded a flush for this stop: no release (1b-next Task
+ // 7; rev 9, Astra round 6). The release deadline makes the
+ // operation stuck; the mutation journals the remote position with
+ // Flushed false, as rev 8 did
+ w.stats["release: no replicator and no recorded flush, refused"]++
+ return nil
+ }
if db := b.DB; db != nil && db.Inc == h.Inc && db.Gen == h.Gen {
if db.Epoch == b.Epoch && db.Session == h.Session {
if len(db.Pending) > 0 {
@@ -171,6 +189,12 @@ func (w *world) relocRelease(b *box, c *claim) *releaseEntry {
// files identity
files := b.Files.clone()
if files == nil || files.Inc != h.Inc {
+ if on("reloc-release-needs-flush") {
+ // ReleaseFlushFiles publishes this box's own files directory;
+ // there is none to publish (rev 9)
+ w.stats["release: no files directory to flush, refused"]++
+ return nil
+ }
files = w.relocFiles(c).clone()
}
// 3. (Task 4) publish every pending file and wake-state mutation,
@@ -199,8 +223,14 @@ func (w *world) relocRelease(b *box, c *claim) *releaseEntry {
seq, b.releaseSeq = b.releaseSeq, 0
b.published = filesState(b.Timers, files)
}
- return &releaseEntry{Box: b.ID, Session: h.Session, Gen: h.Gen, FinalTX: n, ManifestSeq: seq, Flushed: flushed,
+ e := &releaseEntry{Box: b.ID, Session: h.Session, Gen: h.Gen, FinalTX: n, ManifestSeq: seq, Flushed: flushed,
Files: files, Timers: append([]int(nil), b.Timers...), At: w.now}
+ if on("reloc-release-needs-flush") {
+ // ReleasePositions.Record: durable before the producers close
+ b.relRecord = &releaseRecord{Op: c.Op.ID, Phase: c.Op.Phase, E: *e}
+ b.relRecord.E.Files, b.relRecord.E.Timers = files.clone(), append([]int(nil), b.Timers...)
+ }
+ return e
}
// pinSource is the generation the pin reads (StorePinner's genselect.Select
diff --git a/internal/sim/hostlife/rules_test.go b/internal/sim/hostlife/rules_test.go
index ed233915e..12fd15926 100644
--- a/internal/sim/hostlife/rules_test.go
+++ b/internal/sim/hostlife/rules_test.go
@@ -197,6 +197,7 @@ var allRules = []string{
"files-live-set-moves-parks", // finding F4 (rev 4), claim §6.2a: moves' parks (a push's whole-location park, a landing's install park, a rollback's return park) keep the files live set; the mutation archives it with the rest and the next failover can never pin the live files (INV-3)
"reloc-clear-after-cleanup", // 1b-next ruling 20 and rev 6 R4-5 (Finish clears the record only after a successful scoped cleanup): a failed cleanup keeps the controller record, which blocks executable updates (claim §11's witness); the mutation clears it and an update runs over the unfinished cleanup (INV-29)
"fence-converts-committed-landing", // finding F5 (rev 4; claim §11 "automatic target loss racing the record commit CAS"): a landing whose record committed and whose target is then terminated becomes a failover in the target's region; the mutation leaves it to the operator, and it never completes (INV-3)
+ "reloc-release-needs-flush", // rev 9 (Astra round 6; 1b-next rev 19 Task 7 Release, relocationFlushDB, ReleaseFlushFiles, ReleasePositions): a release needs the session's own replicator and the box's own files directory, or the positions it recorded for this stop; otherwise none, and the deadline makes the operation stuck; the mutation journals the remote position with Flushed false (TestReleaseRefusedAfterACrashBeforeTheFlush, TestReleaseReplaysTheRecordedFlush)
"reloc-fence-pending-blocks", // rev 8 (Astra round 5; 1b-next rev 19 Worker.Step returns w.replace(...) before its phase switch): while a relocation participant's fence is pending (a dead target, no spare, before its deadline) no regional phase transition runs; the participant's own release still does; the mutation goes on into the phase switch (TestPendingTargetFenceHoldsThePhase)
"reloc-name-before-pin", // rev 7 (Astra round 4 finding 1; 1b-next rev 19 Worker.pinning): a relocation at pinning with no target names one before it selects a pin, on every path; the mutation pins first, and a manifest that lands while no box is eligible is left out (a loss bound: TestConvertedMoveWaitsForATargetBeforeItsPin)
"reloc-failover-start-guard", // rev 7 (Astra round 4 finding 3; 1b-next rev 19 Worker.Start, StartRelocation): a serving executor's failover starts only with an eligible target, the app admitting the registration (and its recheck) and RelocationStartGuard with the source fenced; the mutation starts at once (TestFailoverStartWaitsForAdmission, TestFailoverStartWaitsForParkReplica)
diff --git a/internal/sim/hostlife/world_test.go b/internal/sim/hostlife/world_test.go
index 5cb9f479f..af8e3645f 100644
--- a/internal/sim/hostlife/world_test.go
+++ b/internal/sim/hostlife/world_test.go
@@ -373,9 +373,10 @@ type box struct {
execRecord string
cleanupPending bool
cleanupFails bool
- fencePending bool // (rev 8) this worker step found a relocation participant's fence it could not consume
- FenceObservedAt int // the box-fences record's observed_at: when a worker first saw it terminated; 0 no record (rev 4)
- Enrolled bool // the roster holds an active entry with its proven cloud identity (H11); a replacement box is enrolled automatically enrollDelay steps after it launches (H16, H17)
+ relRecord *releaseRecord // (rev 9) ReleasePositions: the flushed positions of one release, recorded on disk against its stop; an agent restart keeps it
+ fencePending bool // (rev 8) this worker step found a relocation participant's fence it could not consume
+ FenceObservedAt int // the box-fences record's observed_at: when a worker first saw it terminated; 0 no record (rev 4)
+ Enrolled bool // the roster holds an active entry with its proven cloud identity (H11); a replacement box is enrolled automatically enrollDelay steps after it launches (H16, H17)
EnrollAt int
DeleteOnTermination bool // every volume the box had attached carried the flag while it ran (§5.9): decides only the automatic discharge of its park-local chores (H19), never a fence
installEpoch int // the agent process that made this box's local install for a relocation (B2-5): a later process never remakes it once an admission named P
@@ -928,6 +929,15 @@ type opState struct {
// is false when the session's replicator was gone (an agent restart) and
// the entry names what the remote held: released coverage never applies
// to it, and the next admission declares the rest.
+// releaseRecord is 1b-next Task 7's ReleasePositions record: the
+// release a box computed once its writers stopped and its database and
+// files flushed, bound to the stop (the operation, the phase and the
+// writer session), durable before the producers close (rev 9).
+type releaseRecord struct {
+ Op, Phase string
+ E releaseEntry
+}
+
type releaseEntry struct {
Box, Session int
Gen string