amadan

carlos / platform Public

sim: hostlife exercises rev 4's eight gaps, re-encoded on claim rev 3 (#175)

New actors: staging rounds and cut-round retries that pause before
signing, past their lease; registry holds and opens that pause and wake at
the worst moment; an obligation runner beside the role worker; admission
and reconcile from cached reads (the fresh-admission mutation); door reads
that stall. Claim rev 3: verification after evidence + H with stragglers
as a further event, per-region quiet_after across free with installs
waiting, one adoption routine, compute-fenced declare-lost.

New rules: sign-before-safeuntil (staging signs only before SafeUntil)
and readiness-quiet (T12), with an INV-9 check that no cut waits on
quiet_after. exec-admission now covers the admission table's role column.
A push's open accepts a row already open. Staging lease 400 steps (longer
than the cut budget), readiness wait 800, moves three times as frequent.

30,000 and 100,000 seeds clean. Mutation sweep: 37 rules caught alone,
9 with a stated partner (exec-admission + hold-op-open + start-check as a
triple), 2 liveness-only, restore-lineage-filter shows nothing beyond
quiet-before-ack, readiness-quiet unreachable under claim rev 3.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 7a3e94e7da02b4ccd65037212f4f55d6c4bb88fd parent a80ff90
7 files changed, +253 −36
  • internal/sim/hostlife/background_test.go +7 −2
  • internal/sim/hostlife/console_test.go +18 −1
  • internal/sim/hostlife/moves_test.go +168 −12
  • internal/sim/hostlife/protocol_test.go +30 −10
  • internal/sim/hostlife/rules_test.go +3 −1
  • internal/sim/hostlife/sim_test.go +14 −7
  • internal/sim/hostlife/world_test.go +13 −3
diff --git a/internal/sim/hostlife/background_test.go b/internal/sim/hostlife/background_test.go
index 5e4cd24f2..5781407ad 100644
--- a/internal/sim/hostlife/background_test.go
+++ b/internal/sim/hostlife/background_test.go
@@ -71,6 +71,9 @@ func (w *world) wake(b *box) {
if !ok || !w.admits(b, c, "") {
return
}
+ if b.DB == nil && on("quiet-before-ack") && w.now < c.Quiet[b.Region] {
+ return // a first install into this region waits out quiet_after (claim §6.2a)
+ }
w.spawn(b, c, holderLin(c.Holder), false)
}
@@ -121,12 +124,14 @@ func (w *world) reconcile(b *box) {
}
return
}
- c, ok := w.readClaim(b)
+ c, ok := w.admissionClaim(b) // reconcile's keep-or-create is an admission decision (C4)
if !ok {
return
}
if c == nil {
- w.adopt()
+ if w.claim == nil {
+ w.adopt()
+ }
return
}
w.reconcileClaimed(b, c)
diff --git a/internal/sim/hostlife/console_test.go b/internal/sim/hostlife/console_test.go
index 3ac2eac67..f89eb43fe 100644
--- a/internal/sim/hostlife/console_test.go
+++ b/internal/sim/hostlife/console_test.go
@@ -17,6 +17,7 @@ type doorPlan struct {
Key string
Ver int
Inc int
+ Until int // a stalled read may resume until then (one in four stall long)
}
func (w *world) execBox(c *claim) *box {
@@ -443,7 +444,7 @@ func (w *world) doorAuthorize() {
}
bk := w.buckets[b.Region]
key := "instances/" + host + "/ltx"
- d := &doorPlan{Region: b.Region, Key: key, Inc: c.Holder.Inc}
+ d := &doorPlan{Region: b.Region, Key: key, Inc: c.Holder.Inc, Until: w.now + 1 + w.pauseLen()}
if on("door-by-version") {
for _, v := range bk.objs[key] {
if !v.Marker && v.B.Inc == c.Holder.Inc {
@@ -462,6 +463,22 @@ func (w *world) doorOpen() {
return
}
i := w.rng.Intn(len(w.doorReads))
+ // a stalled read resumes at its worst moment if one comes before its
+ // stall ends: when its location's current object belongs to another
+ // incarnation. Otherwise it waits out its stall.
+ worst := -1
+ for j, d := range w.doorReads {
+ if cur := w.buckets[d.Region].current(d.Key); cur != nil && !cur.Marker && cur.B.Inc != 0 && cur.B.Inc != d.Inc {
+ worst = j
+ break
+ }
+ }
+ switch {
+ case worst >= 0:
+ i = worst
+ case w.doorReads[i].Until > w.now:
+ return
+ }
d := w.doorReads[i]
w.doorReads = append(w.doorReads[:i:i], w.doorReads[i+1:]...)
bk := w.buckets[d.Region]
diff --git a/internal/sim/hostlife/moves_test.go b/internal/sim/hostlife/moves_test.go
index b6c937262..c0ea5c7c1 100644
--- a/internal/sim/hostlife/moves_test.go
+++ b/internal/sim/hostlife/moves_test.go
@@ -5,7 +5,14 @@ import "fmt"
// worker is one box agent's lifecycle worker: exec-lease renewal, the
// obligations it owns, then its role in the current operation. One call
// performs at most one protocol step.
-func (w *world) worker(b *box) {
+func (w *world) worker(b *box) { w.workerMode(b, false) }
+
+// obligationRunner is a separate goroutine from the role worker: a box
+// runs its chores and its operation role concurrently (CC11 is what keeps
+// them from meeting in one namespace).
+func (w *world) obligationRunner(b *box) { w.workerMode(b, true) }
+
+func (w *world) workerMode(b *box, obl bool) {
if b.Old {
return // a pre-floor executable knows nothing of claims
}
@@ -13,6 +20,18 @@ func (w *world) worker(b *box) {
if !ok {
return
}
+ if obl {
+ if c == nil || (contains(c.Lost, b.ID) && on("lost-quarantine")) {
+ return
+ }
+ for _, o := range c.Obl {
+ if w.ownerOf(o) == b {
+ w.doObligation(b, c, o)
+ return
+ }
+ }
+ return
+ }
if b.stale != nil && w.rng.Intn(6) == 0 {
// a stalled worker of this box resumes with the claim it read long ago
old := b.stale.clone()
@@ -50,12 +69,6 @@ func (w *world) worker(b *box) {
}
return
}
- for _, o := range c.Obl {
- if w.ownerOf(o) == b {
- w.doObligation(b, c, o)
- return
- }
- }
if c.Holder == nil {
return
}
@@ -170,8 +183,21 @@ func (w *world) moveSource(b *box, c *claim) {
return
}
db, files := w.snapshotBlob(b, c)
- v1 := w.issue(b.ID, &request{kind: rPut, region: op.TgtRegion, key: fmt.Sprintf("moves/%s/%s/db", host, op.ID), b: db})
- v2 := w.issue(b.ID, &request{kind: rPut, region: op.TgtRegion, key: fmt.Sprintf("moves/%s/%s/files", host, op.ID), b: files})
+ r1 := &request{kind: rPut, region: op.TgtRegion, key: fmt.Sprintf("moves/%s/%s/db", host, op.ID), b: db}
+ r2 := &request{kind: rPut, region: op.TgtRegion, key: fmt.Sprintf("moves/%s/%s/files", host, op.ID), b: files}
+ if w.rng.Intn(3) == 0 {
+ // this round revalidated under the staging-work lease just now,
+ // and its goroutine pauses before signing (J-N2): it may sign
+ // after the final round and the seal, but only before its lease's
+ // SafeUntil (claim C3)
+ until := w.staging[op.TgtRegion].Until
+ for _, r := range []*request{r1, r2} {
+ w.stalled = append(w.stalled, &stalledSign{box: b.ID, epoch: b.Epoch, req: r, signAt: w.now + w.pauseSign(), until: until})
+ }
+ return
+ }
+ v1 := w.issue(b.ID, r1)
+ v2 := w.issue(b.ID, r2)
if v1 > 0 && v2 > 0 && op.Phase == "preparing" {
w.cas(c, func(n *claim) bool { n.Op.Phase = "ready"; return sameOp(n.Op.ID, op.ID) })
}
@@ -214,7 +240,9 @@ func (w *world) moveSource(b *box, c *claim) {
}
db, files := w.snapshotBlob(b, c)
if op.SealDB.Ver == 0 {
- v := w.issue(b.ID, &request{kind: rPut, region: op.TgtRegion, key: fmt.Sprintf("moves/%s/%s/db", host, op.ID), b: db})
+ r := &request{kind: rPut, region: op.TgtRegion, key: fmt.Sprintf("moves/%s/%s/db", host, op.ID), b: db}
+ w.maybeStallDuplicate(b, op, r)
+ v := w.issue(b.ID, r)
if v > 0 {
w.cas(c, func(n *claim) bool {
n.Op.SealDB = ref{Region: op.TgtRegion, Key: fmt.Sprintf("moves/%s/%s/db", host, op.ID), Ver: v}
@@ -223,7 +251,9 @@ func (w *world) moveSource(b *box, c *claim) {
}
return
}
- v := w.issue(b.ID, &request{kind: rPut, region: op.TgtRegion, key: fmt.Sprintf("moves/%s/%s/files", host, op.ID), b: files})
+ r := &request{kind: rPut, region: op.TgtRegion, key: fmt.Sprintf("moves/%s/%s/files", host, op.ID), b: files}
+ w.maybeStallDuplicate(b, op, r)
+ v := w.issue(b.ID, r)
if v > 0 {
w.cas(c, func(n *claim) bool {
if !sameOp(n.Op.ID, op.ID) {
@@ -261,6 +291,9 @@ func (w *world) moveReport(b *box, c *claim) {
return
}
ok := b.HasKey || !on("readiness-key")
+ if on("readiness-quiet") && w.now < c.Quiet[op.TgtRegion] {
+ ok = false // T12: the landing would wait out quiet_after inside the cut window
+ }
w.cas(c, func(n *claim) bool {
if !sameOp(n.Op.ID, op.ID) {
return false
@@ -310,10 +343,21 @@ func (w *world) holdRow(b *box, status, holdOp string, c *claim) {
b.Row = &row{Inc: c.Holder.Inc, App: c.Holder.App}
}
b.Row.Status, b.Row.HoldOp = status, holdOp
+ if w.rng.Intn(25) == 0 {
+ // a paused goroutine's duplicate of this write, executed later
+ b.deferred = append(b.deferred, &localWrite{kind: "hold", status: status, holdOp: holdOp, epoch: b.Epoch, at: w.now + 1 + w.pauseLen()})
+ }
}
// openRow is the conditional open (moves §5.2).
func (w *world) openRow(b *box, holdOp string) bool {
+ if w.rng.Intn(6) == 0 {
+ b.deferred = append(b.deferred, &localWrite{kind: "open", holdOp: holdOp, epoch: b.Epoch, at: w.now + 1 + w.pauseLen(), worst: w.rng.Intn(2) == 0})
+ }
+ return w.openNow(b, holdOp)
+}
+
+func (w *world) openNow(b *box, holdOp string) bool {
if b.Row == nil {
return false
}
@@ -324,6 +368,102 @@ func (w *world) openRow(b *box, holdOp string) bool {
return true
}
+// localWrite is a host-specific registry write whose goroutine passed its
+// revalidation and then paused: it lands later, and only the conditions
+// on the write itself (hold_op) stand between it and a newer operation.
+type localWrite struct {
+ kind string // "open" or "hold"
+ status string
+ holdOp string
+ epoch int
+ at int // the latest it lands
+ worst bool // the scheduler wakes it at the worst moment before at
+}
+
+// pauseLen is how long a paused goroutine sleeps before its registry
+// write lands: mostly short, sometimes as long as a whole operation. A
+// crash or reboot (b.Epoch) kills it first.
+func (w *world) pauseLen() int {
+ if w.rng.Intn(4) == 0 {
+ return w.rng.Intn(2000)
+ }
+ return w.rng.Intn(150)
+}
+
+// pauseSign is how long a staging goroutine pauses between revalidating
+// under the staging-work lease and signing: it may outlast the lease, and
+// then SafeUntil (claim C3) is all that stops the signature.
+func (w *world) pauseSign() int { return w.rng.Intn(2*stagingTTL + horizonH + 1) }
+
+// maybeStallDuplicate models a lost-response retry in the cut's own
+// staging round: the first attempt's goroutine paused before signing and
+// the agent retried. The paused attempt may sign after the seal.
+func (w *world) maybeStallDuplicate(b *box, op opState, r *request) {
+ if w.rng.Intn(3) != 0 {
+ return
+ }
+ dup := *r
+ w.stalled = append(w.stalled, &stalledSign{box: b.ID, epoch: b.Epoch, req: &dup, signAt: w.now + w.pauseSign(), until: w.staging[op.TgtRegion].Until})
+}
+
+type stalledSign struct {
+ box int
+ epoch int
+ req *request
+ signAt int
+ until int
+}
+
+// runStalled lets paused goroutines resume: deferred registry writes and
+// staging rounds that pause between revalidation and signing.
+func (w *world) runStalled() {
+ var keep []*stalledSign
+ for _, s := range w.stalled {
+ b := w.boxByID(s.box)
+ if b == nil || b.Destroyed || b.Epoch != s.epoch {
+ continue // the goroutine died with its agent
+ }
+ if s.signAt > w.now || b.Frozen || b.AgentPaused {
+ keep = append(keep, s)
+ continue
+ }
+ if on("sign-before-safeuntil") && w.localNow(b) > s.until-maxSkew {
+ continue // checked SafeUntil immediately before signing: too late
+ }
+ w.issue(b.ID, s.req)
+ }
+ w.stalled = keep
+ for _, b := range w.boxes {
+ if b.Destroyed || b.Frozen || b.AgentPaused || b.AgentDown {
+ continue
+ }
+ var rest []*localWrite
+ for _, d := range b.deferred {
+ if d.epoch != b.Epoch {
+ continue
+ }
+ // an adversarial wake-up: a stale open resumes while a newer
+ // operation holds the row, at its worst moment: after that
+ // operation committed and stopped re-asserting the hold
+ worst := d.worst && d.kind == "open" && b.Row != nil && b.Row.HoldOp != "" && b.Row.HoldOp != d.holdOp &&
+ w.claim != nil && w.claim.Op.Committed && w.rng.Intn(10) == 0
+ if d.at > w.now && !worst {
+ rest = append(rest, d)
+ continue
+ }
+ switch d.kind {
+ case "open":
+ w.openNow(b, d.holdOp)
+ case "hold":
+ if b.Row != nil && (!on("hold-op-open") || b.Row.HoldOp == "" || b.Row.HoldOp == d.holdOp) {
+ b.Row.Status, b.Row.HoldOp = d.status, d.holdOp
+ }
+ }
+ }
+ b.deferred = rest
+ }
+}
+
func (w *world) revertSource(b *box, c *claim) {
op := c.Op
if !on("revert-journal") && !b.mem[op.ID+"rev"] && (op.RevPhase == "probing" || op.RevPhase == "probed") {
@@ -396,6 +536,7 @@ func (w *world) rollback(c *claim, why string) {
return true
}) {
w.logf("rollback %s: %s", op.ID, why)
+ w.stats["why: "+why+" @"+op.LandPhase]++
w.fixRecord()
}
}
@@ -423,11 +564,23 @@ func (w *world) landTarget(b *box, c *claim) {
if !w.evidence(b, op.ID) {
return
}
+ if w.now < c.Quiet[b.Region] && !b.mem[op.ID+"quietwait"] {
+ // INV-9 (T12): an admitted cut never waits out quiet_after; the
+ // readiness report said the region was quiet
+ b.mem[op.ID+"quietwait"] = true
+ w.violate("INV-9: the cut of %s waits on quiet_after[%d] until %d", op.ID, b.Region, c.Quiet[b.Region])
+ }
if w.parkStep(b, "instances/"+host+"/", fmt.Sprintf("superseded/%s/%s/install/", op.ID, host), c.Quiet[b.Region], "") {
w.cas(c, func(n *claim) bool { n.Op.LandPhase = "install-copy"; return sameOp(n.Op.ID, op.ID) })
}
case "install-copy":
src := op.SealDB
+ if cur := w.buckets[b.Region].current(src.Key); cur != nil && cur.ID != src.Ver {
+ w.stats["diag: staging newer than seal at install"]++
+ if sv := w.buckets[b.Region].byID(src.Key, src.Ver); sv != nil && len(cur.B.Writes) < len(sv.B.Writes) {
+ w.stats["diag: ...and older content"]++
+ }
+ }
if !on("seal-by-version") {
cur := w.buckets[b.Region].current(src.Key)
if cur == nil {
@@ -805,6 +958,9 @@ func (w *world) pushStep(b *box, c *claim) {
w.cas(c, func(n *claim) bool { n.Op.Phase = "install"; return sameOp(n.Op.ID, op.ID) })
}
case "install":
+ if on("quiet-before-ack") && w.now < c.Quiet[b.Region] {
+ return // installs wait out quiet_after (claim §6.2a)
+ }
key := 0
if b.Files != nil {
key = b.Files.Key
@@ -845,7 +1001,7 @@ func (w *world) pushStep(b *box, c *claim) {
})
}
case "probed":
- if w.openRow(b, op.ID) {
+ if w.openRow(b, op.ID) || b.Row != nil && b.Row.Status == "ready" {
w.release(c, b.ID)
}
}
diff --git a/internal/sim/hostlife/protocol_test.go b/internal/sim/hostlife/protocol_test.go
index 241142c2c..a5bb6df57 100644
--- a/internal/sim/hostlife/protocol_test.go
+++ b/internal/sim/hostlife/protocol_test.go
@@ -32,7 +32,12 @@ func (w *world) admissionClaim(b *box) (*claim, bool) {
if on("fresh-admission") {
return w.readClaim(b)
}
- if b.snap != nil {
+ // the mutation: admission decides from an older read (a cache, or a
+ // goroutine that read before it paused)
+ if b.stale != nil && w.agentUp(b) {
+ return b.stale.clone(), true
+ }
+ if b.snap != nil && w.agentUp(b) {
return b.snap.clone(), true
}
return w.readClaim(b)
@@ -279,7 +284,7 @@ func (w *world) spawn(b *box, c *claim, L lin, probe bool) bool {
if w.rng.Intn(2) == 0 {
b.Procs = append(b.Procs, &proc{Main: false, Strict: strict, Inc: h.Inc, Gen: L.Op, Box: b.ID})
}
- w.logf("box %d spawned inc %d gen %q (probe=%v strict=%v)", b.ID, h.Inc, L.Op, probe, strict)
+ w.logf("box %d spawned inc %d gen %q (probe=%v strict=%v) claim=%s/%s exec=%d", b.ID, h.Inc, L.Op, probe, strict, c.Op.Kind, c.Op.ID, c.Op.Exec)
return true
}
@@ -314,26 +319,30 @@ func (w *world) admits(b *box, c *claim, probe string) bool {
b.ExecUntil = 0 // a fresh read with no operation ends the fence (§7.1)
}
op := c.Op
+ // the admission table's role column: in this model each phase's role
+ // is a box id, and it is op.exec in every phase, so exec-admission
+ // governs both
+ role := func(id int) bool { return !on("exec-admission") || b.ID == id }
switch op.Kind {
case "serving":
return probe == "" && b.Row.Status == "ready"
case "moving":
- return probe == "" && b.ID == op.Src && op.CutAt == 0 && b.Row.Status == "ready"
+ return probe == "" && role(op.Src) && op.CutAt == 0 && b.Row.Status == "ready"
case "reverting":
if probe != "" {
- return probe == op.ID && b.ID == op.Src && op.RevPhase == "probing"
+ return probe == op.ID && role(op.Src) && op.RevPhase == "probing"
}
- return b.ID == op.Src && op.RevPhase == "opened" && b.Row.Status == "ready"
+ return role(op.Src) && op.RevPhase == "opened" && b.Row.Status == "ready"
case "landing":
if probe != "" {
- return probe == op.ID && b.ID == op.Tgt && op.LandPhase == "probing"
+ return probe == op.ID && role(op.Tgt) && op.LandPhase == "probing"
}
- return b.ID == op.Tgt && op.Committed && b.Row.Status == "ready"
+ return role(op.Tgt) && op.Committed && b.Row.Status == "ready"
case "restoring":
if probe != "" {
- return probe == op.RollbackID && b.ID == op.Src && op.SrcPhase == "probing"
+ return probe == op.RollbackID && role(op.Src) && op.SrcPhase == "probing"
}
- return b.ID == op.Src && op.SrcPhase == "opened" && b.Row.Status == "ready"
+ return role(op.Src) && op.SrcPhase == "opened" && b.Row.Status == "ready"
case "pushing":
return probe == op.ID && op.Phase == "probing"
}
@@ -405,7 +414,8 @@ func (w *world) parkDepth(b *box, prefix, dest string, quiet int, excludeGen str
w.issue(b.ID, &request{kind: rPut, region: b.Region, key: manKey, b: &blob{Kind: "manifest", Refs: snap}})
return false
}
- // delete the manifest's versions
+ // delete the manifest's versions, in one batch per visit
+ issued := false
for _, r := range man.B.Refs {
if bk.byID(r.Key, r.Ver) == nil {
continue
@@ -415,6 +425,9 @@ func (w *world) parkDepth(b *box, prefix, dest string, quiet int, excludeGen str
} else {
w.issue(b.ID, &request{kind: rDelKey, region: b.Region, key: r.Key})
}
+ issued = true
+ }
+ if issued {
return false
}
if on("quiet-before-ack") {
@@ -510,9 +523,16 @@ func (w *world) doObligation(b *box, c *claim, o obligation) {
return
}
}
+ evid := 0
+ if b.evidAt != nil {
+ evid = b.evidAt[o.ID]
+ }
w.cas(c, func(n *claim) bool {
for i, x := range n.Obl {
if x.ID == o.ID {
+ if (o.Kind == "park-replica" || o.Kind == "park-superseded") && on("quiet-before-ack") && o.Region >= 0 && evid+horizonH > n.Quiet[o.Region] {
+ n.Quiet[o.Region] = evid + horizonH // quiet_after survives free
+ }
n.Obl = append(n.Obl[:i:i], n.Obl[i+1:]...)
if o.Ret == "legacy" && n.oblFor("adopt") == 0 {
n.Complete = true // every region has acknowledged
diff --git a/internal/sim/hostlife/rules_test.go b/internal/sim/hostlife/rules_test.go
index 340df4aa6..ba2431bf4 100644
--- a/internal/sim/hostlife/rules_test.go
+++ b/internal/sim/hostlife/rules_test.go
@@ -27,8 +27,9 @@ var allRules = []string{
"claim-op-exclusive", // C2/§4: one operation at a time
"claim-cas-opid", // C2: every CAS conditional on op.id
"revalidate-before-effect", // C3: a worker re-reads under its lease before acting
+ "sign-before-safeuntil", // C3/§3.1: a paused round signs only before its lease's SafeUntil
"fresh-admission", // C4: admission reads the claim fresh
- "exec-admission", // C4/K1: op.exec names this box
+ "exec-admission", // C4/K1: op.exec names this box (and the role column, which is op.exec here)
"recovery-pass", // C2/§8: console claim-recovery pass
"fresh-incarnation", // §5.1/R3-12
"touched-incomplete", // §9/J-N5: adopted legacy parks every region
@@ -71,6 +72,7 @@ var allRules = []string{
"cut-deadline", // J-N15
"readiness-key", // T2
"source-readiness", // S1: the source can run the instance before the cut
+ "readiness-quiet", // T12: the target region's quiet_after has passed
"files-carried", // §8 (the kpmg outage)
"config-before-probe", // L3 (the kpmg outage)
}
diff --git a/internal/sim/hostlife/sim_test.go b/internal/sim/hostlife/sim_test.go
index ac1850524..ad04fe588 100644
--- a/internal/sim/hostlife/sim_test.go
+++ b/internal/sim/hostlife/sim_test.go
@@ -77,15 +77,16 @@ func (w *world) legacyStart() {
func (w *world) step() {
w.now++
w.completeDue(false)
+ w.runStalled()
bs := w.liveBoxes()
b := bs[w.rng.Intn(len(bs))]
n := w.rng.Intn(100)
switch {
case !w.calm && n < 6:
- switch w.rng.Intn(7) {
+ switch w.rng.Intn(9) {
case 0, 1:
w.userCreate()
- case 2:
+ case 2, 7, 8:
w.userMove()
case 3:
w.userDelete()
@@ -114,11 +115,13 @@ func (w *world) step() {
if w.calm || w.rng.Intn(4) == 0 {
w.operator() // operators act during the incident too, not only after
}
- case n < 58:
+ case n < 56:
w.worker(b)
- case n < 64:
+ case n < 62:
+ w.obligationRunner(b)
+ case n < 67:
w.reconcile(b)
- case n < 72:
+ case n < 73:
w.traffic(b)
w.wake(b)
case n < 77:
@@ -325,8 +328,12 @@ func (w *world) state() string {
if c == nil {
return fmt.Sprintf("claim absent, record %+v", w.rec)
}
- return fmt.Sprintf("op=%s/%s phase=%s land=%s src=%s rev=%s exec=%d obl=%d holder=%+v rec=%+v pending=%d console=%d",
- c.Op.Kind, c.Op.ID, c.Op.Phase, c.Op.LandPhase, c.Op.SrcPhase, c.Op.RevPhase, c.Op.Exec, len(c.Obl), c.Holder, w.rec, len(w.pending), len(w.console))
+ obl := ""
+ for _, o := range c.Obl {
+ obl += fmt.Sprintf(" %s:%s/r%d/b%d/%s", o.ID, o.Kind, o.Region, o.Box, o.Prefix)
+ }
+ return fmt.Sprintf("op=%s/%s phase=%s land=%s src=%s rev=%s exec=%d obl=%d holder=%+v rec=%+v pending=%d console=%d obls=[%s] quiet=%v",
+ c.Op.Kind, c.Op.ID, c.Op.Phase, c.Op.LandPhase, c.Op.SrcPhase, c.Op.RevPhase, c.Op.Exec, len(c.Obl), c.Holder, w.rec, len(w.pending), len(w.console), obl, c.Quiet)
}
// ---------- running ----------
diff --git a/internal/sim/hostlife/world_test.go b/internal/sim/hostlife/world_test.go
index a7add043f..d225b382c 100644
--- a/internal/sim/hostlife/world_test.go
+++ b/internal/sim/hostlife/world_test.go
@@ -40,10 +40,11 @@ const (
execTTL = 40 // exec lease TTL
execRenew = 15 // renew when this close to expiry
lostMargin = 10 // declare-lost margin past expiry
- cutBudget = 220 // absolute cut deadline after admission
- stagingTTL = 30 // staging-work lease TTL
+ cutBudget = 320 // absolute cut deadline after admission
+ stagingTTL = 400 // staging-work lease TTL: longer than the cut budget, as in production
probeTries = 3
- readyWait = 400 // abort a move whose readiness never comes
+ readyWait = 800 // abort a move whose readiness never comes
+ maxSkew = 1 // a box clock is within this of true time (SafeUntil's margin)
)
// ---------- storage ----------
@@ -249,6 +250,8 @@ type box struct {
// in-memory, lost on crash
snap *claim
stale *claim // a stalled worker's old read
+ deferred []*localWrite
+ Epoch int
plans []*plan
stopped map[string]bool
mem map[string]bool
@@ -265,6 +268,8 @@ type plan struct {
}
func (b *box) forget() {
+ b.Epoch++ // every goroutine of the old agent is gone
+ b.deferred = nil
b.snap, b.plans, b.stopped, b.stale = nil, nil, nil, nil
b.mem = map[string]bool{}
b.evidAt = map[string]int{}
@@ -425,6 +430,8 @@ type world struct {
console []*consolePlan
fail map[string]bool // which faults this seed exercises
stats map[string]int
+ stalled []*stalledSign
+ ccache *claim // the console's short-lived claim cache
violations []string
opsDone int
@@ -570,6 +577,9 @@ func (w *world) issue(actorBox int, r *request) int {
}
if w.rng.Intn(tail) == 0 {
r.doneAt = w.now + 1 + w.rng.Intn(horizonH) // the long tail: signed, cancelled, completed late
+ if w.rng.Intn(4) == 0 {
+ r.doneAt = w.now + horizonH // as late as the signature-age limit allows
+ }
}
r.by = fmt.Sprintf("box:%d", actorBox)
w.pending = append(w.pending, r)