amadan

carlos / platform Public

enginesim: clean baseline env, immediate listener checks, completed-transition coverage (#335 plan 1, final review)

mutate.sh's unmutated baseline sets ENGINESIM_MUTATE= explicitly.

The ports witness asserts the listener is gone right after the adoption
and connects with no capability pass in between; a second witness does
the same after --grant doc-engine; TestPortsListenerGoneOnNull checks the
upgrade's NULL write.

An action's run reports whether its transition completed; the walk counts
selections (stats) and completions (done) apart, plus deliveries and
engine calls that reached a process (events). TestEveryActionFires checks
completions and both events; TestStepLoop pins the split.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude a337fb28c2cabdf7397642ae7b43320884c9849d parent 04637ac
9 files changed, +315 −123
  • internal/sim/engines/calls_test.go +57 −30
  • internal/sim/engines/classify_test.go +37 −21
  • internal/sim/engines/dedicated_test.go +25 −15
  • internal/sim/engines/edge_test.go +1 −0
  • internal/sim/engines/launch_test.go +62 −28
  • internal/sim/engines/mutate.sh +3 −2
  • internal/sim/engines/sim_test.go +42 −15
  • internal/sim/engines/witness_test.go +70 −7
  • internal/sim/engines/world_test.go +18 −5
diff --git a/internal/sim/engines/calls_test.go b/internal/sim/engines/calls_test.go
index 78221e170..c38bdbb59 100644
--- a/internal/sim/engines/calls_test.go
+++ b/internal/sim/engines/calls_test.go
@@ -484,6 +484,7 @@ func (w *world) engineAdmit(home, from *box, mark caller, hostname, path, via, r
return false
}
w.logf("box%d: engine call reaches process %d (%s v%d); the access log records caller %s", home.ID, q.ID, q.Group, q.Ver, mark.app())
+ w.events["engine-call-reached"]++
w.checkCall(home, from, mark, via, real, p, q)
return true
}
@@ -666,11 +667,11 @@ func init() {
}
})
actions = append(actions,
- action{name: "engine-call", weight: 4, run: func(w *world) {
+ action{name: "engine-call", weight: 4, run: func(w *world) bool {
b := w.pickBox()
gs := w.callers(b)
if len(gs) == 0 || len(b.Procs) == 0 {
- return
+ return false
}
host := gs[w.rng.Intn(len(gs))]
// usually the instance's own process; one call in three, any
@@ -682,12 +683,13 @@ func init() {
c := call{from: b, p: p, uid: "carlos", host: host, engine: w.pickEngine(), path: w.pickPath()}
c.forged = w.maybeForged()
w.placeCall(c)
+ return true
}},
- action{name: "door-call", weight: 1, run: func(w *world) {
+ action{name: "door-call", weight: 1, run: func(w *world) bool {
b := w.pickBox()
gs := w.groups(b)
if len(gs) == 0 {
- return
+ return false
}
host := gs[w.rng.Intn(len(gs))]
c := call{from: b, viaDoor: true, host: host, engine: w.pickEngine(), path: w.pickPath()}
@@ -701,49 +703,54 @@ func init() {
}
c.forged = w.maybeForged()
w.placeCall(c)
+ return true
}},
- action{name: "client-hop", weight: 1, run: func(w *world) {
+ action{name: "client-hop", weight: 1, run: func(w *world) bool {
hosts := []string{engHost, engSite}
c := callerOf(walkApps[w.rng.Intn(len(walkApps))], w.pickEngine())
w.clientHop(w.pickBox(), hosts[w.rng.Intn(2)], w.pickPath(), c)
+ return true
}},
- action{name: "console-grant", weight: 1, run: func(w *world) { w.consoleGrant(w.pickKey()) }},
- action{name: "console-revoke", weight: 1, run: func(w *world) {
+ action{name: "console-grant", weight: 1, run: func(w *world) bool { return w.consoleGrant(w.pickKey()) }},
+ action{name: "console-revoke", weight: 1, run: func(w *world) bool {
keys := sortedKeys(w.console.Grants)
- if len(keys) > 0 {
- w.consoleRevoke(keys[w.rng.Intn(len(keys))])
- }
+ return len(keys) > 0 && w.consoleRevoke(keys[w.rng.Intn(len(keys))])
}},
- action{name: "console-resign", weight: 1, run: func(w *world) {
- if !w.console.Down {
- w.publish(w.console.Rev + 1)
+ action{name: "console-resign", weight: 1, run: func(w *world) bool {
+ if w.console.Down {
+ return false
}
+ w.publish(w.console.Rev + 1)
+ return true
}},
- action{name: "console-down", weight: 1, run: func(w *world) {
+ action{name: "console-down", weight: 1, run: func(w *world) bool {
w.console.Down = !w.console.Down && !w.calm()
w.logf("console: down = %v", w.console.Down)
+ return true
}},
- action{name: "bucket-down", weight: 1, run: func(w *world) {
+ action{name: "bucket-down", weight: 1, run: func(w *world) bool {
w.setBucketDown(!w.console.BucketDown && !w.calm())
+ return true
}},
- action{name: "fast-forward", weight: 1, run: func(w *world) {
+ action{name: "fast-forward", weight: 1, run: func(w *world) bool {
if w.calm() {
- return
+ return false
}
if w.rng.Intn(8) == 0 {
w.console.Down = true
w.logf("console: down = true")
w.advance(lifetimeTicks + staleTicks) // the console is down for three days
- return
+ return true
}
w.advance(staleTicks) // ten minutes with no poll
+ return true
}},
// any older signed record, at any time: the freshness bound, not the
// walk, limits what a replay can do (R5)
- action{name: "replay", weight: 1, run: func(w *world) {
- w.walkReplay()
+ action{name: "replay", weight: 1, run: func(w *world) bool {
+ return w.walkReplay()
}},
- action{name: "restart-edge", weight: 2, run: func(w *world) {
+ action{name: "restart-edge", weight: 2, run: func(w *world) bool {
switch w.rng.Intn(3) {
case 0: // a release roll restarts every edge
for _, b := range w.boxes {
@@ -759,14 +766,29 @@ func init() {
w.walkReplay()
}
}
+ return true
+ }},
+ action{name: "arm", weight: 2, run: func(w *world) bool {
+ b := w.pickBox()
+ if b.Armed {
+ return false
+ }
+ w.arm(b)
+ return true
+ }},
+ action{name: "disarm", weight: 1, run: func(w *world) bool {
+ b := w.pickBox()
+ if !b.Armed {
+ return false
+ }
+ w.disarm(b)
+ return true
}},
- action{name: "arm", weight: 2, run: func(w *world) { w.arm(w.pickBox()) }},
- action{name: "disarm", weight: 1, run: func(w *world) { w.disarm(w.pickBox()) }},
- action{name: "launch-unisolated", weight: 1, run: func(w *world) {
+ action{name: "launch-unisolated", weight: 1, run: func(w *world) bool {
b := w.pickBox()
kind := unisolatedKinds[w.rng.Intn(len(unisolatedKinds))]
if !w.launchUnisolated(b, kind) || !w.flatEraTamper || b.Armed {
- return
+ return true
}
var gs []string
for _, g := range w.groups(b) {
@@ -777,8 +799,9 @@ func init() {
if len(gs) > 0 {
w.tamper(b, w.newest(b, kind), gs[w.rng.Intn(len(gs))])
}
+ return true
}},
- action{name: "unisolated-exit", weight: 1, run: func(w *world) {
+ action{name: "unisolated-exit", weight: 1, run: func(w *world) bool {
b := w.pickBox()
var ps []*proc
for _, p := range b.Procs {
@@ -787,7 +810,7 @@ func init() {
}
}
if len(ps) == 0 {
- return
+ return false
}
p := ps[w.rng.Intn(len(ps))]
for i, x := range b.Procs {
@@ -797,12 +820,16 @@ func init() {
}
}
w.logf("box%d: process %d (%s) exited", b.ID, p.ID, p.Slice)
+ return true
}},
- action{name: "move-to-nested", weight: 1, run: func(w *world) {
+ action{name: "move-to-nested", weight: 1, run: func(w *world) bool {
b := w.pickBox()
- if ks := sortedKeys(b.Unisolated); len(ks) > 0 {
- w.moveToNested(b, ks[w.rng.Intn(len(ks))])
+ ks := sortedKeys(b.Unisolated)
+ if len(ks) == 0 {
+ return false
}
+ w.moveToNested(b, ks[w.rng.Intn(len(ks))])
+ return true
}},
)
}
diff --git a/internal/sim/engines/classify_test.go b/internal/sim/engines/classify_test.go
index 5e20dbc3c..84f6a9c53 100644
--- a/internal/sim/engines/classify_test.go
+++ b/internal/sim/engines/classify_test.go
@@ -111,8 +111,9 @@ func (w *world) mint(b *box, src, host string, recordOwner bool) {
// reconcileOwners is the reconciler's backfill pass: it records the owner of
// every minted instance row that has none. The row takes its owner group's
-// mark (one group, one mark).
-func (w *world) reconcileOwners(b *box) {
+// mark (one group, one mark). It reports whether it recorded any.
+func (w *world) reconcileOwners(b *box) bool {
+ did := false
for _, h := range hostsOf(b) {
r := b.Rows[h]
if !r.Platform || r.Kind != "instance" || r.Owner != "" || b.Rows[r.Source] == nil {
@@ -122,8 +123,10 @@ func (w *world) reconcileOwners(b *box) {
r.Owner = r.Source
b.Mark[h] = m
b.Wrote++
+ did = true
w.logf("box%d: registry change %d: owner of %s recorded as %s (mark %v)", b.ID, b.Wrote, h, r.Owner, m)
}
+ return did
}
// provision creates instance route host (a new route: NULL, unclassified),
@@ -487,31 +490,38 @@ const maxRows = 24
func init() {
actions = append(actions,
- action{name: "upgrade", weight: 1, run: func(w *world) {
+ action{name: "upgrade", weight: 1, run: func(w *world) bool {
b := w.pickBox()
if why := w.residualAtUpgrade(b); why != "" && !w.unwrittenManifestAcrossUpgrade {
w.logf("box%d: upgrade not generated: the stated residual (%s)", b.ID, why)
- return
+ return false
}
w.upgrade(b)
+ return true
}},
- action{name: "grant", weight: 1, run: func(w *world) {
+ action{name: "grant", weight: 1, run: func(w *world) bool {
b := w.pickBox()
- if h := w.pickInstanceRow(b); h != "" {
- w.grant(b, h)
+ h := w.pickInstanceRow(b)
+ if h == "" || b.Rows[h].Grant {
+ return false
}
+ w.grant(b, h)
+ return true
}},
- action{name: "revoke", weight: 1, run: func(w *world) {
+ action{name: "revoke", weight: 1, run: func(w *world) bool {
b := w.pickBox()
- if h := w.pickInstanceRow(b); h != "" {
- w.revoke(b, h)
+ h := w.pickInstanceRow(b)
+ if h == "" || !b.Rows[h].Grant {
+ return false
}
+ w.revoke(b, h)
+ return true
}},
- action{name: "operator-public", weight: 2, run: func(w *world) {
+ action{name: "operator-public", weight: 2, run: func(w *world) bool {
b := w.pickBox()
g := w.pickGroup(b)
if g == "" {
- return
+ return false
}
switch w.rng.Intn(5) {
case 1: // an adoption tries to run during the clear
@@ -542,17 +552,21 @@ func init() {
w.clearStopMisses = w.rng.Intn(4) == 0
w.operatorPublic(b, g)
w.clearHook, w.clearStopMisses = nil, false
+ return true
}},
- action{name: "adopt-awake", weight: 2, run: func(w *world) {
+ action{name: "adopt-awake", weight: 2, run: func(w *world) bool {
b := w.pickBox()
- if g := w.pickGroup(b); g != "" {
- w.repointAwake(b, g, w.pickVer())
+ g := w.pickGroup(b)
+ if g == "" {
+ return false
}
+ w.repointAwake(b, g, w.pickVer())
+ return true
}},
- action{name: "add-tcp-instance", weight: 1, run: func(w *world) {
+ action{name: "add-tcp-instance", weight: 1, run: func(w *world) bool {
b := w.pickBox()
if len(b.Rows) >= maxRows {
- return
+ return false
}
w.nextID++
host := fmt.Sprintf("t%d", w.nextID)
@@ -560,17 +574,19 @@ func init() {
addr := fmt.Sprintf("10.0.%d.1:%d", b.ID, 9100+w.nextID) // unique: no ambiguous owner in the walk
w.provision(b, host, app, addr, w.pickVer())
w.mint(b, host, host+".acct.apps.example", w.rng.Intn(2) == 0)
+ return true
}},
- action{name: "add-static-app", weight: 1, run: func(w *world) {
+ action{name: "add-static-app", weight: 1, run: func(w *world) bool {
b := w.pickBox()
if len(b.Rows) >= maxRows {
- return
+ return false
}
w.nextID++
w.addStaticApp(b, fmt.Sprintf("s%d", w.nextID))
+ return true
}},
- action{name: "reconcile-owners", weight: 1, run: func(w *world) {
- w.reconcileOwners(w.pickBox())
+ action{name: "reconcile-owners", weight: 1, run: func(w *world) bool {
+ return w.reconcileOwners(w.pickBox())
}},
)
}
diff --git a/internal/sim/engines/dedicated_test.go b/internal/sim/engines/dedicated_test.go
index 47c4e9062..620dc809e 100644
--- a/internal/sim/engines/dedicated_test.go
+++ b/internal/sim/engines/dedicated_test.go
@@ -340,6 +340,7 @@ func (w *world) reach(b *box, p *proc, public bool, how string) *proc {
return nil
}
w.logf("box%d: %s reached process %d (%s v%d) past the edge", b.ID, how, p.ID, p.Group, p.Ver)
+ w.events["delivered"]++
// a ports listener exists only because the group is public: a path the
// upgrade's 0 opens; any other way past the edge is the ingress rule's
listener := strings.HasPrefix(how, "the ports listener")
@@ -427,10 +428,10 @@ func (w *world) localBackends(b *box) []string {
func init() {
actions = append(actions,
- action{name: "add-service-row", weight: 2, run: func(w *world) {
+ action{name: "add-service-row", weight: 2, run: func(w *world) bool {
b := w.pickBox()
if len(b.Rows) >= maxRows {
- return
+ return false
}
w.nextID++
host := fmt.Sprintf("svc%d.example", w.nextID)
@@ -438,12 +439,13 @@ func init() {
// aimed at box 0's address (codex R13-1)
ports := append(w.addressablePorts(w.boxes[0]), "1")
w.addServiceRow(b, host, boxAddr(w.boxes[0])+":"+ports[w.rng.Intn(len(ports))])
- return
+ return true
}
bs := w.localBackends(b)
w.addServiceRow(b, host, bs[w.rng.Intn(len(bs))])
+ return true
}},
- action{name: "delete-service-row", weight: 2, run: func(w *world) {
+ action{name: "delete-service-row", weight: 2, run: func(w *world) bool {
type at struct {
b *box
host string
@@ -457,31 +459,37 @@ func init() {
}
}
if len(rows) == 0 {
- return
+ return false
}
x := rows[w.rng.Intn(len(rows))]
w.deleteRow(x.b, x.host)
+ return true
}},
- action{name: "own-port", weight: 1, run: func(w *world) {
+ action{name: "own-port", weight: 1, run: func(w *world) bool {
b := w.pickBox()
if len(b.Procs) == 0 {
- return
+ return false
}
w.bindOwnPort(b, b.Procs[w.rng.Intn(len(b.Procs))])
+ return true
}},
- action{name: "direct-connect", weight: 2, run: func(w *world) {
+ action{name: "direct-connect", weight: 2, run: func(w *world) bool {
b := w.pickBox()
ports := append(w.addressablePorts(b), "1")
w.directConnect(b, ports[w.rng.Intn(len(ports))], true)
+ return true
}},
- action{name: "ship-ports", weight: 1, run: func(w *world) {
+ action{name: "ship-ports", weight: 1, run: func(w *world) bool {
b := w.pickBox()
- if g := w.pickGroup(b); g != "" {
- w.ship(b, g, w.pickVer(), true)
+ g := w.pickGroup(b)
+ if g == "" {
+ return false
}
+ w.ship(b, g, w.pickVer(), true)
+ return true
}},
- action{name: "caps-pass", weight: 1, run: func(w *world) { w.capsPass(w.pickBox()) }},
- action{name: "remove-route", weight: 1, run: func(w *world) {
+ action{name: "caps-pass", weight: 1, run: func(w *world) bool { w.capsPass(w.pickBox()); return true }},
+ action{name: "remove-route", weight: 1, run: func(w *world) bool {
// an instance route the walk added (the starting eng and shop stay)
b := w.pickBox()
var gs []string
@@ -490,9 +498,11 @@ func init() {
gs = append(gs, g)
}
}
- if len(gs) > 0 {
- w.removeGroup(b, gs[w.rng.Intn(len(gs))])
+ if len(gs) == 0 {
+ return false
}
+ w.removeGroup(b, gs[w.rng.Intn(len(gs))])
+ return true
}},
)
}
diff --git a/internal/sim/engines/edge_test.go b/internal/sim/engines/edge_test.go
index 42d9b7e8a..f6506eee8 100644
--- a/internal/sim/engines/edge_test.go
+++ b/internal/sim/engines/edge_test.go
@@ -302,6 +302,7 @@ func (w *world) deliver(r *req) {
return
}
r.Reached = p.ID
+ w.events["delivered"]++
w.logf("box%d: request %d delivered to process %d (%s v%d)", b.ID, r.ID, p.ID, p.Group, p.Ver)
if r.Public && w.builds[p.Ver].Declares {
w.failProc(b, p, "INV-P", true, "box%d: public request %d (row %s) delivered to process %d of %s running v%d, which declares doc_engine", b.ID, r.ID, r.Row, p.ID, p.Group, p.Ver)
diff --git a/internal/sim/engines/launch_test.go b/internal/sim/engines/launch_test.go
index 61ce48892..b0fe537dd 100644
--- a/internal/sim/engines/launch_test.go
+++ b/internal/sim/engines/launch_test.go
@@ -231,8 +231,9 @@ func (w *world) stagedForApp(b *box, g string) []int {
// wake is the activator waking group g by its wake plan: "recovery" starts
// the recovery build, "fallback" first fails to start the live build and
// falls back; both choose GoodVersion. The triggering request, if any, is
-// delivered afterwards (or finds the fence cancelled it).
-func (w *world) wake(b *box, g string, mode string, trigger *req) {
+// delivered afterwards (or finds the fence cancelled it). It reports whether
+// there was a build to launch.
+func (w *world) wake(b *box, g string, mode string, trigger *req) bool {
ver := b.Good[g]
if mode == "fallback" {
w.logf("box%d: wake %s: live v%d fails to start; falling back to good v%d", b.ID, g, b.Live[g], ver)
@@ -245,6 +246,7 @@ func (w *world) wake(b *box, g string, mode string, trigger *req) {
if trigger != nil {
w.deliver(trigger)
}
+ return ver != 0
}
// moveLive is anything other than adoption moving group g's live link (a
@@ -261,22 +263,25 @@ func (w *world) moveLive(b *box, g string, ver int, who string) bool {
}
// systemdRestart is Restart=always: the unit's main process exits and
-// systemd starts /opt/carlos/live/<host> again, with no activator.
-func (w *world) systemdRestart(b *box, g string) {
+// systemd starts /opt/carlos/live/<host> again, with no activator. False
+// when the route has no live release.
+func (w *world) systemdRestart(b *box, g string) bool {
if b.Live[g] == 0 {
- return
+ return false
}
w.stopGroup(b, g)
w.launch(b, g, b.Live[g], "systemd")
+ return true
}
// socketActivate is carlos-app@.socket starting a stopped unit from the live
-// link, with no activator.
-func (w *world) socketActivate(b *box, g string) {
+// link, with no activator. False when there is nothing to start.
+func (w *world) socketActivate(b *box, g string) bool {
if b.Live[g] == 0 || w.running(b, g) {
- return
+ return false
}
w.launch(b, g, b.Live[g], "socket")
+ return true
}
// ---------- the random walk ----------
@@ -344,9 +349,12 @@ func (w *world) heldFor(b *box, g string) *req {
func init() {
actions = append(actions,
- action{name: "adopt", weight: 3, run: func(w *world) {
+ action{name: "adopt", weight: 3, run: func(w *world) bool {
b := w.pickBox()
g := w.pickGroup(b)
+ if g == "" {
+ return false
+ }
ver := w.pickVer()
var between func()
switch w.rng.Intn(3) {
@@ -361,30 +369,33 @@ func init() {
}
}
w.adopt(b, g, ver, between)
+ return true
}},
- action{name: "reload", weight: 4, run: func(w *world) { w.reload(w.pickBox()) }},
- action{name: "reload-fails", weight: 1, run: func(w *world) {
+ action{name: "reload", weight: 4, run: func(w *world) bool { w.reload(w.pickBox()); return true }},
+ action{name: "reload-fails", weight: 1, run: func(w *world) bool {
b := w.pickBox()
b.ReloadFails = !b.ReloadFails && !w.calm()
w.logf("box%d: reload fails = %v", b.ID, b.ReloadFails)
+ return true
}},
// 9, not 6 (Task 6): fence-box-wide's sweep catch needs a request held
// through a service row that is deleted before a declaring adoption's
// fence; at 6 the 3000-seed sweep caught it 7 times, at 9 12 times
- action{name: "public-request", weight: 9, run: func(w *world) {
+ action{name: "public-request", weight: 9, run: func(w *world) bool {
b := w.pickBox()
r := w.randomPublic(b)
if r == nil {
- return
+ return false
}
if len(b.Admitted) <= 8 && w.rng.Intn(2) == 0 {
r.Held = true
w.logf("box%d: request %d held in the activator's queue", b.ID, r.ID)
- return
+ return true
}
w.deliver(r)
+ return true
}},
- action{name: "resume", weight: 3, run: func(w *world) {
+ action{name: "resume", weight: 3, run: func(w *world) bool {
var held []*req
for _, b := range w.boxes {
for _, r := range b.Admitted {
@@ -394,24 +405,31 @@ func init() {
}
}
if len(held) == 0 {
- return
+ return false
}
sort.Slice(held, func(i, j int) bool { return held[i].ID < held[j].ID })
w.deliver(held[w.rng.Intn(len(held))])
+ return true
}},
- action{name: "wake", weight: 3, run: func(w *world) {
+ action{name: "wake", weight: 3, run: func(w *world) bool {
b := w.pickBox()
g := w.pickGroup(b)
+ if g == "" {
+ return false
+ }
mode := "recovery"
if w.rng.Intn(2) == 0 {
mode = "fallback"
}
w.stopGroup(b, g) // the group had gone to sleep
- w.wake(b, g, mode, w.heldFor(b, g))
+ return w.wake(b, g, mode, w.heldFor(b, g))
}},
- action{name: "swap", weight: 2, run: func(w *world) {
+ action{name: "swap", weight: 2, run: func(w *world) bool {
b := w.pickBox()
g := w.pickGroup(b)
+ if g == "" {
+ return false
+ }
if w.launch(b, g, w.pickVer(), "swap") {
// the incoming generation serves; the outgoing ones stop
newest := b.Procs[len(b.Procs)-1]
@@ -423,33 +441,49 @@ func init() {
}
b.Procs = kept
}
+ return true
}},
- action{name: "seed", weight: 1, run: func(w *world) {
+ action{name: "seed", weight: 1, run: func(w *world) bool {
b := w.pickBox()
g := w.pickGroup(b)
+ if g == "" {
+ return false
+ }
w.stopGroup(b, g)
w.launch(b, g, w.pickVer(), "seed")
+ return true
}},
- action{name: "cmd", weight: 1, run: func(w *world) {
+ action{name: "cmd", weight: 1, run: func(w *world) bool {
b := w.pickBox()
- w.launch(b, w.pickGroup(b), w.pickVer(), "cmd")
+ g := w.pickGroup(b)
+ if g == "" {
+ return false
+ }
+ w.launch(b, g, w.pickVer(), "cmd")
+ return true
}},
- action{name: "systemd-restart", weight: 2, run: func(w *world) {
+ action{name: "systemd-restart", weight: 2, run: func(w *world) bool {
b := w.pickBox()
- w.systemdRestart(b, w.pickGroup(b))
+ return w.systemdRestart(b, w.pickGroup(b))
}},
- action{name: "socket-activation", weight: 2, run: func(w *world) {
+ action{name: "socket-activation", weight: 2, run: func(w *world) bool {
b := w.pickBox()
g := w.pickGroup(b)
w.stopGroup(b, g) // the unit exited idle; a connection arrives
- w.socketActivate(b, g)
+ ok := w.socketActivate(b, g)
if r := w.heldFor(b, g); r != nil {
w.deliver(r)
}
+ return ok
}},
- action{name: "repoint-live", weight: 1, run: func(w *world) {
+ action{name: "repoint-live", weight: 1, run: func(w *world) bool {
b := w.pickBox()
- w.moveLive(b, w.pickGroup(b), w.pickVer(), "a rollout tool")
+ g := w.pickGroup(b)
+ if g == "" {
+ return false
+ }
+ w.moveLive(b, g, w.pickVer(), "a rollout tool")
+ return true
}},
)
}
diff --git a/internal/sim/engines/mutate.sh b/internal/sim/engines/mutate.sh
index a0b05813a..c16aae224 100755
--- a/internal/sim/engines/mutate.sh
+++ b/internal/sim/engines/mutate.sh
@@ -83,9 +83,10 @@ for f in "$SCRATCH"/verdicts/*; do
done
# the unmutated sweep first, at the same seed count: if the model violates
-# with every rule in place, no catch below means anything
+# with every rule in place, no catch below means anything (ENGINESIM_MUTATE
+# set empty here, so a value in the caller's environment cannot leak in)
blog="$SCRATCH/logs/unmutated.log"
-ENGINESIM_SEEDS=$SEEDS "$BIN" -test.run '^TestEngines$' -test.count=1 -test.v -test.timeout 180m >"$blog" 2>&1
+ENGINESIM_MUTATE= ENGINESIM_SEEDS=$SEEDS "$BIN" -test.run '^TestEngines$' -test.count=1 -test.v -test.timeout 180m >"$blog" 2>&1
brc=$?
if [ $brc -ne 0 ] || [ "$(grep -c "^ sim_test.go:[0-9]*: $SEEDS seeds, 0 failed, 0 residuals; mutated=\"\"" "$blog")" -ne 1 ]; then
echo "BASELINE unmutated: FAILED (rc=$brc); see $blog" >&2
diff --git a/internal/sim/engines/sim_test.go b/internal/sim/engines/sim_test.go
index b5bef5d7e..fc66252fb 100644
--- a/internal/sim/engines/sim_test.go
+++ b/internal/sim/engines/sim_test.go
@@ -42,13 +42,19 @@ func TestEngines(t *testing.T) {
failed := 0
count := map[string]int{}
firstSeed := map[string]int64{}
- stats := map[string]int{}
+ stats, done, events := map[string]int{}, map[string]int{}, map[string]int{}
residuals := 0
for seed := int64(1); seed <= int64(n); seed++ {
w := runSeed(seed, false)
for k, v := range w.stats {
stats[k] += v
}
+ for k, v := range w.done {
+ done[k] += v
+ }
+ for k, v := range w.events {
+ events[k] += v
+ }
residuals += len(w.residuals)
if len(w.violations) == 0 {
continue
@@ -64,7 +70,9 @@ func TestEngines(t *testing.T) {
}
}
t.Logf("%d seeds, %d failed, %d residuals; mutated=%q", n, failed, residuals, mutated)
- t.Logf("actions: %v", stats)
+ t.Logf("actions chosen: %v", stats)
+ t.Logf("actions completed: %v", done)
+ t.Logf("events: %v", events)
ids := make([]string, 0, len(count))
for id := range count {
ids = append(ids, id)
@@ -94,7 +102,7 @@ func TestSeedDeterministic(t *testing.T) {
if !reflect.DeepEqual(a.trace, b.trace) {
t.Fatalf("verbose=%v: two runs of seed 7 differ", verbose)
}
- if !reflect.DeepEqual(a.stats, b.stats) || !reflect.DeepEqual(a.violations, b.violations) {
+ if !reflect.DeepEqual(a.stats, b.stats) || !reflect.DeepEqual(a.done, b.done) || !reflect.DeepEqual(a.events, b.events) || !reflect.DeepEqual(a.violations, b.violations) {
t.Fatalf("verbose=%v: two runs of seed 7 differ in stats or violations", verbose)
}
}
@@ -202,8 +210,9 @@ func TestNewWorldShape(t *testing.T) {
}
// TestStepLoop: with no actions a step does nothing but advance w.step; with
-// actions registered each run is counted under the action's name, the walk
-// is weighted, and 400 chaos plus 200 calm steps run.
+// actions registered each run is counted under the action's name (in done
+// only when it reports a completed transition), the walk is weighted, and
+// 400 chaos plus 200 calm steps run.
func TestStepLoop(t *testing.T) {
saved := actions
defer func() { actions = saved }()
@@ -227,20 +236,25 @@ func TestStepLoop(t *testing.T) {
}
}
actions = []action{
- {name: "alpha", weight: 3, run: func(w *world) {
+ {name: "alpha", weight: 3, run: func(w *world) bool {
calls["alpha"]++
phase(w)
+ return true
}},
- {name: "beta", weight: 1, run: func(w *world) {
+ {name: "beta", weight: 1, run: func(w *world) bool {
calls["beta"]++
phase(w)
+ return false // chosen, but found nothing to do
}},
- {name: "never", weight: 0, run: func(w *world) { t.Error("a zero-weight action ran") }},
+ {name: "never", weight: 0, run: func(w *world) bool { t.Error("a zero-weight action ran"); return true }},
}
w = runSeed(3, false)
if w.stats["alpha"] != calls["alpha"] || w.stats["beta"] != calls["beta"] {
t.Fatalf("stats %v do not match calls %v", w.stats, calls)
}
+ if w.done["alpha"] != calls["alpha"] || w.done["beta"] != 0 {
+ t.Fatalf("done %v: want alpha's every run and none of beta's (calls %v)", w.done, calls)
+ }
if calls["alpha"]+calls["beta"] != chaosSteps+calmSteps {
t.Fatalf("each step should run one action: %v", calls)
}
@@ -288,8 +302,11 @@ func TestOnMutation(t *testing.T) {
}
// TestEveryActionFires: over 200 seeds of the unmutated walk every registered
-// action is chosen at least once. A rule that guards an action no seed fires
-// is untested however green the sweep looks (Review Focus 2).
+// action completes its transition at least once (chosen is not enough: an
+// action that always finds nothing to do exercises nothing), and public
+// requests and engine calls really reach processes. A rule that guards an
+// action no seed fires is untested however green the sweep looks (Review
+// Focus 2; final review, minor 3).
func TestEveryActionFires(t *testing.T) {
saved := mutated
mutated = ""
@@ -307,15 +324,25 @@ func TestEveryActionFires(t *testing.T) {
t.Errorf("action %q has weight %d: the walk never picks it", a.name, a.weight)
}
}
- stats := map[string]int{}
+ done, events := map[string]int{}, map[string]int{}
for seed := int64(1); seed <= 200; seed++ {
- for k, v := range runSeed(seed, false).stats {
- stats[k] += v
+ w := runSeed(seed, false)
+ for k, v := range w.done {
+ done[k] += v
+ }
+ for k, v := range w.events {
+ events[k] += v
}
}
for _, a := range actions {
- if stats[a.name] == 0 {
- t.Errorf("action %q never fired over 200 seeds", a.name)
+ if done[a.name] == 0 {
+ t.Errorf("action %q never completed a transition over 200 seeds", a.name)
+ }
+ }
+ for _, e := range []string{"delivered", "engine-call-reached"} {
+ if events[e] == 0 {
+ t.Errorf("no %s over 200 seeds", e)
}
}
+ t.Logf("completed: %v; events: %v", done, events)
}
diff --git a/internal/sim/engines/witness_test.go b/internal/sim/engines/witness_test.go
index 1fa64571f..cbc759729 100644
--- a/internal/sim/engines/witness_test.go
+++ b/internal/sim/engines/witness_test.go
@@ -90,7 +90,9 @@ var witnesses = []witness{
// Task 4: the dedicated engine box and ingress
{"ports-excluded", "ports-excluded", "INV-P",
- "R11-1: the engine route holds a port grant from a plain release, so the box renders its listener; adoption of a declaring build marks the group; a box that keeps the listener lets a public connection to the port, which the firewall admits as granted, reach the engine past the edge", wPortsExcluded},
+ "R11-1: the engine route holds a port grant from a plain release, so the box renders its listener; adoption of a declaring build marks the group; a box that keeps the listener lets a public connection to the port, which the firewall admits as granted, reach the engine past the edge, with no capability pass in between", wPortsExcluded},
+ {"ports-excluded", "ports-excluded", "INV-P",
+ "R11-1, by the grant: the public engine route on a plain build holds a port grant and its listener; --grant doc-engine marks the group; a box that keeps the listener lets a public connection reach the granted group past the edge, with no capability pass in between", wPortsExcludedGrant},
{"dedicated-launch-check", "dedicated-launch-check,dedicated-edge-refusal", "INV-P",
"R12-2: redundant alone: a service row aimed at the engine's backend exists before a declaring adoption; the launch check refuses the adoption; without it the edge's refusal still refuses the row once the group is internal; with both gone a public request through the row reaches the engine", wDedicatedLaunchCheck},
{"dedicated-edge-refusal", "dedicated-edge-refusal", "INV-P",
@@ -1112,11 +1114,10 @@ func (w *world) runningEngine(t *testing.T) (*box, *proc) {
return b, b.Procs[len(b.Procs)-1]
}
-// wPortsExcluded (R11-1): the public engine route ships a plain release
-// declaring ports, and the box renders its listener; then adoption of a
-// declaring build; the capability pass runs; a public connection arrives on
-// the granted port.
-func wPortsExcluded(t *testing.T, w *world) {
+// portsEngine is the public engine route shipping a plain release declaring
+// ports: the box renders its listener, and a public connection to the port
+// reaches the plain build. It returns the port.
+func (w *world) portsEngine(t *testing.T) (*box, string) {
b := w.publicEngine(t)
if !w.ship(b, eng, vPlain, true) {
t.Fatalf("witness: a plain release declaring ports was refused: %s", w.boxesState())
@@ -1129,15 +1130,77 @@ func wPortsExcluded(t *testing.T, w *world) {
t.Fatalf("witness: the granted port of a public route reached nothing: %s", w.boxesState())
}
w.check()
+ return b, port
+}
+
+// listenerGone fails the witness if the write that closed the group left
+// port's listener in place, while ports-excluded holds: the write removes it
+// itself, before any fence and with no capability pass (final review, minor
+// 2).
+func (w *world) listenerGone(t *testing.T, b *box, port, after string) {
+ if on("ports-excluded") && b.Listen[port] != "" {
+ t.Fatalf("witness: the listener on %s is still there right after %s: %s", port, after, w.boxesState())
+ }
+}
+
+// wPortsExcluded (R11-1): the public engine route ships a plain release
+// declaring ports, and the box renders its listener; then adoption of a
+// declaring build; at once, with no capability pass, a public connection
+// arrives on the granted port.
+func wPortsExcluded(t *testing.T, w *world) {
+ b, port := w.portsEngine(t)
if !w.adopt(b, eng, vEngine, nil) {
t.Fatalf("witness: adoption of the engine refused: %s", w.boxesState())
}
+ w.listenerGone(t, b, port, "the adoption")
+ w.check()
+ w.directConnect(b, port, true)
+ w.check()
+}
+
+// wPortsExcludedGrant (R11-1, by the grant): the public engine route serves a
+// plain build on its ports listener; the operator grants doc-engine on it; at
+// once, with no capability pass, a public connection arrives on the port.
+func wPortsExcludedGrant(t *testing.T, w *world) {
+ b, port := w.portsEngine(t)
+ w.grant(b, engHost)
+ w.listenerGone(t, b, port, "the grant")
w.check()
- w.capsPass(b)
w.directConnect(b, port, true)
w.check()
}
+// TestPortsListenerGoneOnNull (final review, minor 2): the upgrade leaves a
+// route with no live release NULL; the write of NULL removes its ports
+// listener at once, before the capability pass runs, and a public connection
+// to the port is refused by the firewall.
+func TestPortsListenerGoneOnNull(t *testing.T) {
+ w := newWitness()
+ b := w.boxes[1]
+ b.PortGrant["shop"] = "40001"
+ w.capsPass(b)
+ if b.Listen["40001"] != "shop" {
+ t.Fatalf("fixture: no listener for shop's grant: %s", w.boxesState())
+ }
+ w.upgrade(b)
+ if m := w.groupMark(b, "shop"); m != markNull {
+ t.Fatalf("fixture: shop is %v after the upgrade, want NULL", m)
+ }
+ if b.Listen["40001"] != "" {
+ t.Errorf("the listener is still there right after the upgrade's NULL: %s", w.boxesState())
+ }
+ if p := w.directConnect(b, "40001", true); p != nil {
+ t.Errorf("a public connection to the NULL route's port reached process %d", p.ID)
+ }
+ w.check()
+ if len(w.violations) > 0 {
+ t.Errorf("violations: %v", w.violations)
+ }
+ if t.Failed() {
+ dump(t, w)
+ }
+}
+
// wDedicatedLaunchCheck (R12-2): the public engine route has a service row
// beside it aimed at its backend; adoption of a declaring build; a public
// request arrives through the service row.
diff --git a/internal/sim/engines/world_test.go b/internal/sim/engines/world_test.go
index f3d8196d4..69497a060 100644
--- a/internal/sim/engines/world_test.go
+++ b/internal/sim/engines/world_test.go
@@ -162,8 +162,10 @@ type world struct {
trace []string
violations []string
residuals []string
- stats map[string]int
- nextID int // process and request ids
+ stats map[string]int // actions selected, by name
+ done map[string]int // actions whose transition completed, by name
+ events map[string]int // deliveries: public requests and engine calls that reached a process
+ nextID int // process and request ids
// clearHook, when set, runs inside operatorPublic at each stage
// ("locked": step 1 passed; "stopped": step 2 done; "confirmed": step 3
@@ -193,10 +195,16 @@ type world struct {
// action is one thing the random walk can do. Tasks 2 to 5 append theirs
// from their own files' init; the walk picks among them by weight.
+//
+// run reports whether the transition completed: false when the action found
+// nothing to act on (no group, no held request, a full table) and returned
+// without calling into the model. A refusal by the protocol is a completed
+// transition. The walk counts selections (stats) and completions (done)
+// apart, so coverage can be checked on what really ran.
type action struct {
name string
weight int
- run func(*world)
+ run func(*world) bool
}
var actions []action
@@ -211,6 +219,8 @@ func newWorld(seed int64, verbose bool) *world {
rng: rand.New(rand.NewSource(seed)),
builds: map[int]build{},
stats: map[string]int{},
+ done: map[string]int{},
+ events: map[string]int{},
verbose: verbose,
}
newBox := func(id int) *box {
@@ -311,7 +321,8 @@ func (w *world) pick() *action {
}
// doStep advances one tick: it counts the step, runs one chosen action and
-// counts it under its name.
+// counts it under its name, in stats when chosen and in done when its
+// transition completed.
func (w *world) doStep() {
w.step++
for _, h := range stepHooks {
@@ -326,7 +337,9 @@ func (w *world) doStep() {
}
w.stats[a.name]++
w.logf("action %s", a.name) // always traced: the determinism test compares the action sequence
- a.run(w)
+ if a.run(w) {
+ w.done[a.name]++
+ }
w.check()
}