amadan
enginesim: the world, step loop and sweep (#335 plan 1)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
4 files changed,
+527
−0
internal/sim/engines/invariants_test.go+26 −0internal/sim/engines/rules_test.go+38 −0internal/sim/engines/sim_test.go+238 −0internal/sim/engines/world_test.go+225 −0
diff --git a/internal/sim/engines/invariants_test.go b/internal/sim/engines/invariants_test.go| new file mode 100644 |
| index 000000000..a1e618777 |
| --- /dev/null |
| +++ b/internal/sim/engines/invariants_test.go |
| @@ -0,0 +1,26 @@ |
| +package engines |
| + |
| +import ( |
| + "fmt" |
| + "strings" |
| +) |
| + |
| +// fail records a violation of invariant inv, once per invariant id: the |
| +// first message is the one the sweep reports and the later ones are the same |
| +// fault again. The stored text begins "<inv>: ", which invariantOf reads back. |
| +func (w *world) fail(inv, format string, args ...any) { |
| + for _, v := range w.violations { |
| + if invariantOf(v) == inv { |
| + return |
| + } |
| + } |
| + w.violations = append(w.violations, inv+": "+fmt.Sprintf(format, args...)) |
| +} |
| + |
| +// invariantOf is the invariant id a violation string was recorded under. |
| +func invariantOf(violation string) string { |
| + if i := strings.Index(violation, ": "); i >= 0 { |
| + return violation[:i] |
| + } |
| + return violation |
| +} |
diff --git a/internal/sim/engines/rules_test.go b/internal/sim/engines/rules_test.go| new file mode 100644 |
| index 000000000..e7dcc6500 |
| --- /dev/null |
| +++ b/internal/sim/engines/rules_test.go |
| @@ -0,0 +1,38 @@ |
| +package engines |
| + |
| +import ( |
| + "os" |
| + "strings" |
| +) |
| + |
| +// Every protocol rule the model enforces has a name here. The mutation |
| +// harness runs one prebuilt test binary with ENGINESIM_MUTATE set to one |
| +// rule name (or a comma-separated combination), and requires at least one |
| +// seed to fail. A rule whose mutation no seed catches is either redundant or |
| +// untested, and the run says which. |
| +// |
| +// Never edit `mutated` in place; the harness sets ENGINESIM_MUTATE. |
| +var mutated = "" |
| + |
| +// ENGINESIM_MUTATE sets `mutated` for one run of a prebuilt test binary, so |
| +// the mutation harness never edits or copies the source (mutate.sh). |
| +func init() { |
| + if v := os.Getenv("ENGINESIM_MUTATE"); v != "" { |
| + mutated = v |
| + } |
| +} |
| + |
| +// on reports whether a rule is enforced: false only when the mutation set |
| +// names it. |
| +func on(rule string) bool { |
| + for _, m := range strings.Split(mutated, ",") { |
| + if m == rule { |
| + return false |
| + } |
| + } |
| + return true |
| +} |
| + |
| +// allRules is the mutation list. Each task appends its rules, in spec order, |
| +// with a comment naming the spec section each answers. |
| +var allRules = []string{} |
diff --git a/internal/sim/engines/sim_test.go b/internal/sim/engines/sim_test.go| new file mode 100644 |
| index 000000000..f909ba937 |
| --- /dev/null |
| +++ b/internal/sim/engines/sim_test.go |
| @@ -0,0 +1,238 @@ |
| +package engines |
| + |
| +import ( |
| + "os" |
| + "reflect" |
| + "sort" |
| + "strconv" |
| + "strings" |
| + "testing" |
| +) |
| + |
| +// seedsFromEnv is the sweep size: ENGINESIM_SEEDS, else def. |
| +func seedsFromEnv(def int) int { |
| + if s := os.Getenv("ENGINESIM_SEEDS"); s != "" { |
| + if n, err := strconv.Atoi(s); err == nil && n > 0 { |
| + return n |
| + } |
| + } |
| + return def |
| +} |
| + |
| +// TestEngines is the sweep. ENGINESIM_SEEDS sets the seed count (the gate's |
| +// default is 200); ENGINESIM_SEED=n runs one seed and prints its trace. |
| +func TestEngines(t *testing.T) { |
| + if s := os.Getenv("ENGINESIM_SEED"); s != "" { |
| + seed, err := strconv.ParseInt(s, 10, 64) |
| + if err != nil { |
| + t.Fatalf("ENGINESIM_SEED=%q: %v", s, err) |
| + } |
| + w := runSeed(seed, true) |
| + for _, line := range w.trace { |
| + t.Log(line) |
| + } |
| + for _, v := range w.violations { |
| + t.Errorf("seed %d: %s", seed, v) |
| + } |
| + return |
| + } |
| + n := seedsFromEnv(200) |
| + failed := 0 |
| + count := map[string]int{} |
| + firstSeed := map[string]int64{} |
| + stats := map[string]int{} |
| + residuals := 0 |
| + for seed := int64(1); seed <= int64(n); seed++ { |
| + w := runSeed(seed, false) |
| + for k, v := range w.stats { |
| + stats[k] += v |
| + } |
| + residuals += len(w.residuals) |
| + if len(w.violations) == 0 { |
| + continue |
| + } |
| + failed++ |
| + for _, v := range w.violations { |
| + id := invariantOf(v) |
| + count[id]++ |
| + if _, ok := firstSeed[id]; !ok { |
| + firstSeed[id] = seed |
| + t.Logf("seed %d: %s", seed, v) |
| + } |
| + } |
| + } |
| + t.Logf("%d seeds, %d failed, %d residuals; mutated=%q", n, failed, residuals, mutated) |
| + t.Logf("actions: %v", stats) |
| + ids := make([]string, 0, len(count)) |
| + for id := range count { |
| + ids = append(ids, id) |
| + } |
| + sort.Strings(ids) |
| + for _, id := range ids { |
| + t.Logf(" %4d x %s (first seed %d)", count[id], id, firstSeed[id]) |
| + } |
| + if failed > 0 { |
| + t.Fatalf("%d of %d seeds violated an invariant", failed, n) |
| + } |
| +} |
| + |
| +// TestSeedDeterministic: one seed gives one trace, or no reported seed can |
| +// be replayed. |
| +func TestSeedDeterministic(t *testing.T) { |
| + for _, verbose := range []bool{false, true} { |
| + a, b := runSeed(7, verbose), runSeed(7, verbose) |
| + if len(a.trace) == 0 { |
| + t.Fatalf("verbose=%v: the trace is empty, so equality proves nothing", verbose) |
| + } |
| + if !reflect.DeepEqual(a.trace, b.trace) { |
| + t.Fatalf("verbose=%v: two runs of seed 7 differ", verbose) |
| + } |
| + if !reflect.DeepEqual(a.stats, b.stats) || !reflect.DeepEqual(a.violations, b.violations) { |
| + t.Fatalf("verbose=%v: two runs of seed 7 differ in stats or violations", verbose) |
| + } |
| + } |
| + if reflect.DeepEqual(newWorld(7, false).builds, newWorld(8, false).builds) && |
| + reflect.DeepEqual(newWorld(7, false).builds, newWorld(9, false).builds) && |
| + reflect.DeepEqual(newWorld(7, false).builds, newWorld(10, false).builds) { |
| + t.Fatal("builds do not vary with the seed") |
| + } |
| +} |
| + |
| +// TestRulesListed prints the rules a run would check, one per line, and runs |
| +// nothing. Without ENGINESIM_LIST it stands aside. |
| +func TestRulesListed(t *testing.T) { |
| + if os.Getenv("ENGINESIM_LIST") != "1" { |
| + t.Skip("ENGINESIM_LIST=1 prints the rules") |
| + } |
| + for _, r := range allRules { |
| + t.Log(r) |
| + } |
| +} |
| + |
| +// TestNewWorldShape pins the starting world the brief describes. |
| +func TestNewWorldShape(t *testing.T) { |
| + w := newWorld(1, false) |
| + if len(w.boxes) != 2 || w.boxes[0].ID != 0 || w.boxes[1].ID != 1 { |
| + t.Fatalf("want boxes 0 and 1, got %d", len(w.boxes)) |
| + } |
| + const eng = "eng.acct.apps.example" |
| + r := w.boxes[0].Rows[eng] |
| + if r == nil || !r.Platform || r.Kind != "instance" || r.Owner != "eng" || r.Backend == "" { |
| + t.Fatalf("box 0 lacks the minted platform row %s: %+v", eng, r) |
| + } |
| + if got := w.group(w.boxes[0], eng); got != "eng" { |
| + t.Fatalf("group of the minted row = %q, want eng", got) |
| + } |
| + if m := w.boxes[0].Mark[eng]; m != markInternal { |
| + t.Fatalf("the engine row's mark = %v, want markInternal", m) |
| + } |
| + s := w.boxes[1].Rows["shop"] |
| + if s == nil || s.Platform || s.Owner != "" || s.Backend == "" { |
| + t.Fatalf("box 1 lacks the public app row shop: %+v", s) |
| + } |
| + if got := w.group(w.boxes[1], "shop"); got != "shop" { |
| + t.Fatalf("group of an unminted row = %q, want its host", got) |
| + } |
| + if m := w.boxes[1].Mark["shop"]; m != markPublic { |
| + t.Fatalf("shop's mark = %v, want markPublic", m) |
| + } |
| + for v := 1; v <= 4; v++ { |
| + if b, ok := w.builds[v]; !ok || b.Ver != v { |
| + t.Fatalf("build %d missing or mislabelled: %+v", v, b) |
| + } |
| + } |
| + if len(w.builds) != 4 { |
| + t.Fatalf("want builds 1 to 4, got %d", len(w.builds)) |
| + } |
| + for _, b := range w.boxes { |
| + if b.Rows == nil || b.Mark == nil || b.Live == nil || b.Good == nil || b.Staged == nil { |
| + t.Fatalf("box %d has a nil map", b.ID) |
| + } |
| + } |
| + // a service row and a static row group by their own host |
| + b := w.boxes[1] |
| + b.Rows["svc"] = &row{Host: "svc", Kind: "service", Owner: "x"} |
| + if got := w.group(b, "svc"); got != "svc" { |
| + t.Fatalf("group of a service row = %q, want svc", got) |
| + } |
| +} |
| + |
| +// TestStepLoop: with no actions a step does nothing but advance w.step; with |
| +// actions registered each run is counted under the action's name, the walk |
| +// is weighted, and 400 chaos plus 200 calm steps run. |
| +func TestStepLoop(t *testing.T) { |
| + saved := actions |
| + defer func() { actions = saved }() |
| + |
| + actions = nil |
| + w := runSeed(3, false) |
| + if w.step != chaosSteps+calmSteps { |
| + t.Fatalf("w.step = %d, want %d", w.step, chaosSteps+calmSteps) |
| + } |
| + if len(w.stats) != 0 || len(w.violations) != 0 { |
| + t.Fatalf("no actions: stats %v violations %v", w.stats, w.violations) |
| + } |
| + |
| + calls := map[string]int{} |
| + var chaosCalls, calmCalls int |
| + actions = []action{ |
| + {name: "alpha", weight: 3, run: func(w *world) { |
| + calls["alpha"]++ |
| + if w.calm() { |
| + calmCalls++ |
| + } else { |
| + chaosCalls++ |
| + } |
| + }}, |
| + {name: "beta", weight: 1, run: func(w *world) { calls["beta"]++ }}, |
| + {name: "never", weight: 0, run: func(w *world) { t.Error("a zero-weight action ran") }}, |
| + } |
| + w = runSeed(3, false) |
| + if w.stats["alpha"] != calls["alpha"] || w.stats["beta"] != calls["beta"] { |
| + t.Fatalf("stats %v do not match calls %v", w.stats, calls) |
| + } |
| + if calls["alpha"]+calls["beta"] != chaosSteps+calmSteps { |
| + t.Fatalf("each step should run one action: %v", calls) |
| + } |
| + if calls["alpha"] < 2*calls["beta"] { |
| + t.Fatalf("weights ignored: %v", calls) |
| + } |
| + if chaosCalls == 0 || calmCalls == 0 { |
| + t.Fatalf("the walk must reach both phases: chaos %d calm %d", chaosCalls, calmCalls) |
| + } |
| + if _, ok := w.stats["never"]; ok { |
| + t.Fatal("a zero-weight action was counted") |
| + } |
| +} |
| + |
| +// TestFailOncePerInvariant: fail records the first violation of each |
| +// invariant id and ignores the repeats. |
| +func TestFailOncePerInvariant(t *testing.T) { |
| + w := newWorld(1, false) |
| + w.fail("INV-X", "first %d", 1) |
| + w.fail("INV-X", "second %d", 2) |
| + w.fail("INV-Y", "other") |
| + if len(w.violations) != 2 { |
| + t.Fatalf("violations = %v, want two", w.violations) |
| + } |
| + if !strings.HasPrefix(w.violations[0], "INV-X: first 1") || !strings.HasPrefix(w.violations[1], "INV-Y: other") { |
| + t.Fatalf("violations = %v", w.violations) |
| + } |
| + if invariantOf(w.violations[0]) != "INV-X" { |
| + t.Fatalf("invariantOf = %q", invariantOf(w.violations[0])) |
| + } |
| +} |
| + |
| +// TestOnMutation: on is false only for a rule named in the mutation set. |
| +func TestOnMutation(t *testing.T) { |
| + saved := mutated |
| + defer func() { mutated = saved }() |
| + mutated = "" |
| + if !on("any-rule") { |
| + t.Fatal("no mutation: every rule is on") |
| + } |
| + mutated = "a-rule,b-rule" |
| + if on("a-rule") || on("b-rule") || !on("c-rule") { |
| + t.Fatal("only the named rules are off") |
| + } |
| +} |
diff --git a/internal/sim/engines/world_test.go b/internal/sim/engines/world_test.go| new file mode 100644 |
| index 000000000..5bcb10d73 |
| --- /dev/null |
| +++ b/internal/sim/engines/world_test.go |
| @@ -0,0 +1,225 @@ |
| +package engines |
| + |
| +import ( |
| + "fmt" |
| + "math/rand" |
| + "sort" |
| +) |
| + |
| +// The model of the fleet-internal engines (spec |
| +// docs/superpowers/specs/2026-10-21-fleet-internal-engines-design.md). One |
| +// tick is 10 s: a step is a tick. Everything is decided from one seeded |
| +// source; nothing reads the clock, starts a goroutine or lets a map's |
| +// iteration order choose. |
| + |
| +const ( |
| + chaosSteps = 400 // faults and every action are eligible |
| + calmSteps = 200 // the same walk, then the final oracle |
| + nBuilds = 4 |
| +) |
| + |
| +// build is a platform build: whether it declares the internal-engine |
| +// protocol, and whether the manifest it writes is in place. |
| +type build struct { |
| + Ver int |
| + Declares, ManifestOK bool |
| +} |
| + |
| +// proc is a running process. Slice is the cgroup the peer check reads: |
| +// inst:<group>, exec:<host> or unit:<host>. |
| +type proc struct { |
| + ID int |
| + Group string |
| + Ver int |
| + Slice string |
| + Unisolated, OwnPort bool |
| +} |
| + |
| +// row is a host row of a box's table. Kind is instance, service or static; |
| +// Backend is a socket path or an address, "" for static. |
| +type row struct { |
| + Host, Kind, Owner, Backend string |
| + Platform, Grant bool |
| + PathTo map[string]string |
| +} |
| + |
| +// mark is a row's exposure mark. |
| +type mark int |
| + |
| +const ( |
| + markNull mark = iota |
| + markPublic |
| + markInternal |
| +) |
| + |
| +func (m mark) String() string { |
| + switch m { |
| + case markPublic: |
| + return "public" |
| + case markInternal: |
| + return "internal" |
| + } |
| + return "null" |
| +} |
| + |
| +// caller names who a request is on behalf of. |
| +type caller struct{ Account, App, Engine string } |
| + |
| +// req is a request the box admitted or refused. |
| +type req struct { |
| + ID, Epoch int |
| + Row, Backend string |
| + Public bool |
| + Caller *caller |
| + Held bool |
| +} |
| + |
| +// box is one host of the fleet: box 0 is the engine's home, box 1 a calling |
| +// box. |
| +type box struct { |
| + ID int |
| + Rows map[string]*row |
| + Mark map[string]mark |
| + Wrote, Fenced, Epoch int |
| + ReloadFails, Armed bool |
| + Live, Good map[string]int |
| + Staged map[string][]int |
| + Procs []*proc |
| + Admitted []*req |
| +} |
| + |
| +type world struct { |
| + seed int64 |
| + rng *rand.Rand |
| + boxes []*box |
| + builds map[int]build |
| + step int |
| + trace []string |
| + violations []string |
| + residuals []string |
| + stats map[string]int |
| + |
| + verbose bool |
| +} |
| + |
| +// action is one thing the random walk can do. Tasks 2 to 5 append theirs |
| +// from their own files' init; the walk picks among them by weight. |
| +type action struct { |
| + name string |
| + weight int |
| + run func(*world) |
| +} |
| + |
| +var actions []action |
| + |
| +func newWorld(seed int64, verbose bool) *world { |
| + w := &world{ |
| + seed: seed, |
| + rng: rand.New(rand.NewSource(seed)), |
| + builds: map[int]build{}, |
| + stats: map[string]int{}, |
| + verbose: verbose, |
| + } |
| + newBox := func(id int) *box { |
| + return &box{ID: id, Rows: map[string]*row{}, Mark: map[string]mark{}, Live: map[string]int{}, Good: map[string]int{}, Staged: map[string][]int{}} |
| + } |
| + home, calling := newBox(0), newBox(1) |
| + w.boxes = []*box{home, calling} |
| + |
| + // one engine group, eng, with its minted platform row |
| + const engHost = "eng.acct.apps.example" |
| + home.Rows[engHost] = &row{Host: engHost, Kind: "instance", Owner: "eng", Backend: "/run/carlos/eng.sock", Platform: true, PathTo: map[string]string{}} |
| + home.Mark[engHost] = markInternal |
| + // one public app group, shop, on the calling box |
| + calling.Rows["shop"] = &row{Host: "shop", Kind: "instance", Backend: "127.0.0.1:9001", PathTo: map[string]string{}} |
| + calling.Mark["shop"] = markPublic |
| + |
| + for v := 1; v <= nBuilds; v++ { |
| + w.builds[v] = build{Ver: v, Declares: w.rng.Intn(2) == 0, ManifestOK: w.rng.Intn(2) == 0} |
| + } |
| + w.logf("world seed %d: builds %s", seed, w.buildsString()) |
| + return w |
| +} |
| + |
| +func (w *world) buildsString() string { |
| + var out []string |
| + for v := 1; v <= nBuilds; v++ { |
| + b := w.builds[v] |
| + out = append(out, fmt.Sprintf("%d{declares=%v manifest=%v}", v, b.Declares, b.ManifestOK)) |
| + } |
| + return fmt.Sprint(out) |
| +} |
| + |
| +// logf appends a trace line. The trace is always kept: it is what the |
| +// determinism test compares, and what ENGINESIM_SEED prints. |
| +func (w *world) logf(format string, args ...any) { |
| + w.trace = append(w.trace, fmt.Sprintf("[%d] ", w.step)+fmt.Sprintf(format, args...)) |
| +} |
| + |
| +// calm is true once the chaos steps are over. |
| +func (w *world) calm() bool { return w.step >= chaosSteps } |
| + |
| +// group is a row's group key: its owner for a minted instance row, else its |
| +// own host. |
| +func (w *world) group(b *box, host string) string { |
| + if r := b.Rows[host]; r != nil && r.Kind == "instance" && r.Owner != "" { |
| + return r.Owner |
| + } |
| + return host |
| +} |
| + |
| +// pick chooses one registered action by weight. The registry is sorted by |
| +// name first, so the order files' init functions ran in never matters. |
| +func (w *world) pick() *action { |
| + sorted := make([]action, 0, len(actions)) |
| + total := 0 |
| + for _, a := range actions { |
| + if a.weight > 0 { |
| + sorted = append(sorted, a) |
| + total += a.weight |
| + } |
| + } |
| + if total == 0 { |
| + return nil |
| + } |
| + sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].name < sorted[j].name }) |
| + n := w.rng.Intn(total) |
| + for i := range sorted { |
| + if n < sorted[i].weight { |
| + return &sorted[i] |
| + } |
| + n -= sorted[i].weight |
| + } |
| + return nil |
| +} |
| + |
| +// doStep advances one tick: it counts the step, runs one chosen action and |
| +// counts it under its name. |
| +func (w *world) doStep() { |
| + w.step++ |
| + a := w.pick() |
| + if a == nil { |
| + if w.verbose { |
| + w.logf("idle") |
| + } |
| + return |
| + } |
| + w.stats[a.name]++ |
| + w.logf("action %s", a.name) // always traced: the determinism test compares the action sequence |
| + a.run(w) |
| +} |
| + |
| +// final is the end-of-run oracle: what must hold once the calm steps are |
| +// over. Later tasks add their end-state checks here. |
| +func (w *world) final() { |
| + w.logf("final: %d steps, %d violations, %d residuals", w.step, len(w.violations), len(w.residuals)) |
| +} |
| + |
| +func runSeed(seed int64, verbose bool) *world { |
| + w := newWorld(seed, verbose) |
| + for i := 0; i < chaosSteps+calmSteps; i++ { |
| + w.doStep() |
| + } |
| + w.final() |
| + return w |
| +} |