| 1 | package engines |
| 2 | |
| 3 | import ( |
| 4 | "fmt" |
| 5 | "math/rand" |
| 6 | "sort" |
| 7 | ) |
| 8 | |
| 9 | // The model of the fleet-internal engines (spec |
| 10 | // docs/superpowers/specs/2026-10-21-fleet-internal-engines-design.md). One |
| 11 | // tick is 10 s: a step is a tick. Everything is decided from one seeded |
| 12 | // source; nothing reads the clock, starts a goroutine or lets a map's |
| 13 | // iteration order choose. |
| 14 | |
| 15 | const ( |
| 16 | chaosSteps = 400 // faults and every action are eligible |
| 17 | calmSteps = 200 // the same walk, then the final oracle |
| 18 | nBuilds = 4 |
| 19 | ) |
| 20 | |
| 21 | // The starting world's names. |
| 22 | const ( |
| 23 | eng = "eng" // the engine group (its owner's host; engine names have no dot) |
| 24 | engHost = "eng.acct.apps.example" |
| 25 | engAlias = "docs.example" // alias row on box 0, to engHost |
| 26 | engSite = "www.example" // static row on box 0, path map to engHost |
| 27 | engSitePath = "/convert" |
| 28 | shopAlias = "shop-alias.example" // alias row on box 1, to shop |
| 29 | shopSite = "shop-www.example" // static row on box 1, path map to shop |
| 30 | shopSitePath = "/app" |
| 31 | ) |
| 32 | |
| 33 | // build is a platform build: whether it declares the internal-engine |
| 34 | // protocol, and whether the manifest it writes is in place. |
| 35 | type build struct { |
| 36 | Ver int |
| 37 | Declares, ManifestOK bool |
| 38 | } |
| 39 | |
| 40 | // proc is a running process. Slice is the cgroup the peer check reads: |
| 41 | // inst:<group>, exec:<host> or unit:<host>. |
| 42 | // |
| 43 | // Code is the account/app whose code the process runs when that is not its |
| 44 | // group's: code a flat-era tamper planted in the group's instance directory |
| 45 | // (Task 5). Residual names the stated residual the process belongs to, if |
| 46 | // any; a breach it causes is recorded as that residual, not a violation. |
| 47 | type proc struct { |
| 48 | ID int |
| 49 | Group string |
| 50 | Ver int |
| 51 | Slice string |
| 52 | Unisolated, OwnPort bool |
| 53 | Code, Residual string |
| 54 | } |
| 55 | |
| 56 | // row is a host row of a box's table. Kind is instance, service, static or |
| 57 | // alias; Backend is a socket path or an address, "" for static and alias. |
| 58 | // An alias row names its target host in Alias and copies its upstream |
| 59 | // (edge.go:929); PathTo maps a path to the host that serves it (edge.go:1364). |
| 60 | // |
| 61 | // A platform-minted row (Platform) names in Source the host it was minted |
| 62 | // from: the truth the oracle reads. Owner is what the registry has recorded |
| 63 | // (routes.owner), "" for a row minted before owners were recorded; the edge |
| 64 | // then finds the owner by App and backend (spec §5.1, "Platform-minted |
| 65 | // rows"). App is the row's account/app. GrantAt is the registry change that |
| 66 | // recorded the doc-engine grant. |
| 67 | type row struct { |
| 68 | Host, Kind, Owner, Backend string |
| 69 | Platform, Grant bool |
| 70 | PathTo map[string]string |
| 71 | Alias string |
| 72 | App, Source string |
| 73 | GrantAt int |
| 74 | } |
| 75 | |
| 76 | // mark is a row's exposure mark. |
| 77 | type mark int |
| 78 | |
| 79 | const ( |
| 80 | markNull mark = iota |
| 81 | markPublic |
| 82 | markInternal |
| 83 | ) |
| 84 | |
| 85 | func (m mark) String() string { |
| 86 | switch m { |
| 87 | case markPublic: |
| 88 | return "public" |
| 89 | case markInternal: |
| 90 | return "internal" |
| 91 | } |
| 92 | return "null" |
| 93 | } |
| 94 | |
| 95 | // caller names who a request is on behalf of. |
| 96 | type caller struct{ Account, App, Engine string } |
| 97 | |
| 98 | // req is a request the box admitted. Row is the row whose mark admitted it; |
| 99 | // Backend the proxy it holds from then on; Reached the process it was |
| 100 | // delivered to (0 for none). |
| 101 | type req struct { |
| 102 | ID, Epoch int |
| 103 | Row, Backend string |
| 104 | Public bool |
| 105 | Caller *caller |
| 106 | Held bool |
| 107 | Box, Reached int |
| 108 | } |
| 109 | |
| 110 | // box is one host of the fleet: box 0 is the engine's home, box 1 a calling |
| 111 | // box. |
| 112 | // |
| 113 | // Rows and Mark are the registry (the intent). EdgeRows and Serving are the |
| 114 | // edge's in-memory map, swapped in only by a successful reload. Wrote counts |
| 115 | // registry mark changes, Fenced is the change the edge has fenced, SetAt the |
| 116 | // change that last closed a group (its fence generation; a group with no key |
| 117 | // has never been fenced closed, spec §5.1, "Setting the mark"). |
| 118 | type box struct { |
| 119 | ID int |
| 120 | Rows map[string]*row |
| 121 | Mark map[string]mark |
| 122 | Wrote, Fenced, Epoch int |
| 123 | ReloadFails, Armed bool |
| 124 | Locked map[string]bool // the group's lock, held by --public for its whole run |
| 125 | Live, Good map[string]int |
| 126 | Staged map[string][]int |
| 127 | Procs []*proc |
| 128 | Admitted []*req |
| 129 | |
| 130 | EdgeRows map[string]row |
| 131 | Serving map[string]mark |
| 132 | SetAt map[string]int |
| 133 | |
| 134 | // PortGrant is the operator's ports grant by group (the port); Listen the |
| 135 | // ListenStream sockets the capability pass has rendered, port to group |
| 136 | // (spec §5.1, "The edge is the only ingress"). |
| 137 | PortGrant, Listen map[string]string |
| 138 | |
| 139 | // Task 5 (calls_test.go). Mirror is the edge's grant mirror (spec §5.3). |
| 140 | // Armed is `carlos engines arm`; Sockets is whether the edge's listener |
| 141 | // manager has the engine sockets open (§5.4, §6.2). Unisolated is every |
| 142 | // route on the box whose tenant code runs outside an instance slice, by |
| 143 | // kind (flat, foreign, worker). Tampered is a nested group's instance |
| 144 | // directory written by an unisolated tenant: the planter's app. |
| 145 | // |
| 146 | // ReadRev is the oracle's, not the protocol's: the highest revision the |
| 147 | // box's mirror has read since it was last rebuilt (a restart keeps it), so |
| 148 | // INV-G can hold a revoke the box has read in force (§6.4). |
| 149 | Mirror mirror |
| 150 | Sockets bool |
| 151 | Unisolated map[string]bool |
| 152 | Tampered map[string]string |
| 153 | ReadRev int |
| 154 | } |
| 155 | |
| 156 | type world struct { |
| 157 | seed int64 |
| 158 | rng *rand.Rand |
| 159 | boxes []*box |
| 160 | builds map[int]build |
| 161 | step int |
| 162 | trace []string |
| 163 | violations []string |
| 164 | residuals []string |
| 165 | stats map[string]int // actions selected, by name |
| 166 | done map[string]int // actions whose transition completed, by name |
| 167 | events map[string]int // deliveries: public requests and engine calls that reached a process |
| 168 | nextID int // process and request ids |
| 169 | |
| 170 | // clearHook, when set, runs inside operatorPublic at each stage |
| 171 | // ("locked": step 1 passed; "stopped": step 2 done; "confirmed": step 3 |
| 172 | // passed, before the write of 0), so a scenario or the walk can |
| 173 | // interleave there. |
| 174 | clearHook func(b *box, g, stage string) |
| 175 | // clearStopMisses, when set, makes operatorPublic's stop leave one |
| 176 | // process of the group behind (a unit or generation whose stop the |
| 177 | // activator or control service failed to complete). |
| 178 | clearStopMisses bool |
| 179 | |
| 180 | // Task 5 (calls_test.go): the console's engines record and the bucket |
| 181 | // that carries it; skew is time that passed without a step (a |
| 182 | // fast-forward), so now() is step+skew. |
| 183 | console engConsole |
| 184 | skew int |
| 185 | |
| 186 | // The spec's two stated residuals. Off by default and set only by tests: |
| 187 | // with one on, the walk may generate its sequence, and the breach it |
| 188 | // causes is recorded in residuals as residual:<name>, never in |
| 189 | // violations. (A fresh box's replay was a third until ruling R5 bounded |
| 190 | // it by the freshness bound; INV-G now covers it.) |
| 191 | unwrittenManifestAcrossUpgrade, flatEraTamper bool |
| 192 | |
| 193 | verbose bool |
| 194 | } |
| 195 | |
| 196 | // action is one thing the random walk can do. Tasks 2 to 5 append theirs |
| 197 | // from their own files' init; the walk picks among them by weight. |
| 198 | // |
| 199 | // run reports whether the transition completed: false when the action found |
| 200 | // nothing to act on (no group, no held request, a full table) and returned |
| 201 | // without calling into the model. A refusal by the protocol is a completed |
| 202 | // transition. The walk counts selections (stats) and completions (done) |
| 203 | // apart, so coverage can be checked on what really ran. |
| 204 | type action struct { |
| 205 | name string |
| 206 | weight int |
| 207 | run func(*world) bool |
| 208 | } |
| 209 | |
| 210 | var actions []action |
| 211 | |
| 212 | // stepHooks run at the start of every step, before its action: what happens |
| 213 | // on a timer whatever the walk chooses (Task 5: the mirror's poll). |
| 214 | var stepHooks []func(*world) |
| 215 | |
| 216 | func newWorld(seed int64, verbose bool) *world { |
| 217 | w := &world{ |
| 218 | seed: seed, |
| 219 | rng: rand.New(rand.NewSource(seed)), |
| 220 | builds: map[int]build{}, |
| 221 | stats: map[string]int{}, |
| 222 | done: map[string]int{}, |
| 223 | events: map[string]int{}, |
| 224 | verbose: verbose, |
| 225 | } |
| 226 | newBox := func(id int) *box { |
| 227 | return &box{ID: id, Rows: map[string]*row{}, Mark: map[string]mark{}, Live: map[string]int{}, Good: map[string]int{}, Staged: map[string][]int{}, |
| 228 | EdgeRows: map[string]row{}, Serving: map[string]mark{}, SetAt: map[string]int{}, |
| 229 | PortGrant: map[string]string{}, Listen: map[string]string{}, Locked: map[string]bool{}, |
| 230 | Unisolated: map[string]bool{}, Tampered: map[string]string{}} |
| 231 | } |
| 232 | home, calling := newBox(0), newBox(1) |
| 233 | w.boxes = []*box{home, calling} |
| 234 | |
| 235 | // one engine group, eng: its own route and its minted platform row |
| 236 | home.Rows[eng] = &row{Host: eng, Kind: "instance", App: "acct/eng", Backend: "/run/carlos/eng.sock", PathTo: map[string]string{}} |
| 237 | home.Rows[engHost] = &row{Host: engHost, Kind: "instance", App: "acct/eng", Owner: eng, Source: eng, Backend: "/run/carlos/eng.sock", Platform: true, PathTo: map[string]string{}} |
| 238 | home.Mark[eng], home.Mark[engHost] = markInternal, markInternal |
| 239 | home.SetAt[eng] = 0 // closed from the start, and loaded below |
| 240 | // one public app group, shop, on the calling box |
| 241 | calling.Rows["shop"] = &row{Host: "shop", Kind: "instance", App: "acct/shop", Backend: "127.0.0.1:9001", PathTo: map[string]string{}} |
| 242 | calling.Mark["shop"] = markPublic |
| 243 | // on each box an alias row and a static row with a path map, both |
| 244 | // resolving to the box's group: rows of kinds that serve no release, 0 |
| 245 | // by construction (spec §5.1, "What the edge does with it") |
| 246 | for _, x := range []struct { |
| 247 | b *box |
| 248 | alias, site, path, to string |
| 249 | }{{home, engAlias, engSite, engSitePath, engHost}, {calling, shopAlias, shopSite, shopSitePath, "shop"}} { |
| 250 | x.b.Rows[x.alias] = &row{Host: x.alias, Kind: "alias", Alias: x.to, PathTo: map[string]string{}} |
| 251 | x.b.Rows[x.site] = &row{Host: x.site, Kind: "static", PathTo: map[string]string{x.path: x.to}} |
| 252 | x.b.Mark[x.alias], x.b.Mark[x.site] = markPublic, markPublic |
| 253 | } |
| 254 | // the edges start with the registry loaded |
| 255 | for _, b := range w.boxes { |
| 256 | b.loadEdge() |
| 257 | } |
| 258 | w.initEngines() // the console's first record, read by every edge |
| 259 | |
| 260 | for v := 1; v <= nBuilds; v++ { |
| 261 | w.builds[v] = build{Ver: v, Declares: w.rng.Intn(2) == 0, ManifestOK: w.rng.Intn(2) == 0} |
| 262 | } |
| 263 | w.logf("world seed %d: builds %s", seed, w.buildsString()) |
| 264 | return w |
| 265 | } |
| 266 | |
| 267 | func (w *world) buildsString() string { |
| 268 | var out []string |
| 269 | for v := 1; v <= nBuilds; v++ { |
| 270 | b := w.builds[v] |
| 271 | out = append(out, fmt.Sprintf("%d{declares=%v manifest=%v}", v, b.Declares, b.ManifestOK)) |
| 272 | } |
| 273 | return fmt.Sprint(out) |
| 274 | } |
| 275 | |
| 276 | // logf appends a trace line. The trace is always kept: it is what the |
| 277 | // determinism test compares, and what ENGINESIM_SEED prints. |
| 278 | func (w *world) logf(format string, args ...any) { |
| 279 | w.trace = append(w.trace, fmt.Sprintf("[%d] ", w.step)+fmt.Sprintf(format, args...)) |
| 280 | } |
| 281 | |
| 282 | // calm is true once the chaos steps are over. doStep counts the step before |
| 283 | // it runs the action, so steps 1 to chaosSteps are chaos and the rest calm. |
| 284 | func (w *world) calm() bool { return w.step > chaosSteps } |
| 285 | |
| 286 | // group is a row's group key: its owner for a minted instance row whose |
| 287 | // owner the registry can find, else its own host (spec §5.1, |
| 288 | // "Platform-minted rows"; resolveOwner). |
| 289 | func (w *world) group(b *box, host string) string { |
| 290 | if r := b.Rows[host]; r != nil && followsOwner(r) { |
| 291 | if o, ok := b.resolveOwner(r); ok { |
| 292 | return o |
| 293 | } |
| 294 | } |
| 295 | return host |
| 296 | } |
| 297 | |
| 298 | // pick chooses one registered action by weight. The registry is sorted by |
| 299 | // name first, so the order files' init functions ran in never matters. |
| 300 | func (w *world) pick() *action { |
| 301 | sorted := make([]action, 0, len(actions)) |
| 302 | total := 0 |
| 303 | for _, a := range actions { |
| 304 | if a.weight > 0 { |
| 305 | sorted = append(sorted, a) |
| 306 | total += a.weight |
| 307 | } |
| 308 | } |
| 309 | if total == 0 { |
| 310 | return nil |
| 311 | } |
| 312 | sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].name < sorted[j].name }) |
| 313 | n := w.rng.Intn(total) |
| 314 | for i := range sorted { |
| 315 | if n < sorted[i].weight { |
| 316 | return &sorted[i] |
| 317 | } |
| 318 | n -= sorted[i].weight |
| 319 | } |
| 320 | return nil |
| 321 | } |
| 322 | |
| 323 | // doStep advances one tick: it counts the step, runs one chosen action and |
| 324 | // counts it under its name, in stats when chosen and in done when its |
| 325 | // transition completed. |
| 326 | func (w *world) doStep() { |
| 327 | w.step++ |
| 328 | for _, h := range stepHooks { |
| 329 | h(w) |
| 330 | } |
| 331 | a := w.pick() |
| 332 | if a == nil { |
| 333 | if w.verbose { |
| 334 | w.logf("idle") |
| 335 | } |
| 336 | return |
| 337 | } |
| 338 | w.stats[a.name]++ |
| 339 | w.logf("action %s", a.name) // always traced: the determinism test compares the action sequence |
| 340 | if a.run(w) { |
| 341 | w.done[a.name]++ |
| 342 | } |
| 343 | w.check() |
| 344 | } |
| 345 | |
| 346 | // final is the end-of-run oracle: what must hold once the calm steps are |
| 347 | // over. Later tasks add their end-state checks here. |
| 348 | // |
| 349 | // The faults are over: every edge's reload works again, and each edge polls |
| 350 | // once, so every row has had its chance to answer. Then INV-AV. |
| 351 | func (w *world) final() { |
| 352 | for _, b := range w.boxes { |
| 353 | if b.ReloadFails { |
| 354 | b.ReloadFails = false |
| 355 | w.logf("box%d: final: the reload fault has cleared", b.ID) |
| 356 | } |
| 357 | w.reload(b) |
| 358 | } |
| 359 | w.checkAV() |
| 360 | w.logf("final: %d steps, %d violations, %d residuals", w.step, len(w.violations), len(w.residuals)) |
| 361 | } |
| 362 | |
| 363 | func runSeed(seed int64, verbose bool) *world { |
| 364 | w := newWorld(seed, verbose) |
| 365 | for i := 0; i < chaosSteps+calmSteps; i++ { |
| 366 | w.doStep() |
| 367 | } |
| 368 | w.final() |
| 369 | return w |
| 370 | } |
| 371 | |
amadan