| 1 | package engines |
| 2 | |
| 3 | import ( |
| 4 | "fmt" |
| 5 | "strings" |
| 6 | ) |
| 7 | |
| 8 | // Calls, identity, grants and arming (spec §5.2 the signed record; §5.3 the |
| 9 | // grant mirror; §5.4 the call path; §6.1 identity; §6.2 armed boxes; §6.4 |
| 10 | // freshness). |
| 11 | // |
| 12 | // The console publishes the grant set as one signed record with a strictly |
| 13 | // increasing revision, its signing time and a 72-hour expiry, and re-signs it |
| 14 | // every two minutes and on every change; every edge mirrors it, keeps a |
| 15 | // durable revision floor, and treats a stale, expired or never-read table as |
| 16 | // empty. Stale is either no good read for ten minutes or a record signed more |
| 17 | // than ten minutes ago (ruling R5). A nested instance calls an engine through its engine socket, which |
| 18 | // is open only on an armed box; a host-app process calls through the door. |
| 19 | // The calling box proves the caller (the peer's cgroup, or a door key from a |
| 20 | // uid other than carlos), checks its own table and hops to the engine's home |
| 21 | // box with the caller signed into the hop's MAC. The engine box checks its |
| 22 | // own table again and dispatches to the engine hostname's own route only. |
| 23 | // |
| 24 | // The model leaves out the per-app cap, the transport limits and the deploy |
| 25 | // watch ticket (Plan 2 tests those directly). |
| 26 | |
| 27 | // Time: one tick is 10 s. |
| 28 | const ( |
| 29 | pollTicks = 3 // the mirror polls every 30 s |
| 30 | staleTicks = 60 // no good read, or a record signed, ten minutes ago: stale (R5) |
| 31 | resignTicks = 12 // the console re-signs every two minutes while it is up (R5) |
| 32 | lifetimeTicks = 25920 // a record expires 72 hours after it is signed |
| 33 | ) |
| 34 | |
| 35 | // now is the model's clock: one tick per step, plus any fast-forward. |
| 36 | func (w *world) now() int { return w.step + w.skew } |
| 37 | |
| 38 | // advance is time passing with no step: n ticks in which nothing polls (an |
| 39 | // outage, a console down for days). |
| 40 | func (w *world) advance(n int) { |
| 41 | w.skew += n |
| 42 | w.logf("%d ticks pass (now %d)", n, w.now()) |
| 43 | } |
| 44 | |
| 45 | // ---------- the console's record (spec §5.2) ---------- |
| 46 | |
| 47 | // record is one signed control/engines.json: its revision, its signed_at and |
| 48 | // expiry (in ticks), and its grants by account/app/engine. |
| 49 | type record struct { |
| 50 | Rev, SignedAt, Expires int |
| 51 | Grants map[string]bool |
| 52 | } |
| 53 | |
| 54 | // mirror is an edge's grant mirror (§5.3): the record it holds, its revision |
| 55 | // floor and the tick of its last good read, and what it has persisted to its |
| 56 | // data directory (the record and the floor, written before either is used). |
| 57 | type mirror struct { |
| 58 | Rec *record |
| 59 | Floor, ReadAt int |
| 60 | Persisted *record |
| 61 | PersistedFloor int |
| 62 | } |
| 63 | |
| 64 | // engConsole is the console's side: its database (Grants, the authority), |
| 65 | // every record it has signed, the newest revision, and the object in the |
| 66 | // bucket. RevokedAt and RevokedRev are the oracle's: the tick and revision of |
| 67 | // the publish that stopped granting a key. BucketDown: the edges cannot read |
| 68 | // the bucket. Down: the console is down, so it neither re-signs nor takes a |
| 69 | // grant or revoke. |
| 70 | // |
| 71 | // Pinned is a bucket writer's replay: an older signed record it keeps in the |
| 72 | // bucket, overwriting each record the console publishes, until tick |
| 73 | // PinnedUntil. |
| 74 | type engConsole struct { |
| 75 | Rev int |
| 76 | Records map[int]*record |
| 77 | Grants map[string]bool |
| 78 | RevokedAt map[string]int |
| 79 | RevokedRev map[string]int |
| 80 | Bucket *record |
| 81 | BucketDown bool |
| 82 | Down bool |
| 83 | Pinned *record |
| 84 | PinnedUntil int |
| 85 | } |
| 86 | |
| 87 | // The engines the operator has named (carlos engines add): name to its |
| 88 | // hostname. eng's hostname is the engine group's minted row; site's is box |
| 89 | // 0's static row, whose path map sends /convert to eng's route (R4-2: the |
| 90 | // console cannot check a hostname, and path targets may cross apps). |
| 91 | const site = "site" |
| 92 | |
| 93 | var engineHosts = map[string]string{eng: engHost, site: engSite} |
| 94 | |
| 95 | // callerOf is the caller account/app calling engine. |
| 96 | func callerOf(app, engine string) caller { |
| 97 | acct, a, _ := strings.Cut(app, "/") |
| 98 | return caller{Account: acct, App: a, Engine: engine} |
| 99 | } |
| 100 | |
| 101 | func (c caller) app() string { return c.Account + "/" + c.App } |
| 102 | func (c caller) key() string { return c.app() + "/" + c.Engine } |
| 103 | |
| 104 | // initEngines is the starting world: the console's first record grants shop |
| 105 | // the engine eng, and every edge has read it. |
| 106 | func (w *world) initEngines() { |
| 107 | w.console = engConsole{Records: map[int]*record{}, Grants: map[string]bool{"acct/shop/eng": true}, RevokedAt: map[string]int{}, RevokedRev: map[string]int{}} |
| 108 | w.publish(1) |
| 109 | for _, b := range w.boxes { |
| 110 | w.mirrorTick(b) |
| 111 | } |
| 112 | } |
| 113 | |
| 114 | // publish is the console signing its grant set as revision rev, signed now |
| 115 | // and expiring 72 hours out, and writing it to the bucket (where a pinned |
| 116 | // replay overwrites it at once). A revision is strictly increasing; a re-sign |
| 117 | // with the same grants is a new revision. |
| 118 | func (w *world) publish(rev int) { |
| 119 | c := &w.console |
| 120 | if rev <= c.Rev { |
| 121 | panic(fmt.Sprintf("publish: revision %d after %d", rev, c.Rev)) |
| 122 | } |
| 123 | rec := &record{Rev: rev, SignedAt: w.now(), Expires: w.now() + lifetimeTicks, Grants: map[string]bool{}} |
| 124 | for k, v := range c.Grants { |
| 125 | if v { |
| 126 | rec.Grants[k] = true |
| 127 | } |
| 128 | } |
| 129 | if prev := c.Records[c.Rev]; prev != nil { |
| 130 | for _, k := range sortedKeys(prev.Grants) { |
| 131 | if !rec.Grants[k] { |
| 132 | c.RevokedAt[k], c.RevokedRev[k] = w.now(), rev |
| 133 | } |
| 134 | } |
| 135 | } |
| 136 | for k := range rec.Grants { |
| 137 | delete(c.RevokedAt, k) |
| 138 | delete(c.RevokedRev, k) |
| 139 | } |
| 140 | c.Records[rev], c.Rev, c.Bucket = rec, rev, rec |
| 141 | w.logf("console: publishes revision %d, signed at %d, expiring at %d, grants %v", rev, rec.SignedAt, rec.Expires, sortedKeys(rec.Grants)) |
| 142 | if c.Pinned != nil { |
| 143 | c.Bucket = c.Pinned |
| 144 | w.logf("bucket writer: overwrites it with revision %d", c.Pinned.Rev) |
| 145 | } |
| 146 | } |
| 147 | |
| 148 | // consoleGrant and consoleRevoke are carlos engines grant/revoke: the |
| 149 | // database changes and the console publishes the next revision. A console |
| 150 | // that is down takes neither (false). |
| 151 | func (w *world) consoleGrant(key string) bool { |
| 152 | if w.console.Down { |
| 153 | w.logf("console: down; carlos engines grant %s fails", key) |
| 154 | return false |
| 155 | } |
| 156 | w.console.Grants[key] = true |
| 157 | w.publish(w.console.Rev + 1) |
| 158 | return true |
| 159 | } |
| 160 | |
| 161 | func (w *world) consoleRevoke(key string) bool { |
| 162 | if w.console.Down { |
| 163 | w.logf("console: down; carlos engines revoke %s fails", key) |
| 164 | return false |
| 165 | } |
| 166 | delete(w.console.Grants, key) |
| 167 | w.publish(w.console.Rev + 1) |
| 168 | return true |
| 169 | } |
| 170 | |
| 171 | // replay is a bucket writer restoring an older signed record (validly signed, |
| 172 | // and unexpired unless 72 hours have passed) and keeping it in the bucket |
| 173 | // over everything the console publishes, until it stops (unpin). |
| 174 | func (w *world) replay(rev int) { w.replayUntil(rev, 1<<62) } |
| 175 | |
| 176 | // replayUntil is replay for a writer who stops at tick until. |
| 177 | func (w *world) replayUntil(rev, until int) { |
| 178 | rec := w.console.Records[rev] |
| 179 | if rec == nil { |
| 180 | panic(fmt.Sprintf("replay: no revision %d", rev)) |
| 181 | } |
| 182 | w.console.Bucket, w.console.Pinned, w.console.PinnedUntil = rec, rec, until |
| 183 | w.logf("bucket writer: replays revision %d (signed at %d; newest %d) into the bucket, until tick %d", rev, rec.SignedAt, w.console.Rev, until) |
| 184 | } |
| 185 | |
| 186 | // unpin is the bucket writer stopping: the newest record is in the bucket |
| 187 | // again. |
| 188 | func (w *world) unpin() { |
| 189 | c := &w.console |
| 190 | c.Pinned, c.Bucket = nil, c.Records[c.Rev] |
| 191 | w.logf("bucket writer: stops; the bucket holds revision %d again", c.Rev) |
| 192 | } |
| 193 | |
| 194 | func (w *world) setBucketDown(down bool) { |
| 195 | w.console.BucketDown = down |
| 196 | w.logf("the bucket is unreachable = %v", down) |
| 197 | } |
| 198 | |
| 199 | // ---------- the grant mirror (spec §5.3) ---------- |
| 200 | |
| 201 | // mirrorTick is one poll of b's mirror. It verifies the record's expiry and |
| 202 | // its signed_at (a record signed more than ten minutes ago is stale, like an |
| 203 | // expired one, grant-freshness-bound) and refuses a revision below its floor |
| 204 | // (an equal revision is the same signed bytes: the model's records are |
| 205 | // immutable per revision). A good read replaces the record, raises the floor, |
| 206 | // and persists both before either is used (floor-persisted). |
| 207 | func (w *world) mirrorTick(b *box) { |
| 208 | m := &b.Mirror |
| 209 | c := &w.console |
| 210 | if c.BucketDown || c.Bucket == nil { |
| 211 | return |
| 212 | } |
| 213 | rec, now := c.Bucket, w.now() |
| 214 | if now >= rec.Expires { |
| 215 | if m.Rec != rec { |
| 216 | w.logf("box%d: mirror refuses revision %d: expired at %d", b.ID, rec.Rev, rec.Expires) |
| 217 | } |
| 218 | return |
| 219 | } |
| 220 | if now-rec.SignedAt >= staleTicks && on("grant-freshness-bound") { |
| 221 | if m.Rec != rec { |
| 222 | w.logf("box%d: mirror refuses revision %d: signed at %d, ten minutes ago or more", b.ID, rec.Rev, rec.SignedAt) |
| 223 | } |
| 224 | return |
| 225 | } |
| 226 | if rec.Rev < m.Floor { |
| 227 | w.logf("box%d: mirror refuses revision %d: below the floor %d", b.ID, rec.Rev, m.Floor) |
| 228 | return |
| 229 | } |
| 230 | if rec.Rev > m.Floor || m.Rec == nil { |
| 231 | w.logf("box%d: mirror reads revision %d (floor %d -> %d)", b.ID, rec.Rev, m.Floor, rec.Rev) |
| 232 | } |
| 233 | m.Rec, m.ReadAt = rec, now |
| 234 | if rec.Rev > m.Floor { |
| 235 | m.Floor = rec.Rev |
| 236 | } |
| 237 | if rec.Rev > b.ReadRev { |
| 238 | b.ReadRev = rec.Rev |
| 239 | } |
| 240 | if on("floor-persisted") { |
| 241 | m.Persisted, m.PersistedFloor = m.Rec, m.Floor |
| 242 | } |
| 243 | } |
| 244 | |
| 245 | // restartEdge is b's edge restarting: it loads the persisted record and |
| 246 | // floor before any engine socket opens. The time of the last good read is |
| 247 | // persisted with the record, so a restart is not a good read. |
| 248 | func (w *world) restartEdge(b *box) { |
| 249 | m := &b.Mirror |
| 250 | m.Rec, m.Floor = m.Persisted, m.PersistedFloor |
| 251 | rev := 0 |
| 252 | if m.Rec != nil { |
| 253 | rev = m.Rec.Rev |
| 254 | } |
| 255 | w.logf("box%d: edge restarts: mirror loads revision %d, floor %d", b.ID, rev, m.Floor) |
| 256 | } |
| 257 | |
| 258 | // rebuild is a new or rebuilt box: its edge's data directory is gone, so it |
| 259 | // has no floor and no record until its first read, and may admit any validly |
| 260 | // signed record a bucket writer replays, until ten minutes after its signing |
| 261 | // (§5.3). The oracle forgets what the box had read (ReadRev). |
| 262 | func (w *world) rebuild(b *box) { |
| 263 | b.Mirror = mirror{} |
| 264 | b.ReadRev = 0 |
| 265 | w.logf("box%d: rebuilt: the edge has no persisted floor", b.ID) |
| 266 | } |
| 267 | |
| 268 | // tableGrants is b's grant table on key. Never read: empty. No good read for |
| 269 | // ten minutes, or expired: empty (stale-clears-grants). A record signed more |
| 270 | // than ten minutes ago: empty (grant-freshness-bound, R5). |
| 271 | func (w *world) tableGrants(b *box, key string) bool { |
| 272 | m := b.Mirror |
| 273 | if m.Rec == nil { |
| 274 | return false |
| 275 | } |
| 276 | if on("stale-clears-grants") && (w.now()-m.ReadAt >= staleTicks || w.now() >= m.Rec.Expires) { |
| 277 | return false |
| 278 | } |
| 279 | if on("grant-freshness-bound") && w.now()-m.Rec.SignedAt >= staleTicks { |
| 280 | return false |
| 281 | } |
| 282 | return m.Rec.Grants[key] |
| 283 | } |
| 284 | |
| 285 | // ---------- the call path (spec §5.4) ---------- |
| 286 | |
| 287 | // call is one engine call as its sender makes it. host is the instance |
| 288 | // whose engine socket the process connects to, or whose door key it |
| 289 | // presents; engine is the Host: header; forged an X-Carlos-Hop-Caller header |
| 290 | // the sender wrote, or nil. |
| 291 | type call struct { |
| 292 | from *box |
| 293 | p *proc |
| 294 | viaDoor bool |
| 295 | uid string |
| 296 | host string |
| 297 | engine string |
| 298 | path string |
| 299 | forged *caller |
| 300 | } |
| 301 | |
| 302 | // engineCall is process p on box from calling engine (its Host:) at path, |
| 303 | // through the engine socket of instance host, or through the door with |
| 304 | // host's door key under uid. It reports whether the call reached an engine |
| 305 | // process. |
| 306 | func (w *world) engineCall(from *box, p *proc, viaDoor bool, uid string, host, engine, path string) bool { |
| 307 | return w.placeCall(call{from: from, p: p, viaDoor: viaDoor, uid: uid, host: host, engine: engine, path: path}) |
| 308 | } |
| 309 | |
| 310 | // unitBacked: the route runs as a unit-backed DynamicUser service. The |
| 311 | // model's engine route is (§8: arm succeeds with nested callers and a |
| 312 | // unit-backed engine); every other instance route is nested. Unit-backed |
| 313 | // routes are not callers (ruling R2): they have no engine socket. |
| 314 | func unitBacked(g string) bool { return g == eng } |
| 315 | |
| 316 | // socketOpen: instance g on b has an engine socket: the box is armed and |
| 317 | // its listener manager has the sockets open, and g is a nested instance. |
| 318 | func (w *world) socketOpen(b *box, g string) bool { |
| 319 | if !b.Armed || !b.Sockets || unitBacked(g) { |
| 320 | return false |
| 321 | } |
| 322 | for _, x := range w.groups(b) { |
| 323 | if x == g { |
| 324 | return true |
| 325 | } |
| 326 | } |
| 327 | return false |
| 328 | } |
| 329 | |
| 330 | // realApp is the account/app whose code process p runs: planted code's app, |
| 331 | // else its group's route's, else (a tenant with no row in the model: a flat, |
| 332 | // foreign or worker tenant) acct/<group). nil is a public client: no app. |
| 333 | func (w *world) realApp(b *box, p *proc) string { |
| 334 | switch { |
| 335 | case p == nil: |
| 336 | return "" |
| 337 | case p.Code != "": |
| 338 | return p.Code |
| 339 | } |
| 340 | if r := b.Rows[p.Group]; r != nil && r.App != "" { |
| 341 | return r.App |
| 342 | } |
| 343 | return "acct/" + p.Group |
| 344 | } |
| 345 | |
| 346 | func (w *world) placeCall(c call) bool { |
| 347 | b := c.from |
| 348 | how := "socket of " + c.host |
| 349 | if c.viaDoor { |
| 350 | how = "door (uid " + c.uid + ", key of " + c.host + ")" |
| 351 | } |
| 352 | who := "a public client" |
| 353 | if c.p != nil { |
| 354 | who = fmt.Sprintf("process %d (%s)", c.p.ID, c.p.Slice) |
| 355 | } |
| 356 | w.logf("box%d: %s calls Host: %s%s through the %s", b.ID, who, c.engine, c.path, how) |
| 357 | if strings.Contains(c.engine, ".") { |
| 358 | w.logf("box%d: Host: %s is no engine name (it has a dot): not an engine call", b.ID, c.engine) |
| 359 | return false |
| 360 | } |
| 361 | r := b.Rows[c.host] |
| 362 | if r == nil || r.Kind != "instance" || r.Platform { |
| 363 | w.logf("box%d: no instance route %s", b.ID, c.host) |
| 364 | return false |
| 365 | } |
| 366 | // who is calling |
| 367 | if !c.viaDoor { |
| 368 | if !w.socketOpen(b, c.host) { |
| 369 | w.logf("box%d: %s has no engine socket (armed %v)", b.ID, c.host, b.Armed) |
| 370 | return false |
| 371 | } |
| 372 | if on("cgroup-identity") && c.p.Slice != "inst:"+c.host { |
| 373 | w.logf("box%d: engine socket of %s: peer cgroup %s is not the instance's slice; closed unread", b.ID, c.host, c.p.Slice) |
| 374 | return false |
| 375 | } |
| 376 | } else if on("door-uid") && c.uid == "carlos" { |
| 377 | w.logf("box%d: door: an engine call from uid carlos: 404 engine not granted", b.ID) |
| 378 | return false |
| 379 | } |
| 380 | mark := callerOf(r.App, c.engine) |
| 381 | // the listener strips every X-Carlos-* header the sender wrote |
| 382 | if c.forged != nil && on("hop-caller-signed") { |
| 383 | w.logf("box%d: X-Carlos-Hop-Caller %s from the sender deleted", b.ID, c.forged.key()) |
| 384 | c.forged = nil |
| 385 | } |
| 386 | if !w.tableGrants(b, mark.key()) { |
| 387 | w.logf("box%d: %s not in the grant table: 404 engine not granted", b.ID, mark.key()) |
| 388 | return false |
| 389 | } |
| 390 | hostname := engineHosts[c.engine] |
| 391 | home := w.homeOf(hostname) |
| 392 | if home == nil { |
| 393 | w.logf("box%d: %s homed nowhere: 502 engine unreachable", b.ID, hostname) |
| 394 | return false |
| 395 | } |
| 396 | seen := mark |
| 397 | if home != b { |
| 398 | w.logf("box%d: hop to box%d, caller %s signed into the MAC", b.ID, home.ID, mark.key()) |
| 399 | if c.forged != nil { |
| 400 | w.logf("box%d: the hop also carries the sender's X-Carlos-Hop-Caller %s; the engine box takes it", home.ID, c.forged.key()) |
| 401 | seen = *c.forged |
| 402 | } |
| 403 | } |
| 404 | return w.engineAdmit(home, b, seen, hostname, c.path, c.host, w.realApp(b, c.p), c.p) |
| 405 | } |
| 406 | |
| 407 | // clientHop is a public client sending b's edge a request for hostname that |
| 408 | // carries X-Carlos-Hop-Caller with no hop MAC. Only a verified hop's caller |
| 409 | // is a caller (hop-caller-signed): the header is deleted and the request is |
| 410 | // a public request like any other. |
| 411 | func (w *world) clientHop(b *box, hostname, path string, forged caller) bool { |
| 412 | w.logf("box%d: a public client sends %s%s with X-Carlos-Hop-Caller %s", b.ID, hostname, path, forged.key()) |
| 413 | if on("hop-caller-signed") { |
| 414 | w.logf("box%d: unverified X-Carlos-Hop-Caller deleted; a public request", b.ID) |
| 415 | if r := w.admitPublic(b, hostname, path); r != nil { |
| 416 | w.deliver(r) |
| 417 | } |
| 418 | return false |
| 419 | } |
| 420 | return w.engineAdmit(b, b, forged, hostname, path, "", "", nil) |
| 421 | } |
| 422 | |
| 423 | // homeOf is the box whose registry has hostname (the edge's home map). |
| 424 | func (w *world) homeOf(hostname string) *box { |
| 425 | for _, b := range w.boxes { |
| 426 | if b.Rows[hostname] != nil { |
| 427 | return b |
| 428 | } |
| 429 | } |
| 430 | return nil |
| 431 | } |
| 432 | |
| 433 | // exactRoute is the row an engine call to hostname resolves to on b: the |
| 434 | // hostname's own row, through an alias (the same route), never a path map |
| 435 | // (exact-route-engine). ok is false for an unknown host. |
| 436 | func (w *world) exactRoute(b *box, hostname string) (row, bool) { |
| 437 | final, ok := b.EdgeRows[hostname] |
| 438 | if ok && final.Alias != "" { |
| 439 | final, ok = b.EdgeRows[final.Alias] |
| 440 | } |
| 441 | return final, ok |
| 442 | } |
| 443 | |
| 444 | // engineAdmit is the engine box (home) deciding an engine-marked request for |
| 445 | // hostname: the route must be internal, the engine box's own table must name |
| 446 | // the hostname for the mark's engine and grant it to the mark's app |
| 447 | // (engine-box-recheck), and the route must be the hostname's own |
| 448 | // (exact-route-engine). Then the activator's dispatch, which may wake the |
| 449 | // engine. via is the instance whose engine socket or door key the call came |
| 450 | // through ("" for a client's hop); real is the calling process's real app; |
| 451 | // p the process. |
| 452 | func (w *world) engineAdmit(home, from *box, mark caller, hostname, path, via, real string, p *proc) bool { |
| 453 | final, ok := w.exactRoute(home, hostname) |
| 454 | if !ok { |
| 455 | w.logf("box%d: engine call to %s: unknown host", home.ID, hostname) |
| 456 | return false |
| 457 | } |
| 458 | if !on("exact-route-engine") { |
| 459 | if to, ok := final.PathTo[path]; ok { |
| 460 | if t, ok := home.EdgeRows[to]; ok { |
| 461 | w.logf("box%d: engine call to %s%s follows the path map to %s", home.ID, hostname, path, to) |
| 462 | final = t |
| 463 | } |
| 464 | } |
| 465 | } |
| 466 | if home.Serving[final.Host] != markInternal { |
| 467 | w.logf("box%d: engine call to %s: %s is not internal (%v): the internal refusal", home.ID, hostname, final.Host, home.Serving[final.Host]) |
| 468 | return false |
| 469 | } |
| 470 | if on("engine-box-recheck") { |
| 471 | if engineHosts[mark.Engine] != hostname || !w.tableGrants(home, mark.key()) { |
| 472 | w.logf("box%d: engine box: %s not granted for %s: the internal refusal", home.ID, mark.key(), hostname) |
| 473 | return false |
| 474 | } |
| 475 | } |
| 476 | g := truthGroup(home, final.Host) |
| 477 | q := w.newest(home, g) |
| 478 | if q == nil { |
| 479 | w.wake(home, g, "recovery", nil) |
| 480 | q = w.newest(home, g) |
| 481 | } |
| 482 | if q == nil { |
| 483 | w.logf("box%d: engine call: %s has no process; the wake failed", home.ID, g) |
| 484 | return false |
| 485 | } |
| 486 | w.logf("box%d: engine call reaches process %d (%s v%d); the access log records caller %s", home.ID, q.ID, q.Group, q.Ver, mark.app()) |
| 487 | w.events["engine-call-reached"]++ |
| 488 | w.checkCall(home, from, mark, via, real, p, q) |
| 489 | return true |
| 490 | } |
| 491 | |
| 492 | // ---------- armed boxes (spec §6.2) ---------- |
| 493 | |
| 494 | // unisolatedKinds are the launch paths that run tenant code outside an |
| 495 | // instance slice: a flat instance, a foreign one, and an app's worker. |
| 496 | var unisolatedKinds = []string{"flat", "foreign", "worker"} |
| 497 | |
| 498 | // arm is `carlos engines arm`: refused unless every exec-backed instance |
| 499 | // route is nested (unit-backed routes are judged by backing and admitted), |
| 500 | // no route spawns workers, and every carlos-exec@ cgroup is empty |
| 501 | // (arm-checks-box). Then the box records the armed state and the edge opens |
| 502 | // the engine sockets. |
| 503 | func (w *world) arm(b *box) bool { |
| 504 | if b.Armed { |
| 505 | return true |
| 506 | } |
| 507 | if on("arm-checks-box") { |
| 508 | var why []string |
| 509 | for _, k := range sortedKeys(b.Unisolated) { |
| 510 | why = append(why, "a "+k+" route") |
| 511 | } |
| 512 | for _, p := range b.Procs { |
| 513 | if p.Unisolated { |
| 514 | why = append(why, fmt.Sprintf("process %d in %s", p.ID, p.Slice)) |
| 515 | } |
| 516 | } |
| 517 | if len(why) > 0 { |
| 518 | w.logf("box%d: carlos engines arm: refused: %s", b.ID, strings.Join(why, ", ")) |
| 519 | return false |
| 520 | } |
| 521 | } |
| 522 | b.Armed = true |
| 523 | b.Sockets = true |
| 524 | w.logf("box%d: armed; engine sockets open (arming vouches for the present, not the past)", b.ID) |
| 525 | return true |
| 526 | } |
| 527 | |
| 528 | // disarm is `carlos engines disarm`: it closes every engine socket first, |
| 529 | // then clears the state; launches follow. |
| 530 | func (w *world) disarm(b *box) { |
| 531 | if !b.Armed { |
| 532 | return |
| 533 | } |
| 534 | b.Sockets = false |
| 535 | b.Armed = false |
| 536 | w.logf("box%d: disarmed: engine sockets closed, then the state cleared", b.ID) |
| 537 | } |
| 538 | |
| 539 | // launchUnisolated starts tenant code outside an instance slice: kind flat |
| 540 | // or foreign (an instance of that layout, through the exec-child path) or |
| 541 | // worker (an app's worker in carlos-exec@). While armed it is refused before |
| 542 | // any process starts (armed-refuses-unisolated). The model's unisolated |
| 543 | // tenants are one route per kind, acct/<kind>. |
| 544 | func (w *world) launchUnisolated(b *box, kind string) bool { |
| 545 | if b.Armed && on("armed-refuses-unisolated") { |
| 546 | w.logf("box%d: %s launch refused: the box is armed (carlos engines disarm to run it)", b.ID, kind) |
| 547 | return false |
| 548 | } |
| 549 | b.Unisolated[kind] = true |
| 550 | w.nextID++ |
| 551 | p := &proc{ID: w.nextID, Group: kind, Slice: "exec:" + kind, Unisolated: true} |
| 552 | b.Procs = append(b.Procs, p) |
| 553 | w.logf("box%d: %s tenant started as process %d in %s", b.ID, kind, p.ID, p.Slice) |
| 554 | return true |
| 555 | } |
| 556 | |
| 557 | // moveToNested is the operator moving the kind route off the box's |
| 558 | // unisolated layouts (or its workers into instance slices): its processes |
| 559 | // stop and the route no longer blocks arming. |
| 560 | func (w *world) moveToNested(b *box, kind string) { |
| 561 | w.stopGroup(b, kind) |
| 562 | delete(b.Unisolated, kind) |
| 563 | w.logf("box%d: the %s route moved to nested; its processes stopped", b.ID, kind) |
| 564 | } |
| 565 | |
| 566 | // tamper is unisolated process by writing nested group g's instance |
| 567 | // directory (an environment file and an LD_PRELOAD): g's next start runs by's |
| 568 | // code inside g's slice. Only an unisolated process can, which an armed box |
| 569 | // never runs; the walk generates it only under flatEraTamper. |
| 570 | func (w *world) tamper(b *box, by *proc, g string) { |
| 571 | b.Tampered[g] = w.realApp(b, by) |
| 572 | w.logf("box%d: process %d (%s) plants code in %s's instance directory", b.ID, by.ID, by.Slice, g) |
| 573 | } |
| 574 | |
| 575 | // applyTamper is a nested start picking up planted code (launch calls it). |
| 576 | func (w *world) applyTamper(b *box, p *proc) { |
| 577 | if app := b.Tampered[p.Group]; app != "" { |
| 578 | p.Code = app |
| 579 | w.logf("box%d: process %d of %s runs code %s planted", b.ID, p.ID, p.Group, app) |
| 580 | } |
| 581 | } |
| 582 | |
| 583 | // ---------- the random walk ---------- |
| 584 | |
| 585 | // walkApps are the apps the console may grant: the model's routes' apps and |
| 586 | // the unisolated tenants'. |
| 587 | var walkApps = []string{"acct/eng", "acct/flat", "acct/shop", "acct/tcp"} |
| 588 | |
| 589 | func (w *world) pickEngine() string { return []string{eng, site}[w.rng.Intn(2)] } |
| 590 | func (w *world) pickPath() string { return []string{"/", engSitePath}[w.rng.Intn(2)] } |
| 591 | func (w *world) pickKey() string { |
| 592 | return walkApps[w.rng.Intn(len(walkApps))] + "/" + w.pickEngine() |
| 593 | } |
| 594 | |
| 595 | // maybeForged is, one call in four, an X-Carlos-Hop-Caller the sender wrote. |
| 596 | func (w *world) maybeForged() *caller { |
| 597 | if w.rng.Intn(4) != 0 { |
| 598 | return nil |
| 599 | } |
| 600 | c := callerOf(walkApps[w.rng.Intn(len(walkApps))], w.pickEngine()) |
| 601 | return &c |
| 602 | } |
| 603 | |
| 604 | // pickReplay is the revision a bucket writer replays: one that grants a key |
| 605 | // the console has since revoked, if any (the damaging choice: half the time |
| 606 | // the newest such, signed most recently, else any), else any older one. |
| 607 | func (w *world) pickReplay() int { |
| 608 | newest := w.console.Records[w.console.Rev] |
| 609 | var damaging []int |
| 610 | for r := 1; r < w.console.Rev; r++ { |
| 611 | for k := range w.console.Records[r].Grants { |
| 612 | if !newest.Grants[k] { |
| 613 | damaging = append(damaging, r) |
| 614 | break |
| 615 | } |
| 616 | } |
| 617 | } |
| 618 | if len(damaging) > 0 { |
| 619 | if w.rng.Intn(2) == 0 { |
| 620 | return damaging[len(damaging)-1] |
| 621 | } |
| 622 | return damaging[w.rng.Intn(len(damaging))] |
| 623 | } |
| 624 | return 1 + w.rng.Intn(w.console.Rev-1) |
| 625 | } |
| 626 | |
| 627 | // walkReplay is the walk's bucket writer: it replays an older record and |
| 628 | // keeps it in place for up to twenty minutes. |
| 629 | func (w *world) walkReplay() bool { |
| 630 | if w.console.Rev < 2 { |
| 631 | return false |
| 632 | } |
| 633 | w.replayUntil(w.pickReplay(), w.now()+1+w.rng.Intn(2*staleTicks)) |
| 634 | return true |
| 635 | } |
| 636 | |
| 637 | // callers are the nested instance groups on b with an engine socket open. |
| 638 | func (w *world) callers(b *box) []string { |
| 639 | var out []string |
| 640 | for _, g := range w.groups(b) { |
| 641 | if w.socketOpen(b, g) { |
| 642 | out = append(out, g) |
| 643 | } |
| 644 | } |
| 645 | return out |
| 646 | } |
| 647 | |
| 648 | func init() { |
| 649 | stepHooks = append(stepHooks, |
| 650 | // a bucket writer whose time is up stops; the console, while up, |
| 651 | // re-signs every two minutes (R5) |
| 652 | func(w *world) { |
| 653 | c := &w.console |
| 654 | if c.Pinned != nil && w.now() >= c.PinnedUntil { |
| 655 | w.unpin() |
| 656 | } |
| 657 | if !c.Down && w.now()-c.Records[c.Rev].SignedAt >= resignTicks { |
| 658 | w.publish(c.Rev + 1) |
| 659 | } |
| 660 | }, |
| 661 | // the mirror polls every 30 s, whatever the walk does |
| 662 | func(w *world) { |
| 663 | if w.now()%pollTicks == 0 { |
| 664 | for _, b := range w.boxes { |
| 665 | w.mirrorTick(b) |
| 666 | } |
| 667 | } |
| 668 | }) |
| 669 | actions = append(actions, |
| 670 | action{name: "engine-call", weight: 4, run: func(w *world) bool { |
| 671 | b := w.pickBox() |
| 672 | gs := w.callers(b) |
| 673 | if len(gs) == 0 || len(b.Procs) == 0 { |
| 674 | return false |
| 675 | } |
| 676 | host := gs[w.rng.Intn(len(gs))] |
| 677 | // usually the instance's own process; one call in three, any |
| 678 | // process on the box connects to the socket (mode 0666) |
| 679 | p := w.newest(b, host) |
| 680 | if p == nil || w.rng.Intn(3) == 0 { |
| 681 | p = b.Procs[w.rng.Intn(len(b.Procs))] |
| 682 | } |
| 683 | c := call{from: b, p: p, uid: "carlos", host: host, engine: w.pickEngine(), path: w.pickPath()} |
| 684 | c.forged = w.maybeForged() |
| 685 | w.placeCall(c) |
| 686 | return true |
| 687 | }}, |
| 688 | action{name: "door-call", weight: 1, run: func(w *world) bool { |
| 689 | b := w.pickBox() |
| 690 | gs := w.groups(b) |
| 691 | if len(gs) == 0 { |
| 692 | return false |
| 693 | } |
| 694 | host := gs[w.rng.Intn(len(gs))] |
| 695 | c := call{from: b, viaDoor: true, host: host, engine: w.pickEngine(), path: w.pickPath()} |
| 696 | if len(b.Procs) > 0 && w.rng.Intn(2) == 0 { |
| 697 | // any process running as carlos presents host's door key |
| 698 | // (every flat tenant can read every other's config) |
| 699 | c.p, c.uid = b.Procs[w.rng.Intn(len(b.Procs))], "carlos" |
| 700 | } else { |
| 701 | // an enrolled host-app user: its own app's key, its own uid |
| 702 | c.p, c.uid = &proc{Group: host, Slice: "user:u-" + host}, "u-"+host |
| 703 | } |
| 704 | c.forged = w.maybeForged() |
| 705 | w.placeCall(c) |
| 706 | return true |
| 707 | }}, |
| 708 | action{name: "client-hop", weight: 1, run: func(w *world) bool { |
| 709 | hosts := []string{engHost, engSite} |
| 710 | c := callerOf(walkApps[w.rng.Intn(len(walkApps))], w.pickEngine()) |
| 711 | w.clientHop(w.pickBox(), hosts[w.rng.Intn(2)], w.pickPath(), c) |
| 712 | return true |
| 713 | }}, |
| 714 | action{name: "console-grant", weight: 1, run: func(w *world) bool { return w.consoleGrant(w.pickKey()) }}, |
| 715 | action{name: "console-revoke", weight: 1, run: func(w *world) bool { |
| 716 | keys := sortedKeys(w.console.Grants) |
| 717 | return len(keys) > 0 && w.consoleRevoke(keys[w.rng.Intn(len(keys))]) |
| 718 | }}, |
| 719 | action{name: "console-resign", weight: 1, run: func(w *world) bool { |
| 720 | if w.console.Down { |
| 721 | return false |
| 722 | } |
| 723 | w.publish(w.console.Rev + 1) |
| 724 | return true |
| 725 | }}, |
| 726 | action{name: "console-down", weight: 1, run: func(w *world) bool { |
| 727 | w.console.Down = !w.console.Down && !w.calm() |
| 728 | w.logf("console: down = %v", w.console.Down) |
| 729 | return true |
| 730 | }}, |
| 731 | action{name: "bucket-down", weight: 1, run: func(w *world) bool { |
| 732 | w.setBucketDown(!w.console.BucketDown && !w.calm()) |
| 733 | return true |
| 734 | }}, |
| 735 | action{name: "fast-forward", weight: 1, run: func(w *world) bool { |
| 736 | if w.calm() { |
| 737 | return false |
| 738 | } |
| 739 | if w.rng.Intn(8) == 0 { |
| 740 | w.console.Down = true |
| 741 | w.logf("console: down = true") |
| 742 | w.advance(lifetimeTicks + staleTicks) // the console is down for three days |
| 743 | return true |
| 744 | } |
| 745 | w.advance(staleTicks) // ten minutes with no poll |
| 746 | return true |
| 747 | }}, |
| 748 | // any older signed record, at any time: the freshness bound, not the |
| 749 | // walk, limits what a replay can do (R5) |
| 750 | action{name: "replay", weight: 1, run: func(w *world) bool { |
| 751 | return w.walkReplay() |
| 752 | }}, |
| 753 | action{name: "restart-edge", weight: 2, run: func(w *world) bool { |
| 754 | switch w.rng.Intn(3) { |
| 755 | case 0: // a release roll restarts every edge |
| 756 | for _, b := range w.boxes { |
| 757 | w.restartEdge(b) |
| 758 | } |
| 759 | case 1: |
| 760 | w.restartEdge(w.pickBox()) |
| 761 | default: |
| 762 | // a rebuilt box, the engine's home included; a bucket writer |
| 763 | // may replay at once |
| 764 | w.rebuild(w.pickBox()) |
| 765 | if w.rng.Intn(2) == 0 { |
| 766 | w.walkReplay() |
| 767 | } |
| 768 | } |
| 769 | return true |
| 770 | }}, |
| 771 | action{name: "arm", weight: 2, run: func(w *world) bool { |
| 772 | b := w.pickBox() |
| 773 | if b.Armed { |
| 774 | return false |
| 775 | } |
| 776 | w.arm(b) |
| 777 | return true |
| 778 | }}, |
| 779 | action{name: "disarm", weight: 1, run: func(w *world) bool { |
| 780 | b := w.pickBox() |
| 781 | if !b.Armed { |
| 782 | return false |
| 783 | } |
| 784 | w.disarm(b) |
| 785 | return true |
| 786 | }}, |
| 787 | action{name: "launch-unisolated", weight: 1, run: func(w *world) bool { |
| 788 | b := w.pickBox() |
| 789 | kind := unisolatedKinds[w.rng.Intn(len(unisolatedKinds))] |
| 790 | if !w.launchUnisolated(b, kind) || !w.flatEraTamper || b.Armed { |
| 791 | return true |
| 792 | } |
| 793 | var gs []string |
| 794 | for _, g := range w.groups(b) { |
| 795 | if !unitBacked(g) { |
| 796 | gs = append(gs, g) |
| 797 | } |
| 798 | } |
| 799 | if len(gs) > 0 { |
| 800 | w.tamper(b, w.newest(b, kind), gs[w.rng.Intn(len(gs))]) |
| 801 | } |
| 802 | return true |
| 803 | }}, |
| 804 | action{name: "unisolated-exit", weight: 1, run: func(w *world) bool { |
| 805 | b := w.pickBox() |
| 806 | var ps []*proc |
| 807 | for _, p := range b.Procs { |
| 808 | if p.Unisolated { |
| 809 | ps = append(ps, p) |
| 810 | } |
| 811 | } |
| 812 | if len(ps) == 0 { |
| 813 | return false |
| 814 | } |
| 815 | p := ps[w.rng.Intn(len(ps))] |
| 816 | for i, x := range b.Procs { |
| 817 | if x == p { |
| 818 | b.Procs = append(b.Procs[:i], b.Procs[i+1:]...) |
| 819 | break |
| 820 | } |
| 821 | } |
| 822 | w.logf("box%d: process %d (%s) exited", b.ID, p.ID, p.Slice) |
| 823 | return true |
| 824 | }}, |
| 825 | action{name: "move-to-nested", weight: 1, run: func(w *world) bool { |
| 826 | b := w.pickBox() |
| 827 | ks := sortedKeys(b.Unisolated) |
| 828 | if len(ks) == 0 { |
| 829 | return false |
| 830 | } |
| 831 | w.moveToNested(b, ks[w.rng.Intn(len(ks))]) |
| 832 | return true |
| 833 | }}, |
| 834 | ) |
| 835 | } |
| 836 | |
| 837 | // mirrorsState is a one-line summary of the console and every mirror, for a |
| 838 | // failing witness. |
| 839 | func (w *world) mirrorsState() string { |
| 840 | out := []string{fmt.Sprintf("console{rev %d bucket %d bucketDown %v down %v pinned %v now %d}", w.console.Rev, w.console.Bucket.Rev, w.console.BucketDown, w.console.Down, w.console.Pinned != nil, w.now())} |
| 841 | for _, b := range w.boxes { |
| 842 | m := b.Mirror |
| 843 | rev := 0 |
| 844 | if m.Rec != nil { |
| 845 | rev = m.Rec.Rev |
| 846 | } |
| 847 | out = append(out, fmt.Sprintf("box%d{rec %d floor %d readAt %d persisted %d readRev %d armed %v sockets %v}", b.ID, rev, m.Floor, m.ReadAt, m.PersistedFloor, b.ReadRev, b.Armed, b.Sockets)) |
| 848 | } |
| 849 | return strings.Join(out, " ") |
| 850 | } |
| 851 | |
amadan