| 1 | // The box half of `carlos restart` (issue #113): when an app's |
| 2 | // restart-generation object (release.RestartGenKey) MOVES, cycle that app's |
| 3 | // live processes on this box. Pull-based like everything else here — no push |
| 4 | // channel, no box access, no version or config change involved. |
| 5 | // |
| 6 | // IT RIDES THE SERIAL, and that is the load-bearing design decision. The |
| 7 | // console door bumps control/serial after touching the object, exactly as |
| 8 | // every other console write does, and the agent's existing 2s serial poll |
| 9 | // (fastTick) already turns a bump into the full convergence pass set. The |
| 10 | // restart check is one more leg of THAT — so a box with nothing happening |
| 11 | // pays its one serial GET per tick and NOT ONE READ MORE. An earlier revision |
| 12 | // polled every app's restart generation on its own 3s timer; measured, that |
| 13 | // was ~575k needless GETs per box per day to notice an event that arrives a |
| 14 | // handful of times a month. A verb that costs the fleet a permanent poll storm |
| 15 | // to exist is not a verb worth having. |
| 16 | // |
| 17 | // Direct-bucket touches that never bump the serial (a script writing the |
| 18 | // object straight into the bucket) converge on the hourly root pass instead — |
| 19 | // the same floor every other direct-bucket write already relies on. |
| 20 | // |
| 21 | // The work is split across two processes, for the reason config fast |
| 22 | // convergence is split the same way: |
| 23 | // |
| 24 | // - the AGENT (carlos-edge.service, User=carlos, ProtectSystem=strict) owns |
| 25 | // the EXEC-backed instances outright, so its half is the etag comparison, |
| 26 | // a bounce of its own children, and one marker file for the rest. |
| 27 | // systemctl cannot reach a child of another process, so a root-only |
| 28 | // dispatch would leave every warm exec instance — the commonest shape on |
| 29 | // the box, and the one a wedged instance usually is — serving on, |
| 30 | // unrestarted. |
| 31 | // - the ROOT pass (carlos adopt, out of carlos-restart.service via |
| 32 | // carlos-restart.path, or the hourly timer) is the only one that can |
| 33 | // restart a systemd unit — the carlos-app@ tenants and the bespoke units |
| 34 | // addr-backed routes name (issue #110's consoles). Its half is the |
| 35 | // comparison against a persisted, per-app high-water mark and the |
| 36 | // dispatch. |
| 37 | // |
| 38 | // SEED-DON'T-REPLAY governs both halves, PER APP. An app this box has not |
| 39 | // recorded yet is recorded and NOT restarted, whether its object exists or |
| 40 | // not; only movement away from a value this box has already written counts. |
| 41 | // The absent case gets its own recorded sentinel precisely so that an app's |
| 42 | // first ever request (absent -> present) is still movement rather than a first |
| 43 | // sight — that distinction is the difference between the verb working and the |
| 44 | // verb cycling a box's whole route table the day a marks file goes missing. |
| 45 | package host |
| 46 | |
| 47 | import ( |
| 48 | "context" |
| 49 | "encoding/json" |
| 50 | "errors" |
| 51 | "fmt" |
| 52 | "log" |
| 53 | "os" |
| 54 | "path/filepath" |
| 55 | "time" |
| 56 | |
| 57 | "github.com/carlosframework/platform/internal/registry" |
| 58 | "github.com/carlosframework/platform/internal/release" |
| 59 | "github.com/carlosframework/platform/internal/safefile" |
| 60 | ) |
| 61 | |
| 62 | // restartEligible widens release.Adoptable BY NAME, for this feature alone: |
| 63 | // a route release.Adoptable would cycle, plus a sidecar. release.Adoptable |
| 64 | // itself is deliberately NOT changed — it is the most reused predicate in |
| 65 | // the repo (adopt membership, capability convergence, socket and backing |
| 66 | // advisories, status pages), and every one of those is correct to exclude |
| 67 | // a sidecar. Widening it centrally would silently enrol sidecars in |
| 68 | // capability drop-ins and adopt bookkeeping that make no sense for them. |
| 69 | // `carlos restart -app X` genuinely wants sidecars too (design §9.3), so it |
| 70 | // opts in here, explicitly, rather than through Adoptable itself. |
| 71 | func restartEligible(rt registry.Route) bool { |
| 72 | return release.Adoptable(rt) || rt.IsSidecar() |
| 73 | } |
| 74 | |
| 75 | // restartCandidates filters routes to the ones the restart feature may |
| 76 | // cycle: restartEligible, applied. Exported to this package's tests as the |
| 77 | // single predicate restartApps, bounceExecRoutes' membership check and |
| 78 | // cycleWorkers all trace back to, so the three cannot silently diverge on |
| 79 | // who counts. |
| 80 | func restartCandidates(routes []registry.Route) []registry.Route { |
| 81 | var out []registry.Route |
| 82 | for _, rt := range routes { |
| 83 | if restartEligible(rt) { |
| 84 | out = append(out, rt) |
| 85 | } |
| 86 | } |
| 87 | return out |
| 88 | } |
| 89 | |
| 90 | // restartApps lists the distinct (account, app) pairs with at least one |
| 91 | // restartable route on this box — the apps whose restart generation this box |
| 92 | // reads when a serial bump (or an hourly pass) gives it a reason to. |
| 93 | // |
| 94 | // The membership predicate is restartCandidates (release.Adoptable plus a |
| 95 | // sidecar — see its doc comment): a route neither would cycle is not one |
| 96 | // whose process this feature has any business reading a restart signal |
| 97 | // for, and a second copy of that judgement here is exactly the drift that |
| 98 | // would let them disagree. It admits addr-backed routes that name their |
| 99 | // own unit (issue #110's consoles) — which is half of why #113 exists — |
| 100 | // because Adoptable filters on lifecycle and app, never on backing. |
| 101 | func restartApps(routes []registry.Route) [][2]string { |
| 102 | seen := map[[2]string]bool{} |
| 103 | var out [][2]string |
| 104 | for _, rt := range restartCandidates(routes) { |
| 105 | k := [2]string{release.RouteAccount(rt), rt.App} |
| 106 | if seen[k] { |
| 107 | continue |
| 108 | } |
| 109 | seen[k] = true |
| 110 | out = append(out, k) |
| 111 | } |
| 112 | return out |
| 113 | } |
| 114 | |
| 115 | // restartGenPass is the agent's comparison, split out for tests: read the |
| 116 | // restart-generation etag of every app this box serves and report WHICH of |
| 117 | // them moved. Same semantics — and the same reasons — as genPollPass: |
| 118 | // |
| 119 | // - First sight of an app records silently and asks for nothing. A box |
| 120 | // coming up must not restart everything it serves because it has never |
| 121 | // seen these etags before; that is the seed-don't-replay rule, and it is |
| 122 | // the whole safety property of this feature. restartSeed runs one such |
| 123 | // silent pass at arming, so the FIRST request after a boot is acted on |
| 124 | // rather than swallowed as the baseline. |
| 125 | // - An unchanged etag does nothing. |
| 126 | // - Absent that stays absent does nothing, FOREVER: an app nobody has ever |
| 127 | // restarted this way has no object and never will until someone asks. |
| 128 | // That is the ordinary state of almost every app, not a fault. |
| 129 | // - Any movement asks: the etag changed, or absent -> present (an app's |
| 130 | // first ever restart request). present -> absent should not happen |
| 131 | // (restart generations are only created and overwritten, never deleted), |
| 132 | // but it counts as movement too — one extra root pass is cheap, and |
| 133 | // pretending we did not see it would be a lie about the only thing this |
| 134 | // function reports. |
| 135 | // - The moved apps come back as a LIST, but the root request is one marker |
| 136 | // touch however long that list is: the root pass re-reads every app's etag |
| 137 | // itself and dispatches per app, so asking once per moved app would only |
| 138 | // ask the same pass to run several times. The list is for the caller's own |
| 139 | // half — which of ITS exec children to bounce. |
| 140 | // - A read error that is not "not found" leaves that app's cache entry |
| 141 | // exactly as it was and keeps evaluating the rest — the errors come back |
| 142 | // for the caller to aggregate into ONE log line per pass. Clearing the |
| 143 | // entry would turn a recovered blip into a bogus first sight or a bogus |
| 144 | // restart. |
| 145 | // - Apps that vanish from the route set lose their cache entries: a re-added |
| 146 | // app is a first sight again, which is right, because a route that has |
| 147 | // just been (re-)minted is starting its process fresh anyway. |
| 148 | // |
| 149 | // The cache is mutated in place and lives for the life of the agent process |
| 150 | // (fastConvergeLoop owns it, restartSeed fills it), which is the whole point: |
| 151 | // there is nowhere else to remember what "changed" means between two serial |
| 152 | // moves that may be days apart. |
| 153 | func restartGenPass(ctx context.Context, routes []registry.Route, store release.Store, cache map[[2]string]string) (moved [][2]string, readErrs []error) { |
| 154 | apps := restartApps(routes) |
| 155 | live := make(map[[2]string]bool, len(apps)) |
| 156 | for _, k := range apps { |
| 157 | live[k] = true |
| 158 | _, etag, err := store.Get(ctx, release.RestartGenKey(k[0], k[1])) |
| 159 | switch { |
| 160 | case errors.Is(err, os.ErrNotExist): |
| 161 | etag = genAbsent |
| 162 | case err != nil: |
| 163 | readErrs = append(readErrs, fmt.Errorf("%s/%s: %w", k[0], k[1], err)) |
| 164 | continue // cache entry untouched: this app is unknown this tick, not changed |
| 165 | } |
| 166 | prev, seen := cache[k] |
| 167 | cache[k] = etag |
| 168 | if seen && prev != etag { |
| 169 | moved = append(moved, k) |
| 170 | } |
| 171 | } |
| 172 | for k := range cache { |
| 173 | if !live[k] { |
| 174 | delete(cache, k) |
| 175 | } |
| 176 | } |
| 177 | return moved, readErrs |
| 178 | } |
| 179 | |
| 180 | // bounceExecRoutes cycles the agent's OWN children for the apps whose restart |
| 181 | // generation just moved, and returns how many it actually stopped. This is the |
| 182 | // half no root pass can perform: an exec instance is a child of THIS process, |
| 183 | // and systemctl has no name for it. |
| 184 | // |
| 185 | // The route filter is the complement of what rootRestartRoute's systemctl arms |
| 186 | // can reach — no named unit, no unit backing — so the two halves never cycle |
| 187 | // the same process twice. |
| 188 | // |
| 189 | // Coverage is NOT total, and the gap is worth naming: a route that is |
| 190 | // activator-backed (Backing "", an exec child) but ALSO carries a Unit stamp |
| 191 | // is skipped here (it names a unit) while the root half try-restarts that |
| 192 | // named unit, which is not the process the activator is running. Such a route |
| 193 | // is a misconfiguration — `carlos route -unit` is for addr-backed tenants |
| 194 | // whose process an operator owns — but nothing in this file repairs it, and |
| 195 | // its exec child is cycled by NEITHER half until somebody clears the stamp. |
| 196 | // |
| 197 | // HIBERNATE-flagged routes are included, and that is not a contradiction of |
| 198 | // "never wake a sleeper": Bounce provably cannot start anything (it is a no-op |
| 199 | // unless the instance is AWAKE), so a sleeping instance is untouched and picks |
| 200 | // its next start up naturally, while a WARM hibernator — which is what a |
| 201 | // wedged instance usually is — gets exactly the cycle that was asked for. This |
| 202 | // is release.RestartHibernating's argument, in the one process that can act on |
| 203 | // it. |
| 204 | // |
| 205 | // A nil bounce (no activator armed on this box) is a working no-op: there are |
| 206 | // no children to cycle, and the unit half still runs. |
| 207 | func bounceExecRoutes(routes []registry.Route, moved [][2]string, bounce func(host string) bool) int { |
| 208 | if bounce == nil || len(moved) == 0 { |
| 209 | return 0 |
| 210 | } |
| 211 | want := make(map[[2]string]bool, len(moved)) |
| 212 | for _, k := range moved { |
| 213 | want[k] = true |
| 214 | } |
| 215 | n := 0 |
| 216 | for _, rt := range routes { |
| 217 | if !release.Adoptable(rt) || !execBounceable(rt) { |
| 218 | continue |
| 219 | } |
| 220 | if !want[[2]string{release.RouteAccount(rt), rt.App}] { |
| 221 | continue |
| 222 | } |
| 223 | if bounce(rt.Host) { |
| 224 | n++ |
| 225 | log.Printf("agent: restart requested: bounced exec instance %s", rt.Host) |
| 226 | } |
| 227 | } |
| 228 | return n |
| 229 | } |
| 230 | |
| 231 | // dispatchBoundedWakes wakes every route in toWake, bounded by |
| 232 | // workerBootConcurrency wakes running at once, entirely inside ONE |
| 233 | // goroutine this function launches — so dispatchBoundedWakes ITSELF |
| 234 | // returns to its caller immediately, no matter how many routes are queued |
| 235 | // behind it. |
| 236 | // |
| 237 | // This is fix round 2's own correction of round 1's own fix (sidecar- |
| 238 | // members Task 13): the round-1 shape acquired the semaphore in the |
| 239 | // CALLER's loop, one route at a time — which dispatches each wake off the |
| 240 | // blocking path individually, but still leaves the loop ITSELF, and |
| 241 | // therefore the caller, blocked on the sem acquisition once |
| 242 | // workerBootConcurrency wakes are already in flight and a |
| 243 | // (workerBootConcurrency+1)th route needs dispatching. For an app shaped |
| 244 | // like Eleven's — a sidecar PER ENROLLED MEMBER, so routinely more than 8 |
| 245 | // sidecars for one app on one box — that reintroduces exactly the bug this |
| 246 | // idiom exists to fix: a restart or wake burst on such an app would stall |
| 247 | // the box's single per-tick convergence goroutine for up to wakeWaitBound |
| 248 | // (30s) per sidecar past the 8th. Moving the WHOLE bounded fan-out inside |
| 249 | // one dispatched goroutine fixes it: the bound still governs how many |
| 250 | // wakes run concurrently, and now ONLY that — never when the caller gets |
| 251 | // to continue. |
| 252 | // |
| 253 | // Shared by cycleWorkers (this file) and sidecarwake.go's |
| 254 | // dispatchSidecarWakes, the two callers of `carlos restart` and the |
| 255 | // sidecar wake feature respectively that both need "wake N sidecars |
| 256 | // without blocking the caller, bounded" — one idiom, not two copies that |
| 257 | // would each need this fix applied separately. |
| 258 | // |
| 259 | // wake is called once per route in toWake; report, if non-nil, is called |
| 260 | // once per wake with its result (nil error on success) so each caller can |
| 261 | // log in its own voice. A nil wake or an empty toWake is a working no-op. |
| 262 | func dispatchBoundedWakes(toWake []registry.Route, wake func(rt registry.Route) error, report func(rt registry.Route, err error)) { |
| 263 | if wake == nil || len(toWake) == 0 { |
| 264 | return |
| 265 | } |
| 266 | go func() { |
| 267 | sem := make(chan struct{}, workerBootConcurrency) |
| 268 | for _, rt := range toWake { |
| 269 | rt := rt |
| 270 | sem <- struct{}{} |
| 271 | go func() { |
| 272 | defer func() { <-sem }() |
| 273 | err := wake(rt) |
| 274 | if report != nil { |
| 275 | report(rt, err) |
| 276 | } |
| 277 | }() |
| 278 | } |
| 279 | }() |
| 280 | } |
| 281 | |
| 282 | // cycleWorkers is bounceExecRoutes' sidecar counterpart, and deliberately a |
| 283 | // separate function rather than a widened bounceExecRoutes: a sidecar is |
| 284 | // activator-managed exactly like a socket-exec instance, but Bounce alone |
| 285 | // cannot cycle one the way it cycles an instance. Nothing dials a worker |
| 286 | // the way a request dials an instance — an instance's next request supplies |
| 287 | // the wake half of "stop now, respawn with fresh config"; a worker stopped |
| 288 | // here has no other wake source at all (an always-on one is stranded until |
| 289 | // the agent restarts, since the boot pass that starts it runs once, at |
| 290 | // arming, never again; a hibernating one stays silent until its schedule |
| 291 | // next comes due, which "carlos restart" did not ask for). Stop and wake |
| 292 | // are therefore ONE operation for a sidecar — see |
| 293 | // activator.Service.Bounce's own doc comment, which states the same |
| 294 | // contract from the activator's side. |
| 295 | // |
| 296 | // The BOUNCE half is synchronous, in-process and effectively instant — it |
| 297 | // is counted in the return value, exactly like bounceExecRoutes' own bounce |
| 298 | // call, and it stays in THIS function's own loop: only the eligible, |
| 299 | // actually-stopped routes are collected into toWake. The WAKE half is NOT |
| 300 | // instant: WakeWorker can block up to wakeWaitBound (30s, activator.go) per |
| 301 | // sidecar waiting out the settle window, and this function's only caller |
| 302 | // (restartRequestLeg) runs synchronously on the box's single per-tick |
| 303 | // convergence goroutine (fastTick) — waiting on it, even indirectly via a |
| 304 | // per-route semaphore acquisition in this function's own loop, would delay |
| 305 | // every other app's serial-driven convergence behind it (see |
| 306 | // dispatchBoundedWakes' own doc comment for the exact shape of that bug, |
| 307 | // caught and fixed in review round 2). The whole bounded fan-out is |
| 308 | // therefore handed to dispatchBoundedWakes, which returns immediately |
| 309 | // regardless of how many routes it was given. Safe because WakeWorker is |
| 310 | // idempotent and `carlos restart`'s contract is "cycle the processes", not |
| 311 | // "return once they are up" — a sidecar that comes up a moment after this |
| 312 | // tick has still been restarted. A wake that errors is logged and does not |
| 313 | // stop, delay, or fail the others. |
| 314 | // |
| 315 | // wake is called ONLY for a host bounce actually stopped (true): a sidecar |
| 316 | // that was already asleep is left asleep, the same "never wake a sleeper" |
| 317 | // rule bounceExecRoutes' own HIBERNATE handling already gives an instance. |
| 318 | // A nil bounce or wake (no activator armed) is a working no-op. |
| 319 | func cycleWorkers(routes []registry.Route, moved [][2]string, bounce func(host string) bool, wake func(rt registry.Route) error) int { |
| 320 | if bounce == nil || wake == nil || len(moved) == 0 { |
| 321 | return 0 |
| 322 | } |
| 323 | want := make(map[[2]string]bool, len(moved)) |
| 324 | for _, k := range moved { |
| 325 | want[k] = true |
| 326 | } |
| 327 | n := 0 |
| 328 | var toWake []registry.Route |
| 329 | for _, rt := range routes { |
| 330 | if !rt.IsSidecar() || !want[[2]string{release.RouteAccount(rt), rt.App}] { |
| 331 | continue |
| 332 | } |
| 333 | if !bounce(rt.Host) { |
| 334 | continue |
| 335 | } |
| 336 | n++ |
| 337 | log.Printf("agent: restart requested: bounced sidecar %s", rt.Host) |
| 338 | toWake = append(toWake, rt) |
| 339 | } |
| 340 | dispatchBoundedWakes(toWake, wake, func(rt registry.Route, err error) { |
| 341 | if err != nil { |
| 342 | log.Printf("agent: restart requested: re-wake sidecar %s: %v", rt.Host, err) |
| 343 | } |
| 344 | }) |
| 345 | return n |
| 346 | } |
| 347 | |
| 348 | // restartRequestMarker is the file the agent's restart leg touches to ask a |
| 349 | // root pass to run now: carlos-restart.path watches it (PathChanged) and starts |
| 350 | // carlos-restart.service (`carlos adopt -restart-only`), which owns the half |
| 351 | // this unprivileged process cannot do — restarting units — and only that half. |
| 352 | // |
| 353 | // A sibling of configSyncMarker rather than a reuse of it, because the two |
| 354 | // relays land on different services: config's target is `carlos adopt |
| 355 | // -config-only`, which renders /etc/carlos/apps and restarts only the routes |
| 356 | // whose env file actually changed. A restart request changes no file, so that |
| 357 | // pass would find nothing to do and cycle nothing at all. |
| 358 | // |
| 359 | // It lives in the data dir, deliberately outside the *.config.env namespace |
| 360 | // reapConfigEnv sweeps — the reaper matches that suffix alone and must go on |
| 361 | // ignoring this. |
| 362 | const restartRequestMarker = "restart-requested" |
| 363 | |
| 364 | // requestRootRestart touches the marker. The content is a timestamp only so an |
| 365 | // operator reading /data can see when the last request went out; systemd |
| 366 | // watches the path, not the bytes. Errors are the caller's to log and swallow: |
| 367 | // a request that cannot be written costs latency, not correctness, because |
| 368 | // carlos-adopt.timer's hourly pass runs the same comparison underneath it. |
| 369 | func requestRootRestart(dd string) error { |
| 370 | return safefile.WriteFile(filepath.Join(dd, restartRequestMarker), |
| 371 | []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644) |
| 372 | } |
| 373 | |
| 374 | // restartMarksFile is the ROOT pass's memory: the last restart generation it |
| 375 | // has already acted on, per app. It lives in the data dir beside the bake |
| 376 | // clock, in the same JSON + temp-file-rename shape and for the same |
| 377 | // cross-uid reason (see restartMarks.save). |
| 378 | const restartMarksFile = "restart-marks.json" |
| 379 | |
| 380 | func restartMarksPath(dd string) string { return filepath.Join(dd, restartMarksFile) } |
| 381 | |
| 382 | // restartMarks is that memory. Apps is keyed "<account>|<app>" — "|" is |
| 383 | // outside every character class a sqid or an app name allows, the same |
| 384 | // separator argument internal/release's bake clock makes for its own keys. |
| 385 | // |
| 386 | // A value is either the etag this box last acted on OR restartAbsent, the |
| 387 | // recorded "this app has no restart generation and I have looked". On disk: |
| 388 | // |
| 389 | // { |
| 390 | // "schema": 1, |
| 391 | // "apps": { |
| 392 | // "bab|keymail": "\"9f86d0…\"", |
| 393 | // "bab|titogo": "\u0000absent" |
| 394 | // } |
| 395 | // } |
| 396 | // |
| 397 | // The absent sentinel is what makes per-app seed-don't-replay possible. Without |
| 398 | // it "no entry" has to mean both "never looked" and "looked, nothing there", |
| 399 | // and one of those two has to be wrong: read as movement, a box gaining a new |
| 400 | // route for a previously-restarted app cycles it for a request served weeks |
| 401 | // ago (and a box whose registry lands after its marks file cycles EVERY route |
| 402 | // at once); read as a first sight, an app's first ever request is silently |
| 403 | // dropped. Recording the absence tells them apart — a later object sighting is |
| 404 | // movement from a value this box wrote, which is exactly what it is. |
| 405 | // |
| 406 | // Entries are never pruned when a route leaves the box. A vanished app's mark |
| 407 | // costs one map entry; DROPPING it would make the app's return look like a |
| 408 | // brand-new object and cycle it once for a request that was already served. |
| 409 | type restartMarks struct { |
| 410 | Schema int `json:"schema"` // 1 |
| 411 | Apps map[string]string `json:"apps"` |
| 412 | // Scopes is the same memory for NARROWED requests (discussion #16), |
| 413 | // keyed identically: the id of the last request in |
| 414 | // release.RestartScopeKey's log this box has acted on, or |
| 415 | // release.RestartScopeNone for "looked, nothing there". |
| 416 | // |
| 417 | // Additive in both directions, because this file is read and written by |
| 418 | // two binaries that roll at different times. A binary that predates the |
| 419 | // field ignores it on read (encoding/json drops unknown keys) and DROPS |
| 420 | // it on write — which costs a downgraded box exactly one re-baseline, |
| 421 | // after which it cycles nothing it was not asked to. |
| 422 | // |
| 423 | // The other direction is free of REPLAYS but not of drops, and the |
| 424 | // difference matters: a marks file with no `scopes` key is a box that has |
| 425 | // never looked at the log, which is a first sight, which is a seed — safe, |
| 426 | // and exactly wrong for a narrowed request that lands before that seed is |
| 427 | // written. Since EVERY box that ran the app-wide feature has such a file, |
| 428 | // that window is closed from the agent's side rather than left to the |
| 429 | // hourly floor; see seedRestartMarks' needRootPass. |
| 430 | Scopes map[string]string `json:"scopes,omitempty"` |
| 431 | } |
| 432 | |
| 433 | // restartAbsent is the marks value for an app this box has SEEN and found to |
| 434 | // have no restart-generation object at all. It is deliberately the SAME value |
| 435 | // the in-memory sweep uses (genAbsent), so the agent's memory and the root |
| 436 | // pass's memory cannot drift apart on what "absent" is, and it is safe for the |
| 437 | // same reason: no store produces an etag containing a NUL byte (S3's are |
| 438 | // hex/quoted, DirStore's is hex). JSON renders it "\u0000absent". |
| 439 | const restartAbsent = genAbsent |
| 440 | |
| 441 | func restartMarkKey(acct, app string) string { return acct + "|" + app } |
| 442 | |
| 443 | // lockRestartMarks takes an exclusive flock for the whole of one restartPass. |
| 444 | // Three root passes can run this — carlos-restart.service (the relay), |
| 445 | // carlos-stage.service and carlos-adopt.service — and systemd will happily |
| 446 | // start them at the same second. Unserialized, two passes read the same marks, |
| 447 | // both dispatch (one request, two try-restarts, on a verb whose entire job is |
| 448 | // stopping processes) and then both write, so whichever finishes last silently |
| 449 | // reverts the other's advance and the NEXT pass restarts everything a third |
| 450 | // time. |
| 451 | // |
| 452 | // The lock lives in a sibling <marks>.lock rather than on the marks file |
| 453 | // itself, because the marks file is replaced by rename: a flock is held on an |
| 454 | // inode, and the winner's rename would leave the loser holding a lock on a file |
| 455 | // that is no longer the marks file. |
| 456 | // |
| 457 | // Opening it is ALSO the writability test the dispatch is gated on (see |
| 458 | // restartPass): it is created in the same directory, with the same permissions |
| 459 | // the marks save needs, so a full or read-only /data fails here — before |
| 460 | // anything is cycled — instead of after, which is the difference between one |
| 461 | // logged refusal and an hourly restart of every app on the box forever. |
| 462 | // |
| 463 | // The lock itself is taken through lockFileEx, which is per-OS: linux and |
| 464 | // darwin flock, Windows refuses (see flock_windows.go — refusing dispatches |
| 465 | // nothing, which is the safe answer on a platform that never runs a pass). |
| 466 | func lockRestartMarks(path string) (unlock func(), err error) { |
| 467 | return lockExclusive(path + ".lock") |
| 468 | } |
| 469 | |
| 470 | // newRestartMarks is an empty memory: both maps non-nil, so every caller can |
| 471 | // write to them without a nil check and a marks file that predates one of them |
| 472 | // cannot leave a nil map behind. |
| 473 | func newRestartMarks() *restartMarks { |
| 474 | return &restartMarks{Schema: 1, Apps: map[string]string{}, Scopes: map[string]string{}} |
| 475 | } |
| 476 | |
| 477 | // loadRestartMarks reads the marks file. `known` is the seed-don't-replay |
| 478 | // switch and the only subtle thing here: it is FALSE when the file is missing, |
| 479 | // unreadable, or does not parse — three different accidents with one correct |
| 480 | // response, which is to record what the bucket currently says and restart |
| 481 | // NOTHING this pass. |
| 482 | // |
| 483 | // That direction is the opposite of the bake clock's ("an unreadable clock |
| 484 | // fails toward baking"), and deliberately so: baking is the cautious act there, |
| 485 | // while restarting is the DISRUPTIVE act here. The cost of failing this way is |
| 486 | // that a box whose marks file was just wiped can drop ONE restart request — |
| 487 | // logged, and re-runnable by the person who typed it. The cost of failing the |
| 488 | // other way is a box that cycles every app it serves the moment its state file |
| 489 | // goes missing. |
| 490 | func loadRestartMarks(path string) (marks *restartMarks, known bool) { |
| 491 | m := newRestartMarks() |
| 492 | raw, err := safefile.ReadFile(path) |
| 493 | if err != nil { |
| 494 | if !errors.Is(err, os.ErrNotExist) { |
| 495 | log.Printf("adopt: restart marks %s unreadable (%v) — re-baselining, no restarts this pass", path, err) |
| 496 | } |
| 497 | return m, false |
| 498 | } |
| 499 | if err := json.Unmarshal(raw, m); err != nil { |
| 500 | log.Printf("adopt: restart marks %s do not parse (%v) — re-baselining, no restarts this pass", path, err) |
| 501 | return newRestartMarks(), false |
| 502 | } |
| 503 | if m.Apps == nil { |
| 504 | m.Apps = map[string]string{} |
| 505 | } |
| 506 | // A file written before narrowed restarts existed carries no `scopes` key |
| 507 | // at all; that is a first sight of every app's log, not an empty memory of |
| 508 | // one, and the seed rules below turn it into exactly that. Which is why |
| 509 | // the agent asks for a pass that performs that seed as it arms, instead of |
| 510 | // leaving a request typed in the meantime to meet it — scopeBaselineIncomplete. |
| 511 | if m.Scopes == nil { |
| 512 | m.Scopes = map[string]string{} |
| 513 | } |
| 514 | m.Schema = 1 |
| 515 | return m, true |
| 516 | } |
| 517 | |
| 518 | // save persists the marks via temp-file-in-the-same-dir + rename, the |
| 519 | // co-written-file pattern release.FirstSeen.Save documents at length: /data is |
| 520 | // written by BOTH uids on this box (carlos for the agent's seed below, root |
| 521 | // for this pass), and an in-place O_TRUNC write needs write permission on the |
| 522 | // existing file, so whichever uid created it would lock the other out forever. |
| 523 | // rename(2) only needs write permission on the directory, which both have. The |
| 524 | // temp name is minted fresh per attempt so a crash between write and rename |
| 525 | // can never leave cross-uid residue for the next save to trip over, and the |
| 526 | // 0600 CreateTemp mints is widened to 0644 BEFORE the rename so the other uid |
| 527 | // can still read it. |
| 528 | func (m *restartMarks) save(path string) error { |
| 529 | raw, err := json.MarshalIndent(m, "", " ") |
| 530 | if err != nil { |
| 531 | return err |
| 532 | } |
| 533 | raw = append(raw, '\n') |
| 534 | return safefile.WriteFile(path, raw, 0o644) |
| 535 | } |
| 536 | |
| 537 | // seedRestartMarks writes the baseline the ROOT pass compares against, ONCE, |
| 538 | // from the agent's arming sweep — create-only (O_EXCL), so it can never |
| 539 | // overwrite a mark the root pass has already advanced. |
| 540 | // |
| 541 | // It exists to close a real hole in the handover between the two halves. The |
| 542 | // agent seeds its own baseline as it arms; the root pass would otherwise not |
| 543 | // seed until it next ran, and a restart requested inside that window would |
| 544 | // arrive at a pass with no baseline — which, by the rule above, records and |
| 545 | // restarts nothing. The request would be silently dropped, having told its |
| 546 | // caller "requested". Creating the file here shrinks that window to the moment |
| 547 | // the agent starts, where the box has just restarted everything anyway. |
| 548 | // |
| 549 | // An EMPTY cache writes nothing at all. A sweep that saw no adoptable routes |
| 550 | // is not a baseline — most often it is a box whose registry has not landed yet |
| 551 | // (a fresh machine, or one restoring /data from litestream) — and a file |
| 552 | // claiming "this box knows about no apps" is precisely the file that makes the |
| 553 | // root pass treat every route it later gains as movement, and cycle the lot in |
| 554 | // one pass. |
| 555 | // |
| 556 | // Absent objects ARE recorded, with restartAbsent: "looked, nothing there" is |
| 557 | // the fact that makes an app's first ever request movement rather than a first |
| 558 | // sight. |
| 559 | // |
| 560 | // needRootPass is the SECOND half of the handover, and it exists because |
| 561 | // create-only cannot repair a baseline that is merely INCOMPLETE. Every box |
| 562 | // that ran the app-wide feature already has a marks file, written before |
| 563 | // narrowed restarts existed and so carrying no `scopes` key at all. This |
| 564 | // function will not touch it — that is the whole create-only rule — so the |
| 565 | // narrowed mark stays missing until a ROOT pass records it, and until then |
| 566 | // restartScopeLeg reads every app as a first sight and cycles nothing for a |
| 567 | // request the CLI has already answered "restart requested". Exec-shaped routes |
| 568 | // survive that (the agent bounces its own children from mem.scope), but |
| 569 | // unit-shaped ones — Tito's relay and router, the shape these flags exist for |
| 570 | // — are dropped silently, for as long as it takes the next root pass to run: |
| 571 | // worst case the hourly timer. |
| 572 | // |
| 573 | // So when the baseline on disk is missing a narrowed mark for an app this box |
| 574 | // serves, say so, and let restartSeed ask for ONE root pass. That pass is the |
| 575 | // same pass carlos-adopt.timer runs anyway, under the same lock, against the |
| 576 | // same marks file and the same seed rules — this changes WHEN it runs, never |
| 577 | // what it decides — so it can neither replay a request (it dispatches on the |
| 578 | // per-app marks, which nothing here writes) nor drop one (it strictly shrinks |
| 579 | // the window in which a narrowed request meets a pass with no narrowed |
| 580 | // baseline, from an hour to the moment the agent arms, which is the same floor |
| 581 | // seedRestartMarks already buys the app-wide half). |
| 582 | func seedRestartMarks(path string, mem *restartMemory) (needRootPass bool, err error) { |
| 583 | if mem == nil || len(mem.gen) == 0 { |
| 584 | return false, nil // nothing was seen; nothing may be claimed |
| 585 | } |
| 586 | if _, err := os.Stat(path); err == nil { |
| 587 | // The root pass (or an earlier agent) already has a baseline — but it |
| 588 | // may predate narrowing, and a half-baseline is what silently eats a |
| 589 | // narrowed request. |
| 590 | return scopeBaselineIncomplete(path, mem), nil |
| 591 | } else if !errors.Is(err, os.ErrNotExist) { |
| 592 | return false, err |
| 593 | } |
| 594 | m := newRestartMarks() |
| 595 | for k, etag := range mem.gen { |
| 596 | m.Apps[restartMarkKey(k[0], k[1])] = etag |
| 597 | } |
| 598 | // Both memories or neither: restartSeed only gets here when every app's |
| 599 | // BOTH objects read cleanly, so a baseline that recorded one and left the |
| 600 | // other for the root pass to establish would be a baseline with a known |
| 601 | // hole in it — which is the one shape restartPass refuses to write. |
| 602 | for k, id := range mem.scope { |
| 603 | m.Scopes[restartMarkKey(k[0], k[1])] = id |
| 604 | } |
| 605 | raw, err := json.MarshalIndent(m, "", " ") |
| 606 | if err != nil { |
| 607 | return false, err |
| 608 | } |
| 609 | raw = append(raw, '\n') |
| 610 | f, err := safefile.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) |
| 611 | if err != nil { |
| 612 | if errors.Is(err, os.ErrExist) { |
| 613 | // Lost the race to the root pass; its mark is the better one — but |
| 614 | // a pass running an OLD binary writes no scopes key, so the same |
| 615 | // question has to be asked of what actually landed. |
| 616 | return scopeBaselineIncomplete(path, mem), nil |
| 617 | } |
| 618 | return false, err |
| 619 | } |
| 620 | _, writeErr := f.Write(raw) |
| 621 | if err := f.Close(); err != nil && writeErr == nil { |
| 622 | writeErr = err |
| 623 | } |
| 624 | // The file this call just wrote carries both memories, so there is nothing |
| 625 | // left for a root pass to record. |
| 626 | return false, writeErr |
| 627 | } |
| 628 | |
| 629 | // scopeBaselineIncomplete reports whether the marks file on disk leaves the |
| 630 | // ROOT pass with no narrowed-restart position for an app this box serves. |
| 631 | // |
| 632 | // Two shapes count, and they are the same fault: |
| 633 | // |
| 634 | // - the file does not load (missing, unreadable, unparseable), so the next |
| 635 | // pass re-baselines everything — which is right, and worth doing NOW |
| 636 | // rather than at the hourly floor, since a re-baselining pass dispatches |
| 637 | // nothing; |
| 638 | // - the file loads but has no `scopes` entry for some app in this sweep, |
| 639 | // which is exactly what a marks file written before narrowing existed |
| 640 | // looks like. |
| 641 | // |
| 642 | // It reads and never writes. The agent cannot take restartPass's lock — that |
| 643 | // lock file is created by root and doubles as the ROOT pass's writability |
| 644 | // test — and a lock-free read-modify-write here would race a running pass and |
| 645 | // revert the advance it had just made, which is the one accident the lock |
| 646 | // exists to prevent. Reporting the gap and letting the pass that owns the file |
| 647 | // fill it in is the whole fix. |
| 648 | func scopeBaselineIncomplete(path string, mem *restartMemory) bool { |
| 649 | marks, known := loadRestartMarks(path) |
| 650 | if !known { |
| 651 | return true |
| 652 | } |
| 653 | for k := range mem.scope { |
| 654 | if _, seen := marks.Scopes[restartMarkKey(k[0], k[1])]; !seen { |
| 655 | return true |
| 656 | } |
| 657 | } |
| 658 | return false |
| 659 | } |
| 660 | |
| 661 | // restartPass is the ROOT half: for every app on this box whose restart |
| 662 | // generation has moved since the mark this box recorded, cycle that app's |
| 663 | // routes. Called from RunAdopt — on the marker-triggered `-restart-only` run |
| 664 | // (seconds after the request) and on the ordinary pass (the hourly floor |
| 665 | // underneath the relay, which can drop an event that lands while the service |
| 666 | // is mid-run). |
| 667 | // |
| 668 | // The rules, each a deliberate choice: |
| 669 | // |
| 670 | // - ZERO adoptable routes ends the pass before anything is read or written. |
| 671 | // A box whose registry has not landed yet (fresh, or restoring /data) must |
| 672 | // not be able to record "I know about no apps" — that file is what turns |
| 673 | // every route the box later gains into movement, and cycles the whole |
| 674 | // table in one pass. |
| 675 | // - An app this box has NOT recorded yet is recorded and NOT cycled, |
| 676 | // whether its object exists or not. Per-app seed-don't-replay: the marks |
| 677 | // file is this box's memory of what it has acted on, and a request it |
| 678 | // never saw is not one it owes anybody. |
| 679 | // - An app with NO restart generation records restartAbsent — the sentinel |
| 680 | // that makes its FIRST ever request (absent -> present) movement rather |
| 681 | // than another first sight. Absent is legal forever; unrecorded absence is |
| 682 | // what was not. |
| 683 | // - An unknown baseline (see loadRestartMarks) records every app and |
| 684 | // dispatches nothing — and if any app could not be READ, records nothing |
| 685 | // at all: a baseline missing the apps a transient 503 hid is a baseline |
| 686 | // that turns those apps into movement on the next pass. A dropped request |
| 687 | // is recoverable (run the verb again); a restart nobody asked for is not. |
| 688 | // - HIBERNATING routes are skipped outright and not marked as a failure. |
| 689 | // Restarting a process that is not running is meaningless, waking it to |
| 690 | // restart it is worse (it would defeat hibernation for no gain), and it |
| 691 | // will exec fresh on its next wake regardless — which is the restart. A |
| 692 | // WARM hibernating exec instance is not left serving, though: the agent's |
| 693 | // own bounce (bounceExecRoutes) reaches it, in the one process that can. |
| 694 | // - A dispatch FAILURE still advances the mark, and logs. A restart request |
| 695 | // is a point-in-time event, not a desired state: a named unit that exits |
| 696 | // non-zero (a wrong unit name, a service that will not come back) would |
| 697 | // otherwise be retried on every pass, forever, for a request somebody made |
| 698 | // once. The journal line is the report; `carlos restart` again is the |
| 699 | // retry. |
| 700 | // - The mark is written only AFTER the dispatch, so a pass that dies |
| 701 | // mid-cycle retries rather than swallowing the request. |
| 702 | // - The whole pass holds an exclusive lock (lockRestartMarks), because three |
| 703 | // root units can run it at once; failing to TAKE that lock dispatches |
| 704 | // nothing, since the lock file's creation is also the proof that the marks |
| 705 | // can be saved afterwards. A pass that cycles processes and then cannot |
| 706 | // record what it did would cycle them again every hour, forever. |
| 707 | // |
| 708 | // Holds, bake windows and the release ladder take no part in any of this: |
| 709 | // nothing about which version runs changes here, so there is nothing for them |
| 710 | // to gate. |
| 711 | func restartPass(ctx context.Context, routes []registry.Route, store release.Store, marksPath string, restart func(host string) error) { |
| 712 | apps := restartApps(routes) |
| 713 | if len(apps) == 0 { |
| 714 | return // nothing to converge, and nothing this box may claim to know |
| 715 | } |
| 716 | unlock, err := lockRestartMarks(marksPath) |
| 717 | if err != nil { |
| 718 | // No lock means either a concurrent pass we cannot serialize against or |
| 719 | // a /data we cannot write. Both make dispatching the wrong move: the |
| 720 | // first double-restarts, the second restarts hourly and forever because |
| 721 | // the mark can never advance. |
| 722 | log.Printf("adopt: restart pass: cannot lock %s.lock (%v) — nothing cycled this pass", marksPath, err) |
| 723 | return |
| 724 | } |
| 725 | defer unlock() |
| 726 | |
| 727 | marks, known := loadRestartMarks(marksPath) |
| 728 | changed := false |
| 729 | readFailed := false |
| 730 | // What this pass has already cycled. A route restarted by an app-wide |
| 731 | // request must not be restarted a second time by a narrowed one that |
| 732 | // landed in the same window: one pass, one cycle per host. |
| 733 | already := map[string]bool{} |
| 734 | for _, k := range apps { |
| 735 | _, etag, err := store.Get(ctx, release.RestartGenKey(k[0], k[1])) |
| 736 | switch { |
| 737 | case errors.Is(err, os.ErrNotExist): |
| 738 | etag = restartAbsent |
| 739 | case err != nil: |
| 740 | log.Printf("adopt: restart generation %s/%s: %v (skipped this pass)", k[0], k[1], err) |
| 741 | readFailed = true |
| 742 | continue |
| 743 | } |
| 744 | key := restartMarkKey(k[0], k[1]) |
| 745 | prev, seen := marks.Apps[key] |
| 746 | if !known || !seen { |
| 747 | // SEED. Either this box's whole memory is untrustworthy, or this is |
| 748 | // the first time it has looked at this app — a route it has just |
| 749 | // gained, whose process is starting fresh anyway. Record, cycle |
| 750 | // nothing. |
| 751 | if !seen || prev != etag { |
| 752 | marks.Apps[key] = etag |
| 753 | changed = true |
| 754 | } |
| 755 | continue |
| 756 | } |
| 757 | if prev == etag { |
| 758 | continue |
| 759 | } |
| 760 | dispatched := 0 |
| 761 | for _, rt := range routes { |
| 762 | if !release.Adoptable(rt) || rt.App != k[1] || release.RouteAccount(rt) != k[0] { |
| 763 | continue |
| 764 | } |
| 765 | if rt.Hibernate { |
| 766 | log.Printf("adopt: restart requested for %s/%s: %s is hibernating — left asleep (it execs fresh on its next wake)", |
| 767 | k[0], k[1], rt.Host) |
| 768 | continue |
| 769 | } |
| 770 | if err := restart(rt.Host); err != nil { |
| 771 | log.Printf("adopt: restart requested for %s/%s: %s did not cycle: %v", k[0], k[1], rt.Host, err) |
| 772 | continue |
| 773 | } |
| 774 | already[rt.Host] = true |
| 775 | dispatched++ |
| 776 | } |
| 777 | // One line per APP, not per route, and it says DISPATCHED rather than |
| 778 | // cycled: an exec-backed route reaching rootRestartRoute lands on a |
| 779 | // try-restart of a carlos-app@ unit that does not exist — a deliberate |
| 780 | // no-op (the agent bounced that child itself) which this pass cannot |
| 781 | // tell apart from a real one. Claiming "cycled" for it would be the |
| 782 | // kind of lie issue #110 was made of. |
| 783 | log.Printf("adopt: restart requested for %s/%s: dispatched to %d route(s)", k[0], k[1], dispatched) |
| 784 | marks.Apps[key] = etag |
| 785 | changed = true |
| 786 | } |
| 787 | // The narrowed half (discussion #16), under the SAME lock, against the |
| 788 | // same marks file and the same seed rules — a second pass with a second |
| 789 | // lock would be two memories of one box's restarts, free to disagree. |
| 790 | scopeChanged, scopeReadFailed := restartScopeLeg(ctx, routes, store, apps, marks, known, restart, already) |
| 791 | changed = changed || scopeChanged |
| 792 | readFailed = readFailed || scopeReadFailed |
| 793 | if !known && readFailed { |
| 794 | // A baseline that omits the apps a blip hid is worse than no baseline: |
| 795 | // this pass would write "known", and the next one would read those |
| 796 | // missing apps as movement and cycle them. Leave the file as it is and |
| 797 | // re-baseline on a pass that can read everything. |
| 798 | log.Printf("adopt: restart pass: baselining deferred — some apps could not be read this pass") |
| 799 | return |
| 800 | } |
| 801 | // A first, unknown baseline is written even when nothing changed: that |
| 802 | // write is what makes the NEXT pass a known one, and until it lands every |
| 803 | // pass would re-baseline and drop whatever it was asked to do. |
| 804 | if changed || !known { |
| 805 | if err := marks.save(marksPath); err != nil { |
| 806 | log.Printf("adopt: save restart marks %s: %v (the next pass may repeat a restart)", marksPath, err) |
| 807 | } |
| 808 | } |
| 809 | } |
| 810 | |
| 811 | // restartRequestLeg is the agent's half of one serial move: compare each app's |
| 812 | // restart generation against the in-memory cache, bounce this process's own |
| 813 | // exec children for the apps that moved, and touch the marker that asks a root |
| 814 | // pass to cycle everything unit-shaped. Reports whether it asked. |
| 815 | // |
| 816 | // It runs ONLY when the serial moved (fastTick), which is what makes the |
| 817 | // feature free at rest: the console bumps the serial after touching the |
| 818 | // object, so every request is announced, and an idle box reads no restart |
| 819 | // generation at all. There is no loop of its own and there must not be one — |
| 820 | // see this file's package comment for what the 3s version of this cost. |
| 821 | // |
| 822 | // The bounce goes first, and not only for tidiness: it is in-process and |
| 823 | // instant, while the marker write can block on a busy /data, and the children |
| 824 | // bounced here are the ones cycled here or nowhere. That "instant" holds for |
| 825 | // EVERY bounce call this leg makes (bounceExecRoutes' and cycleWorkers' |
| 826 | // alike) — cycleWorkers' own wake calls are the one thing on this path that |
| 827 | // are not instant (WakeWorker can block up to 30s per sidecar), which is |
| 828 | // exactly why cycleWorkers dispatches them off its own return path rather |
| 829 | // than this leg waiting on them: this whole function still returns promptly |
| 830 | // after the bounces, this leg's synchronous work (the marker touch) is |
| 831 | // never delayed by a sidecar's wake, and neither is the next app's turn on |
| 832 | // this box's single per-tick convergence goroutine (fastTick). |
| 833 | // |
| 834 | // wake is svc.WakeWorker in production (wakeFn), nil-safe like bounce: it is |
| 835 | // cycleWorkers' other half, called only for the sidecars this leg actually |
| 836 | // stops (see cycleWorkers' doc comment for why a sidecar needs both calls, |
| 837 | // where an instance needs only bounce, and for why the wake is async). |
| 838 | func restartRequestLeg(ctx context.Context, routes []registry.Route, store release.Store, dd string, |
| 839 | bounce func(host string) bool, wake func(rt registry.Route) error, mem *restartMemory) bool { |
| 840 | moved, readErrs := restartGenPass(ctx, routes, store, mem.gen) |
| 841 | // The narrowed requests are read on the same move and reported the same |
| 842 | // way. Both, always: an app-wide request and a narrowed one can land in |
| 843 | // one poll window, and a leg that stopped at the first would leave the |
| 844 | // other for the hourly floor. |
| 845 | scoped, scopeErrs := restartScopePass(ctx, routes, store, mem.scope) |
| 846 | readErrs = append(readErrs, scopeErrs...) |
| 847 | // `carlos instances retry` grants ride their own log (release.HostRetryKey), |
| 848 | // read on the same move. |
| 849 | grants, grantErrs := hostRetryPass(ctx, routes, store, mem) |
| 850 | readErrs = append(readErrs, grantErrs...) |
| 851 | if len(readErrs) > 0 { |
| 852 | // ONE line per pass, never one per app: the count carries the scale, |
| 853 | // the first error the cause — they are the same cause in practice. |
| 854 | log.Printf("agent: restart check: %d apps unreadable: %v", len(readErrs), readErrs[0]) |
| 855 | } |
| 856 | // Each grant is verified, then applied where the hold is (applyRetries): |
| 857 | // the host-retry log's, and a signed retry a console built before that |
| 858 | // log left in the narrowed one, with the same checks. |
| 859 | applyRetries(routes, grants, mem) |
| 860 | applyRetries(routes, withoutHistory(scoped, mem), mem) |
| 861 | scoped = withoutRetries(scoped) |
| 862 | if len(moved) == 0 && len(scoped) == 0 { |
| 863 | return false |
| 864 | } |
| 865 | // CARLOS_SWAP=overlap: each exec restart is the scheduler's attempt, |
| 866 | // paced and identified by its request (spec §10). While managed starts |
| 867 | // are refused (refuseManaged), no exec restart is made: the wake after |
| 868 | // the stop would be refused, and the box's status names why. A box that |
| 869 | // declined the flag (swapArming) bounces as today, except an instance a |
| 870 | // failure record from an armed run holds (final review F6): its wake |
| 871 | // would be refused, so it keeps running. |
| 872 | m := armedRollout() |
| 873 | legacy := m == nil && rolloutUnavailable() == "" |
| 874 | if h := heldHolds(); legacy && h != nil { |
| 875 | bounce = h.skipRoutes(routes, bounce) |
| 876 | } |
| 877 | if m != nil { |
| 878 | feedRestarts(routes, moved, scoped, mem.gen, m.Restart) |
| 879 | } else if legacy { |
| 880 | bounceExecRoutes(routes, moved, bounce) |
| 881 | cycleWorkers(routes, moved, bounce, wake) |
| 882 | } |
| 883 | // The narrowed half reaches no sidecar at all: scopedHosts gates on |
| 884 | // release.Adoptable, which is Kind=="instance" and stays that way (design |
| 885 | // §9.3 — widening it centrally would enrol sidecars in adopt bookkeeping |
| 886 | // and capability drop-ins that make no sense for them). So a narrowed |
| 887 | // `carlos restart --host <sidecar>` cycles nothing, which is the SAFE |
| 888 | // direction — a scoped bounce with no wake beside it would stop a worker |
| 889 | // that has no other wake source — but it is a gap, not a decision this |
| 890 | // design made: narrowed restarts landed on main after this branch was |
| 891 | // written. Reaching a sidecar by name wants its own selector, and is a |
| 892 | // follow-up rather than something to widen here by accident. |
| 893 | if legacy { |
| 894 | bounceScopedRoutes(routes, scoped, bounce) |
| 895 | } |
| 896 | if err := requestRootRestart(dd); err != nil { |
| 897 | log.Printf("agent: request root restart pass: %v", err) |
| 898 | return false |
| 899 | } |
| 900 | log.Printf("agent: restart requested for %d app(s) and narrowed for %d on this box, asked for a root restart pass", |
| 901 | len(moved), len(scoped)) |
| 902 | return true |
| 903 | } |
| 904 | |
| 905 | // restartSeed is the ONE sweep this agent makes on its own initiative: at |
| 906 | // arming, read each app's restart generation once, silently, so the cache the |
| 907 | // leg compares against exists before the first serial move — and hand the same |
| 908 | // sightings to seedRestartMarks so the root pass has a baseline too. |
| 909 | // |
| 910 | // Without it the first request after a boot is the sweep that establishes the |
| 911 | // baseline, and is therefore the one request that does nothing: the person who |
| 912 | // typed `carlos restart` would have to type it twice, having been told |
| 913 | // "requested" the first time. One read per app per agent start is the whole |
| 914 | // cost, and it is paid where a box has just restarted everything anyway. |
| 915 | // |
| 916 | // A read error leaves that app out of the cache (its next sighting is a first |
| 917 | // sight — safe) and blocks the marks seed entirely, for the reason |
| 918 | // seedRestartMarks and restartPass both give: a baseline with holes in it is |
| 919 | // the thing that manufactures restarts. |
| 920 | // |
| 921 | // The one thing it may ask for is a root pass, and only when the baseline on |
| 922 | // disk is INCOMPLETE — see seedRestartMarks' needRootPass. That is the case of |
| 923 | // every box already carrying a pre-narrowing marks file, and asking is what |
| 924 | // keeps the first narrowed request after a roll from being answered "restart |
| 925 | // requested" and then cycling nothing unit-shaped. It is one marker touch, |
| 926 | // once, at arming: the pass it starts is the ordinary one, and once that pass |
| 927 | // has recorded the narrowed marks no later agent start asks again. |
| 928 | // |
| 929 | // Retry grants are the exception to "record, act on nothing" (Astra plan |
| 930 | // 9): every grant in the host-retry log, and every legacy retry in the |
| 931 | // narrowed log, is verified and applied here (applyRetries), so one written |
| 932 | // before this agent started is not lost to the baseline. One spent or made |
| 933 | // stale since clears nothing. |
| 934 | func restartSeed(ctx context.Context, reg *registry.Registry, store release.Store, dd string, mem *restartMemory) { |
| 935 | routes, err := reg.List() |
| 936 | if err != nil { |
| 937 | log.Printf("agent: restart baseline: list routes: %v (the first request may need a second try)", err) |
| 938 | return |
| 939 | } |
| 940 | _, readErrs := restartGenPass(ctx, routes, store, mem.gen) |
| 941 | // A first sight cycles nothing, but it hands over the retry grants in |
| 942 | // both logs: they are not seed-skipped (Astra plan 9), so a grant |
| 943 | // written before this agent started, and not yet applied, is applied |
| 944 | // now. A grant spent or made stale since clears nothing. |
| 945 | scoped, scopeErrs := restartScopePass(ctx, routes, store, mem.scope) |
| 946 | readErrs = append(readErrs, scopeErrs...) |
| 947 | grants, grantErrs := hostRetryPass(ctx, routes, store, mem) |
| 948 | if len(grantErrs) > 0 { |
| 949 | // Not a baseline hole: the host-retry log is no restart signal, and |
| 950 | // an app it could not be read for is read again on the next move. |
| 951 | log.Printf("agent: retry baseline: %d apps unreadable: %v", len(grantErrs), grantErrs[0]) |
| 952 | } |
| 953 | applyRetries(routes, grants, mem) |
| 954 | if mem.retryHost != nil { |
| 955 | // A box with nothing that could clear a hold reads every legacy |
| 956 | // retry at its start as history, silently, as it always has: its |
| 957 | // line would only say, at every start, that nothing is held. |
| 958 | applyRetries(routes, scoped, mem) |
| 959 | } |
| 960 | if len(readErrs) > 0 { |
| 961 | log.Printf("agent: restart baseline: %d apps unreadable: %v", len(readErrs), readErrs[0]) |
| 962 | return |
| 963 | } |
| 964 | needRootPass, err := seedRestartMarks(restartMarksPath(dd), mem) |
| 965 | if err != nil { |
| 966 | log.Printf("agent: seed restart marks: %v", err) |
| 967 | return |
| 968 | } |
| 969 | if !needRootPass { |
| 970 | return |
| 971 | } |
| 972 | if err := requestRootRestart(dd); err != nil { |
| 973 | // Latency, not correctness: carlos-adopt.timer's hourly pass records |
| 974 | // the same marks, it just leaves the window open for longer. |
| 975 | log.Printf("agent: ask for a baselining root restart pass: %v", err) |
| 976 | return |
| 977 | } |
| 978 | log.Printf("agent: restart baseline: the root pass has no narrowed-restart mark for at least one app on this box " + |
| 979 | "(a marks file written before narrowed restarts existed) — asked for one root pass to record it") |
| 980 | } |
| 981 | |
amadan