| index 0f206b725..7ebe61726 100644 |
| --- a/internal/host/restart.go |
| +++ b/internal/host/restart.go |
| @@ -1,30 +1,47 @@ |
| -// The box half of `carlos restart` (issue #113): watch each app's |
| -// restart-generation object (release.RestartGenKey) and, when one MOVES, |
| -// cycle that app's live processes on this box. Pull-based like everything |
| -// else here — no push channel, no box access, no version or config change |
| -// involved. |
| -// |
| -// The shape is config fast convergence's, one step sideways, and it is split |
| -// across the same two processes for the same reason: |
| -// |
| -// - the AGENT (carlos-edge.service, User=carlos, ProtectSystem=strict) can |
| -// read the bucket and see a request within seconds, and it owns the |
| -// EXEC-backed instances outright — so its half is the etag sweep, a bounce |
| -// of its own children, and one marker file for the rest. |
| +// The box half of `carlos restart` (issue #113): when an app's |
| +// restart-generation object (release.RestartGenKey) MOVES, cycle that app's |
| +// live processes on this box. Pull-based like everything else here — no push |
| +// channel, no box access, no version or config change involved. |
| +// |
| +// IT RIDES THE SERIAL, and that is the load-bearing design decision. The |
| +// console door bumps control/serial after touching the object, exactly as |
| +// every other console write does, and the agent's existing 2s serial poll |
| +// (fastTick) already turns a bump into the full convergence pass set. The |
| +// restart check is one more leg of THAT — so a box with nothing happening |
| +// pays its one serial GET per tick and NOT ONE READ MORE. An earlier revision |
| +// polled every app's restart generation on its own 3s timer; measured, that |
| +// was ~575k needless GETs per box per day to notice an event that arrives a |
| +// handful of times a month. A verb that costs the fleet a permanent poll storm |
| +// to exist is not a verb worth having. |
| +// |
| +// Direct-bucket touches that never bump the serial (a script writing the |
| +// object straight into the bucket) converge on the hourly root pass instead — |
| +// the same floor every other direct-bucket write already relies on. |
| +// |
| +// The work is split across two processes, for the reason config fast |
| +// convergence is split the same way: |
| +// |
| +// - the AGENT (carlos-edge.service, User=carlos, ProtectSystem=strict) owns |
| +// the EXEC-backed instances outright, so its half is the etag comparison, |
| +// a bounce of its own children, and one marker file for the rest. |
| +// systemctl cannot reach a child of another process, so a root-only |
| +// dispatch would leave every warm exec instance — the commonest shape on |
| +// the box, and the one a wedged instance usually is — serving on, |
| +// unrestarted. |
| // - the ROOT pass (carlos adopt, out of carlos-restart.service via |
| // carlos-restart.path, or the hourly timer) is the only one that can |
| // restart a systemd unit — the carlos-app@ tenants and the bespoke units |
| // addr-backed routes name (issue #110's consoles). Its half is the |
| -// comparison against a persisted high-water mark and the dispatch. |
| +// comparison against a persisted, per-app high-water mark and the |
| +// dispatch. |
| // |
| -// The split is exactly config fast convergence's: systemctl cannot reach a |
| -// child of another process, so a root-only dispatch would leave every warm |
| -// exec instance — the commonest shape on the box, and the one a wedged |
| -// instance usually is — serving on, unrestarted. |
| -// |
| -// SEED-DON'T-REPLAY governs both halves. A box that has never recorded a |
| -// baseline records one and restarts NOTHING: the first sight of an object is |
| -// never a request, or every upgrade would bounce the whole box. |
| +// SEED-DON'T-REPLAY governs both halves, PER APP. An app this box has not |
| +// recorded yet is recorded and NOT restarted, whether its object exists or |
| +// not; only movement away from a value this box has already written counts. |
| +// The absent case gets its own recorded sentinel precisely so that an app's |
| +// first ever request (absent -> present) is still movement rather than a first |
| +// sight — that distinction is the difference between the verb working and the |
| +// verb cycling a box's whole route table the day a marks file goes missing. |
| package host |
| |
| import ( |
| @@ -33,19 +50,18 @@ import ( |
| "errors" |
| "fmt" |
| "log" |
| - "math/rand/v2" |
| "os" |
| "path/filepath" |
| + "syscall" |
| "time" |
| |
| - "github.com/carlosframework/platform/internal/activator" |
| "github.com/carlosframework/platform/internal/registry" |
| "github.com/carlosframework/platform/internal/release" |
| ) |
| |
| // restartApps lists the distinct (account, app) pairs with at least one |
| -// restartable route on this box — the apps whose restart generation is worth |
| -// one small GET per tick. |
| +// restartable route on this box — the apps whose restart generation this box |
| +// reads when a serial bump (or an hourly pass) gives it a reason to. |
| // |
| // The membership predicate is release.Adoptable, deliberately the SAME one the |
| // adopt pass moves and the fast sweep counts: a route a pass would not adopt |
| @@ -71,14 +87,16 @@ func restartApps(routes []registry.Route) [][2]string { |
| return out |
| } |
| |
| -// restartGenPass is one tick of restartGenLoop, split out for tests: read the |
| +// restartGenPass is the agent's comparison, split out for tests: read the |
| // restart-generation etag of every app this box serves and report WHICH of |
| // them moved. Same semantics — and the same reasons — as genPollPass: |
| // |
| // - First sight of an app records silently and asks for nothing. A box |
| // coming up must not restart everything it serves because it has never |
| // seen these etags before; that is the seed-don't-replay rule, and it is |
| -// the whole safety property of this feature. |
| +// the whole safety property of this feature. restartSeed runs one such |
| +// silent pass at arming, so the FIRST request after a boot is acted on |
| +// rather than swallowed as the baseline. |
| // - An unchanged etag does nothing. |
| // - Absent that stays absent does nothing, FOREVER: an app nobody has ever |
| // restarted this way has no object and never will until someone asks. |
| @@ -103,8 +121,10 @@ func restartApps(routes []registry.Route) [][2]string { |
| // app is a first sight again, which is right, because a route that has |
| // just been (re-)minted is starting its process fresh anyway. |
| // |
| -// The cache is mutated in place — it lives across ticks in the loop, which is |
| -// the whole point (there is nowhere else to remember what "changed" means). |
| +// The cache is mutated in place and lives for the life of the agent process |
| +// (fastConvergeLoop owns it, restartSeed fills it), which is the whole point: |
| +// there is nowhere else to remember what "changed" means between two serial |
| +// moves that may be days apart. |
| func restartGenPass(ctx context.Context, routes []registry.Route, store release.Store, cache map[[2]string]string) (moved [][2]string, readErrs []error) { |
| apps := restartApps(routes) |
| live := make(map[[2]string]bool, len(apps)) |
| @@ -137,10 +157,17 @@ func restartGenPass(ctx context.Context, routes []registry.Route, store release. |
| // half no root pass can perform: an exec instance is a child of THIS process, |
| // and systemctl has no name for it. |
| // |
| -// The route filter is the complement of what rootRestartRoute can reach — no |
| -// named unit, no unit backing — so the two halves cannot cycle the same |
| -// process twice, and every restartable route is covered by exactly one of |
| -// them. |
| +// The route filter is the complement of what rootRestartRoute's systemctl arms |
| +// can reach — no named unit, no unit backing — so the two halves never cycle |
| +// the same process twice. |
| +// |
| +// Coverage is NOT total, and the gap is worth naming: a route that is |
| +// activator-backed (Backing "", an exec child) but ALSO carries a Unit stamp |
| +// is skipped here (it names a unit) while the root half try-restarts that |
| +// named unit, which is not the process the activator is running. Such a route |
| +// is a misconfiguration — `carlos route -unit` is for addr-backed tenants |
| +// whose process an operator owns — but nothing in this file repairs it, and |
| +// its exec child is cycled by NEITHER half until somebody clears the stamp. |
| // |
| // HIBERNATE-flagged routes are included, and that is not a contradiction of |
| // "never wake a sleeper": Bounce provably cannot start anything (it is a no-op |
| @@ -176,8 +203,8 @@ func bounceExecRoutes(routes []registry.Route, moved [][2]string, bounce func(ho |
| return n |
| } |
| |
| -// restartRequestMarker is the file restartGenLoop touches to ask a root pass |
| -// to run now: carlos-restart.path watches it (PathChanged) and starts |
| +// restartRequestMarker is the file the agent's restart leg touches to ask a |
| +// root pass to run now: carlos-restart.path watches it (PathChanged) and starts |
| // carlos-restart.service (`carlos adopt -restart-only`), which owns the half |
| // this unprivileged process cannot do — restarting units — and only that half. |
| // |
| @@ -214,6 +241,26 @@ func restartMarksPath(dd string) string { return filepath.Join(dd, restartMarksF |
| // outside every character class a sqid or an app name allows, the same |
| // separator argument internal/release's bake clock makes for its own keys. |
| // |
| +// A value is either the etag this box last acted on OR restartAbsent, the |
| +// recorded "this app has no restart generation and I have looked". On disk: |
| +// |
| +// { |
| +// "schema": 1, |
| +// "apps": { |
| +// "bab|keymail": "\"9f86d0…\"", |
| +// "bab|titogo": "\u0000absent" |
| +// } |
| +// } |
| +// |
| +// The absent sentinel is what makes per-app seed-don't-replay possible. Without |
| +// it "no entry" has to mean both "never looked" and "looked, nothing there", |
| +// and one of those two has to be wrong: read as movement, a box gaining a new |
| +// route for a previously-restarted app cycles it for a request served weeks |
| +// ago (and a box whose registry lands after its marks file cycles EVERY route |
| +// at once); read as a first sight, an app's first ever request is silently |
| +// dropped. Recording the absence tells them apart — a later object sighting is |
| +// movement from a value this box wrote, which is exactly what it is. |
| +// |
| // Entries are never pruned when a route leaves the box. A vanished app's mark |
| // costs one map entry; DROPPING it would make the app's return look like a |
| // brand-new object and cycle it once for a request that was already served. |
| @@ -222,8 +269,53 @@ type restartMarks struct { |
| Apps map[string]string `json:"apps"` |
| } |
| |
| +// restartAbsent is the marks value for an app this box has SEEN and found to |
| +// have no restart-generation object at all. It is deliberately the SAME value |
| +// the in-memory sweep uses (genAbsent), so the agent's memory and the root |
| +// pass's memory cannot drift apart on what "absent" is, and it is safe for the |
| +// same reason: no store produces an etag containing a NUL byte (S3's are |
| +// hex/quoted, DirStore's is hex). JSON renders it "\u0000absent". |
| +const restartAbsent = genAbsent |
| + |
| func restartMarkKey(acct, app string) string { return acct + "|" + app } |
| |
| +// lockRestartMarks takes an exclusive flock for the whole of one restartPass. |
| +// Three root passes can run this — carlos-restart.service (the relay), |
| +// carlos-stage.service and carlos-adopt.service — and systemd will happily |
| +// start them at the same second. Unserialized, two passes read the same marks, |
| +// both dispatch (one request, two try-restarts, on a verb whose entire job is |
| +// stopping processes) and then both write, so whichever finishes last silently |
| +// reverts the other's advance and the NEXT pass restarts everything a third |
| +// time. |
| +// |
| +// The lock lives in a sibling <marks>.lock rather than on the marks file |
| +// itself, because the marks file is replaced by rename: a flock is held on an |
| +// inode, and the winner's rename would leave the loser holding a lock on a file |
| +// that is no longer the marks file. |
| +// |
| +// Opening it is ALSO the writability test the dispatch is gated on (see |
| +// restartPass): it is created in the same directory, with the same permissions |
| +// the marks save needs, so a full or read-only /data fails here — before |
| +// anything is cycled — instead of after, which is the difference between one |
| +// logged refusal and an hourly restart of every app on the box forever. |
| +// |
| +// syscall.Flock is available on both linux and darwin, which is the whole |
| +// build matrix (CI cross-compiles darwin because the CLI runs on Macs). |
| +func lockRestartMarks(path string) (unlock func(), err error) { |
| + f, err := os.OpenFile(path+".lock", os.O_CREATE|os.O_RDWR, 0o644) |
| + if err != nil { |
| + return nil, err |
| + } |
| + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { |
| + f.Close() |
| + return nil, err |
| + } |
| + return func() { |
| + syscall.Flock(int(f.Fd()), syscall.LOCK_UN) |
| + f.Close() |
| + }, nil |
| +} |
| + |
| // loadRestartMarks reads the marks file. `known` is the seed-don't-replay |
| // switch and the only subtle thing here: it is FALSE when the file is missing, |
| // unreadable, or does not parse — three different accidents with one correct |
| @@ -300,21 +392,31 @@ func (m *restartMarks) save(path string) error { |
| } |
| |
| // seedRestartMarks writes the baseline the ROOT pass compares against, ONCE, |
| -// from the agent's first sweep — create-only (O_EXCL), so it can never |
| +// from the agent's arming sweep — create-only (O_EXCL), so it can never |
| // overwrite a mark the root pass has already advanced. |
| // |
| // It exists to close a real hole in the handover between the two halves. The |
| -// agent seeds its own baseline within seconds of starting; the root pass would |
| -// otherwise not seed until it next ran, and a restart requested inside that |
| -// window would arrive at a pass with no baseline — which, by the rule above, |
| -// records and restarts nothing. The request would be silently dropped, having |
| -// told its caller "requested". Creating the file here shrinks that window to |
| -// the agent's first tick after boot, where the box has just restarted |
| -// everything anyway. |
| -// |
| -// Absent objects are NOT recorded: an app with no restart generation has |
| -// nothing to baseline, and its object's later appearance is a genuine request. |
| +// agent seeds its own baseline as it arms; the root pass would otherwise not |
| +// seed until it next ran, and a restart requested inside that window would |
| +// arrive at a pass with no baseline — which, by the rule above, records and |
| +// restarts nothing. The request would be silently dropped, having told its |
| +// caller "requested". Creating the file here shrinks that window to the moment |
| +// the agent starts, where the box has just restarted everything anyway. |
| +// |
| +// An EMPTY cache writes nothing at all. A sweep that saw no adoptable routes |
| +// is not a baseline — most often it is a box whose registry has not landed yet |
| +// (a fresh machine, or one restoring /data from litestream) — and a file |
| +// claiming "this box knows about no apps" is precisely the file that makes the |
| +// root pass treat every route it later gains as movement, and cycle the lot in |
| +// one pass. |
| +// |
| +// Absent objects ARE recorded, with restartAbsent: "looked, nothing there" is |
| +// the fact that makes an app's first ever request movement rather than a first |
| +// sight. |
| func seedRestartMarks(path string, cache map[[2]string]string) error { |
| + if len(cache) == 0 { |
| + return nil // nothing was seen; nothing may be claimed |
| + } |
| if _, err := os.Stat(path); err == nil { |
| return nil // the root pass (or an earlier agent) already has a baseline |
| } else if !errors.Is(err, os.ErrNotExist) { |
| @@ -322,9 +424,6 @@ func seedRestartMarks(path string, cache map[[2]string]string) error { |
| } |
| m := &restartMarks{Schema: 1, Apps: map[string]string{}} |
| for k, etag := range cache { |
| - if etag == genAbsent { |
| - continue |
| - } |
| m.Apps[restartMarkKey(k[0], k[1])] = etag |
| } |
| raw, err := json.MarshalIndent(m, "", " ") |
| @@ -355,13 +454,24 @@ func seedRestartMarks(path string, cache map[[2]string]string) error { |
| // |
| // The rules, each a deliberate choice: |
| // |
| -// - An app with NO restart generation is skipped entirely and records |
| -// nothing: absent is legal forever. |
| -// - An unknown baseline (see loadRestartMarks) records and dispatches |
| -// nothing. Seed-don't-replay, at the only layer that could cycle a whole |
| -// box at once. |
| -// - A missing mark for ONE app under a known baseline IS movement: the |
| -// object exists, so somebody asked, and this box has never acted on it. |
| +// - ZERO adoptable routes ends the pass before anything is read or written. |
| +// A box whose registry has not landed yet (fresh, or restoring /data) must |
| +// not be able to record "I know about no apps" — that file is what turns |
| +// every route the box later gains into movement, and cycles the whole |
| +// table in one pass. |
| +// - An app this box has NOT recorded yet is recorded and NOT cycled, |
| +// whether its object exists or not. Per-app seed-don't-replay: the marks |
| +// file is this box's memory of what it has acted on, and a request it |
| +// never saw is not one it owes anybody. |
| +// - An app with NO restart generation records restartAbsent — the sentinel |
| +// that makes its FIRST ever request (absent -> present) movement rather |
| +// than another first sight. Absent is legal forever; unrecorded absence is |
| +// what was not. |
| +// - An unknown baseline (see loadRestartMarks) records every app and |
| +// dispatches nothing — and if any app could not be READ, records nothing |
| +// at all: a baseline missing the apps a transient 503 hid is a baseline |
| +// that turns those apps into movement on the next pass. A dropped request |
| +// is recoverable (run the verb again); a restart nobody asked for is not. |
| // - HIBERNATING routes are skipped outright and not marked as a failure. |
| // Restarting a process that is not running is meaningless, waking it to |
| // restart it is worse (it would defeat hibernation for no gain), and it |
| @@ -376,34 +486,58 @@ func seedRestartMarks(path string, cache map[[2]string]string) error { |
| // retry. |
| // - The mark is written only AFTER the dispatch, so a pass that dies |
| // mid-cycle retries rather than swallowing the request. |
| +// - The whole pass holds an exclusive lock (lockRestartMarks), because three |
| +// root units can run it at once; failing to TAKE that lock dispatches |
| +// nothing, since the lock file's creation is also the proof that the marks |
| +// can be saved afterwards. A pass that cycles processes and then cannot |
| +// record what it did would cycle them again every hour, forever. |
| // |
| // Holds, bake windows and the release ladder take no part in any of this: |
| // nothing about which version runs changes here, so there is nothing for them |
| // to gate. |
| func restartPass(ctx context.Context, routes []registry.Route, store release.Store, marksPath string, restart func(host string) error) { |
| + apps := restartApps(routes) |
| + if len(apps) == 0 { |
| + return // nothing to converge, and nothing this box may claim to know |
| + } |
| + unlock, err := lockRestartMarks(marksPath) |
| + if err != nil { |
| + // No lock means either a concurrent pass we cannot serialize against or |
| + // a /data we cannot write. Both make dispatching the wrong move: the |
| + // first double-restarts, the second restarts hourly and forever because |
| + // the mark can never advance. |
| + log.Printf("adopt: restart pass: cannot lock %s.lock (%v) — nothing cycled this pass", marksPath, err) |
| + return |
| + } |
| + defer unlock() |
| + |
| marks, known := loadRestartMarks(marksPath) |
| changed := false |
| - for _, k := range restartApps(routes) { |
| + readFailed := false |
| + for _, k := range apps { |
| _, etag, err := store.Get(ctx, release.RestartGenKey(k[0], k[1])) |
| - if errors.Is(err, os.ErrNotExist) { |
| - continue // never asked for: nothing to record, nothing to do |
| - } |
| - if err != nil { |
| + switch { |
| + case errors.Is(err, os.ErrNotExist): |
| + etag = restartAbsent |
| + case err != nil: |
| log.Printf("adopt: restart generation %s/%s: %v (skipped this pass)", k[0], k[1], err) |
| + readFailed = true |
| continue |
| } |
| key := restartMarkKey(k[0], k[1]) |
| prev, seen := marks.Apps[key] |
| - if !known { |
| - // Baseline only: this box has no memory it can trust, so it |
| - // records what the bucket says and cycles nothing. |
| + if !known || !seen { |
| + // SEED. Either this box's whole memory is untrustworthy, or this is |
| + // the first time it has looked at this app — a route it has just |
| + // gained, whose process is starting fresh anyway. Record, cycle |
| + // nothing. |
| if !seen || prev != etag { |
| marks.Apps[key] = etag |
| changed = true |
| } |
| continue |
| } |
| - if seen && prev == etag { |
| + if prev == etag { |
| continue |
| } |
| dispatched := 0 |
| @@ -432,6 +566,14 @@ func restartPass(ctx context.Context, routes []registry.Route, store release.Sto |
| marks.Apps[key] = etag |
| changed = true |
| } |
| + if !known && readFailed { |
| + // A baseline that omits the apps a blip hid is worse than no baseline: |
| + // this pass would write "known", and the next one would read those |
| + // missing apps as movement and cycle them. Leave the file as it is and |
| + // re-baseline on a pass that can read everything. |
| + log.Printf("adopt: restart pass: baselining deferred — some apps could not be read this pass") |
| + return |
| + } |
| // A first, unknown baseline is written even when nothing changed: that |
| // write is what makes the NEXT pass a known one, and until it lands every |
| // pass would re-baseline and drop whatever it was asked to do. |
| @@ -442,82 +584,67 @@ func restartPass(ctx context.Context, routes []registry.Route, store release.Sto |
| } |
| } |
| |
| -// restartGenLoop is the agent's half: every ~3s, read one small object per app |
| -// this box serves and, when one moves, bounce that app's exec children here |
| -// and touch the marker that asks a root pass to cycle everything else. The |
| -// tick's whole decision lives in restartGenPass and bounceExecRoutes, which |
| -// are tested; this is the thin plumbing around them, like adoptLoop and |
| -// configGenLoop. |
| -// |
| -// Deliberately NOT folded into configGenLoop, though it is the same shape: |
| -// that loop is gated on CARLOS_SECRETS_KEY (without the key it cannot render |
| -// an app's environment at all, so it would poll to signal no one). Restarting |
| -// needs no secrets key, no release key and no signature — so gating it on one |
| -// would mean a box that happens to carry no secrets key silently has no |
| -// restart verb, and the person typing `carlos restart` would be told |
| -// "requested" by a console that was right, about a box that would never act. |
| -// |
| -// Deliberately NOT doing here: |
| -// |
| -// - No fast first pass. The loop sleeps BEFORE its first tick and the first |
| -// sight of every app is silent (see restartGenPass): a box coming up must |
| -// not cycle everything it serves, and it has in any case just started all |
| -// of it. |
| -// - No unit restarting, and no marks bookkeeping. This process writes no |
| -// /etc and runs no systemctl: for anything unit-shaped it only touches |
| -// <dataDir>/restart-requested, which carlos-restart.path relays into a |
| -// root `carlos adopt -restart-only` run. That pass owns the marks file, |
| -// and this loop's in-memory cache is deliberately a SECOND, independent |
| -// memory — the two halves cycle disjoint sets of routes, so neither can |
| -// mark the other's work done. |
| -func restartGenLoop(reg *registry.Registry, svc *activator.Service, dd string) { |
| - ctx := context.Background() |
| - store, err := openDeployStore(ctx) |
| +// restartRequestLeg is the agent's half of one serial move: compare each app's |
| +// restart generation against the in-memory cache, bounce this process's own |
| +// exec children for the apps that moved, and touch the marker that asks a root |
| +// pass to cycle everything unit-shaped. Reports whether it asked. |
| +// |
| +// It runs ONLY when the serial moved (fastTick), which is what makes the |
| +// feature free at rest: the console bumps the serial after touching the |
| +// object, so every request is announced, and an idle box reads no restart |
| +// generation at all. There is no loop of its own and there must not be one — |
| +// see this file's package comment for what the 3s version of this cost. |
| +// |
| +// The bounce goes first, and not only for tidiness: it is in-process and |
| +// instant, while the marker write can block on a busy /data, and the children |
| +// bounced here are the ones cycled here or nowhere. |
| +func restartRequestLeg(ctx context.Context, routes []registry.Route, store release.Store, dd string, |
| + bounce func(host string) bool, cache map[[2]string]string) bool { |
| + moved, readErrs := restartGenPass(ctx, routes, store, cache) |
| + if len(readErrs) > 0 { |
| + // ONE line per pass, never one per app: the count carries the scale, |
| + // the first error the cause — they are the same cause in practice. |
| + log.Printf("agent: restart check: %d apps unreadable: %v", len(readErrs), readErrs[0]) |
| + } |
| + if len(moved) == 0 { |
| + return false |
| + } |
| + bounceExecRoutes(routes, moved, bounce) |
| + if err := requestRootRestart(dd); err != nil { |
| + log.Printf("agent: request root restart pass: %v", err) |
| + return false |
| + } |
| + log.Printf("agent: restart requested for %d app(s) on this box, asked for a root restart pass", len(moved)) |
| + return true |
| +} |
| + |
| +// restartSeed is the ONE sweep this agent makes on its own initiative: at |
| +// arming, read each app's restart generation once, silently, so the cache the |
| +// leg compares against exists before the first serial move — and hand the same |
| +// sightings to seedRestartMarks so the root pass has a baseline too. |
| +// |
| +// Without it the first request after a boot is the sweep that establishes the |
| +// baseline, and is therefore the one request that does nothing: the person who |
| +// typed `carlos restart` would have to type it twice, having been told |
| +// "requested" the first time. One read per app per agent start is the whole |
| +// cost, and it is paid where a box has just restarted everything anyway. |
| +// |
| +// A read error leaves that app out of the cache (its next sighting is a first |
| +// sight — safe) and blocks the marks seed entirely, for the reason |
| +// seedRestartMarks and restartPass both give: a baseline with holes in it is |
| +// the thing that manufactures restarts. |
| +func restartSeed(ctx context.Context, reg *registry.Registry, store release.Store, dd string, cache map[[2]string]string) { |
| + routes, err := reg.List() |
| if err != nil { |
| - log.Printf("agent: restart request loop disabled: %v", err) |
| + log.Printf("agent: restart baseline: list routes: %v (the first request may need a second try)", err) |
| return |
| } |
| - bounce := bounceFn(svc) |
| - // One cache for the life of the process: what "changed" means is exactly |
| - // "differs from the last tick that could read it". |
| - cache := map[[2]string]string{} |
| - seeded := false |
| - log.Printf("agent: restart request loop armed (3s + jitter)") |
| - for { |
| - time.Sleep(3*time.Second + rand.N(time.Second)) |
| - routes, err := reg.List() |
| - if err != nil { |
| - log.Printf("agent: restart poll: list routes: %v", err) |
| - continue |
| - } |
| - moved, readErrs := restartGenPass(ctx, routes, store, cache) |
| - if len(readErrs) > 0 { |
| - // ONE line per pass, never one per app: a bucket having a bad |
| - // minute would otherwise flood the journal at 20 lines a minute per |
| - // app. The count carries the scale, the first error the cause. |
| - log.Printf("agent: restart poll: %d apps unreadable: %v", len(readErrs), readErrs[0]) |
| - } |
| - if !seeded && len(readErrs) == 0 { |
| - // After the first CLEAN pass, whose sightings are the baseline by |
| - // definition. A pass that could not read some app is not a |
| - // baseline: seeding from it would leave that app unrecorded, and |
| - // an unrecorded app under a known baseline reads as a request. |
| - // Create-only; see seedRestartMarks. |
| - if err := seedRestartMarks(restartMarksPath(dd), cache); err != nil { |
| - log.Printf("agent: seed restart marks: %v", err) |
| - } |
| - seeded = true |
| - } |
| - if len(moved) == 0 { |
| - continue |
| - } |
| - // This process's own children first — they are cycled here or nowhere, |
| - // and the write below can block on a busy /data. |
| - bounceExecRoutes(routes, moved, bounce) |
| - if err := requestRootRestart(dd); err != nil { |
| - log.Printf("agent: request root restart pass: %v", err) |
| - continue |
| - } |
| - log.Printf("agent: restart requested for %d app(s) on this box, asked for a root restart pass", len(moved)) |
| + _, readErrs := restartGenPass(ctx, routes, store, cache) |
| + if len(readErrs) > 0 { |
| + log.Printf("agent: restart baseline: %d apps unreadable: %v", len(readErrs), readErrs[0]) |
| + return |
| + } |
| + if err := seedRestartMarks(restartMarksPath(dd), cache); err != nil { |
| + log.Printf("agent: seed restart marks: %v", err) |
| } |
| } |
amadan