amadan
host: a moved restart generation cycles that app's processes
The box half of #113, split exactly the way config fast convergence is. The agent sweeps each app's restart-gen etag every ~3s, bounces its OWN exec children (systemctl cannot reach a child of another process, and a warm exec instance is what a wedged instance usually is), and touches /data/restart-requested for everything unit-shaped. carlos-restart.path relays that to a root 'carlos adopt -restart-only', which compares each app's etag against /data/restart-marks.json and dispatches rootRestartRoute per non-hibernating route — including the bespoke units addr-backed routes name (#110's consoles). SEED-DON'T-REPLAY on both halves: a first sight, and a marks file that is missing, unreadable or unparsable, record the baseline and restart NOTHING. That direction is the opposite of the bake clock's on purpose — restarting is the disruptive act here — and the agent creates the marks file (create-only, O_EXCL) on its first clean pass so the handover cannot silently drop the first request. A missing mark for ONE app under a known baseline is movement, or every app's first ever restart would be the one that never works. Failed dispatches advance the mark: a request is a point-in-time event, not a standing order. The full hourly pass runs the same comparison as the floor under the relay, BEFORE release.Pass (which is fatal on error, and a wedged instance is most worth cycling on a box where something else is already wrong). -config-only stays out of it.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
7 files changed,
+1116
−5
internal/host/agent.go+49 −1internal/host/bootstrap.go+6 −4internal/host/host_test.go+2 −0internal/host/restart.go+523 −0internal/host/restart_test.go+456 −0internal/host/units/carlos-restart.path+27 −0internal/host/units/carlos-restart.service+53 −0
diff --git a/internal/host/agent.go b/internal/host/agent.go| index 8b6928fb7..36d9e9d4a 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -165,6 +165,7 @@ func RunAgent(args []string) { |
| go metricsLoop() |
| go configSyncLoop(reg, svc, dd) |
| go configGenLoop(reg, svc, dd) |
| + go restartGenLoop(reg, svc, dd) |
| go socketAdvisoryLoop(reg, svc) |
| // The same serving loop as `carlos edge`, with the activator injected. |
| @@ -1689,11 +1690,22 @@ func openDeployStore(ctx context.Context) (release.Store, error) { |
| // seconds of an unrelated write, and would fire the canary-TTL nag per |
| // config write instead of hourly — the "hourly root pass alone" split |
| // release/adopt.go and the nag comment below both depend on. |
| +// |
| +// -restart-only runs the RESTART-REQUEST half alone (issue #113): compare each |
| +// app's restart generation against this box's marks and cycle the ones that |
| +// moved, nothing else. That is what carlos-restart.service execs, and it is |
| +// separated from -config-only for the same reason -config-only is separated |
| +// from the full pass: the trigger behind it is a per-request path unit, and a |
| +// request to cycle one app's processes must not stage a release, run the |
| +// capability converge, or nag about canaries. It deliberately needs NO release |
| +// key and NO secrets key — restarting verifies no signature and renders no |
| +// config — so a box missing either still honours it. |
| func RunAdopt(args []string) { |
| fs := flag.NewFlagSet("adopt", flag.ExitOnError) |
| regPath := fs.String("registry", filepath.Join(dataDir(), "registry.db"), "registry database") |
| configOnly := fs.Bool("config-only", false, "sync unit-backed config and restart changed routes only: no release adoption, no canary nag") |
| stageOnly := fs.Bool("stage-only", false, "adopt releases without the canary nag: the agent-requested pass (carlos-stage.service)") |
| + restartOnly := fs.Bool("restart-only", false, "cycle the routes of apps whose restart generation moved, and nothing else: the agent-requested pass (carlos-restart.service)") |
| _ = fs.Parse(args) |
| reg, err := registry.Open(*regPath) |
| if err != nil { |
| @@ -1717,6 +1729,24 @@ func RunAdopt(args []string) { |
| // construction. It is handed the same callback anyway because there should |
| // be ONE restart dispatch on this pass, not two that could drift. |
| restart := rootRestartRoute(reg) |
| + // The restart-request half, and the ONE place it dispatches from — |
| + // including for exec-backed routes, whose children belong to the agent: |
| + // rootRestartRoute's try-restart arm reaches a carlos-app@ unit, and an |
| + // exec child that this pass cannot cycle is cycled by nothing else, which |
| + // is a real limit of the -restart-only path recorded in restartPass. |
| + // |
| + // -restart-only returns here: no config sync, no capability converge, no |
| + // release.Pass, no nag, and — above all — no release public key, which is |
| + // parsed below and is fatal when unusable. A restart request must not fail |
| + // on a key it never needs. |
| + if *restartOnly { |
| + routes, err := reg.List() |
| + if err != nil { |
| + log.Fatalf("adopt: list routes: %v", err) |
| + } |
| + restartPass(ctx, routes, store, restartMarksPath(dataDir()), restart) |
| + return |
| + } |
| // Parsed here, before config sync, exactly as the full pass always has |
| // (a box with an unusable release key must die before it converges |
| // anything), but SKIPPED entirely for -config-only: that mode never |
| @@ -1745,10 +1775,28 @@ func RunAdopt(args []string) { |
| if *configOnly { |
| // Everything below is release adoption and its reporting — the half |
| // carlos-config-sync.service must never reach, however often a |
| - // config write triggers it. |
| + // config write triggers it. The restart half is below for the same |
| + // reason: a config write is not a restart request, and running the |
| + // comparison here would only move it earlier by minutes at the cost of |
| + // firing on a trigger that has nothing to do with it. |
| return |
| } |
| + // The restart-request half again (issue #113), here as the FLOOR under |
| + // carlos-restart.path: that relay can drop an event that lands while the |
| + // service is mid-run, exactly as the config one can, and a dropped restart |
| + // with no floor would be a request that never happens at all. |
| + // |
| + // BEFORE release.Pass deliberately. That call is fatal on error, so a box |
| + // with an unstageable release would otherwise never reach this line — and |
| + // a wedged instance is most worth cycling precisely on a box where |
| + // something else is already wrong. |
| + if routes, err := reg.List(); err != nil { |
| + log.Printf("adopt: list routes for the restart pass: %v", err) |
| + } else { |
| + restartPass(ctx, routes, store, restartMarksPath(dataDir()), restart) |
| + } |
| + |
| // Step 2 (spec §4d): the grant-drift capability converge, BETWEEN config |
| // sync and release adoption. Before release.Pass because it repairs the |
| // shape of what is ALREADY live — a revoked grant, a deleted drop-in, a |
diff --git a/internal/host/bootstrap.go b/internal/host/bootstrap.go| index 1baf02394..f5bb7765a 100644 |
| --- a/internal/host/bootstrap.go |
| +++ b/internal/host/bootstrap.go |
| @@ -58,14 +58,16 @@ const tmpfilesConf = "d /run/carlos 2750 " + serviceUser + " " + socketsGroup + |
| "d /run/carlos/apps 0750 " + serviceUser + " " + serviceUser + " -\n" |
| // enabledUnits are what a live bootstrap turns on: the always-on front door, |
| -// the hourly adoption timer, and the two path units that let the sandboxed |
| +// the hourly adoption timer, and the three path units that let the sandboxed |
| // agent ask a root pass for the halves it cannot do itself: carlos-adopt.path |
| -// (config fast convergence, issue #44) and carlos-stage.path (release staging |
| -// — the agent sees a promotion in seconds but cannot write /opt). The litestream + restore-verify units install |
| +// (config fast convergence, issue #44), carlos-stage.path (release staging |
| +// — the agent sees a promotion in seconds but cannot write /opt) and |
| +// carlos-restart.path (a member's `carlos restart`, issue #113 — the agent |
| +// bounces its own exec children but cannot cycle a unit). The litestream + restore-verify units install |
| // everywhere but stay inert until an operator stages /data/litestream.env and |
| // generates /etc/litestream.yml (`carlos ops litestream`) — a box with no |
| // replication bucket is unaffected. |
| -var enabledUnits = []string{"carlos-edge.service", "carlos-adopt.timer", "carlos-adopt.path", "carlos-stage.path"} |
| +var enabledUnits = []string{"carlos-edge.service", "carlos-adopt.timer", "carlos-adopt.path", "carlos-stage.path", "carlos-restart.path"} |
| // systemctlPlan is the exact systemctl sequence a live bootstrap runs — and |
| // the same sequence a staging run (-root <dir>) PRINTS instead of running, so |
diff --git a/internal/host/host_test.go b/internal/host/host_test.go| index e432768fc..52c7d1a4e 100644 |
| --- a/internal/host/host_test.go |
| +++ b/internal/host/host_test.go |
| @@ -33,6 +33,8 @@ var expectedUnits = []string{ |
| "carlos-config-sync.service", |
| "carlos-edge.service", |
| "carlos-litestream.service", |
| + "carlos-restart.path", |
| + "carlos-restart.service", |
| "carlos-restore-verify.service", |
| "carlos-restore-verify.timer", |
| "carlos-stage.path", |
diff --git a/internal/host/restart.go b/internal/host/restart.go| new file mode 100644 |
| index 000000000..0f206b725 |
| --- /dev/null |
| +++ b/internal/host/restart.go |
| @@ -0,0 +1,523 @@ |
| +// The box half of `carlos restart` (issue #113): watch each app's |
| +// restart-generation object (release.RestartGenKey) and, when one MOVES, |
| +// cycle that app's live processes on this box. Pull-based like everything |
| +// else here — no push channel, no box access, no version or config change |
| +// involved. |
| +// |
| +// The shape is config fast convergence's, one step sideways, and it is split |
| +// across the same two processes for the same reason: |
| +// |
| +// - the AGENT (carlos-edge.service, User=carlos, ProtectSystem=strict) can |
| +// read the bucket and see a request within seconds, and it owns the |
| +// EXEC-backed instances outright — so its half is the etag sweep, a bounce |
| +// of its own children, and one marker file for the rest. |
| +// - the ROOT pass (carlos adopt, out of carlos-restart.service via |
| +// carlos-restart.path, or the hourly timer) is the only one that can |
| +// restart a systemd unit — the carlos-app@ tenants and the bespoke units |
| +// addr-backed routes name (issue #110's consoles). Its half is the |
| +// comparison against a persisted high-water mark and the dispatch. |
| +// |
| +// The split is exactly config fast convergence's: systemctl cannot reach a |
| +// child of another process, so a root-only dispatch would leave every warm |
| +// exec instance — the commonest shape on the box, and the one a wedged |
| +// instance usually is — serving on, unrestarted. |
| +// |
| +// SEED-DON'T-REPLAY governs both halves. A box that has never recorded a |
| +// baseline records one and restarts NOTHING: the first sight of an object is |
| +// never a request, or every upgrade would bounce the whole box. |
| +package host |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "fmt" |
| + "log" |
| + "math/rand/v2" |
| + "os" |
| + "path/filepath" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/activator" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// restartApps lists the distinct (account, app) pairs with at least one |
| +// restartable route on this box — the apps whose restart generation is worth |
| +// one small GET per tick. |
| +// |
| +// The membership predicate is release.Adoptable, deliberately the SAME one the |
| +// adopt pass moves and the fast sweep counts: a route a pass would not adopt |
| +// is not one whose process this feature has any business cycling, and a second |
| +// copy of that judgement here is exactly the drift that would let the two |
| +// disagree. It admits addr-backed routes that name their own unit (issue |
| +// #110's consoles) — which is half of why #113 exists — because Adoptable |
| +// filters on lifecycle and app, never on backing. |
| +func restartApps(routes []registry.Route) [][2]string { |
| + seen := map[[2]string]bool{} |
| + var out [][2]string |
| + for _, rt := range routes { |
| + if !release.Adoptable(rt) { |
| + continue |
| + } |
| + k := [2]string{release.RouteAccount(rt), rt.App} |
| + if seen[k] { |
| + continue |
| + } |
| + seen[k] = true |
| + out = append(out, k) |
| + } |
| + return out |
| +} |
| + |
| +// restartGenPass is one tick of restartGenLoop, split out for tests: read the |
| +// restart-generation etag of every app this box serves and report WHICH of |
| +// them moved. Same semantics — and the same reasons — as genPollPass: |
| +// |
| +// - First sight of an app records silently and asks for nothing. A box |
| +// coming up must not restart everything it serves because it has never |
| +// seen these etags before; that is the seed-don't-replay rule, and it is |
| +// the whole safety property of this feature. |
| +// - An unchanged etag does nothing. |
| +// - Absent that stays absent does nothing, FOREVER: an app nobody has ever |
| +// restarted this way has no object and never will until someone asks. |
| +// That is the ordinary state of almost every app, not a fault. |
| +// - Any movement asks: the etag changed, or absent -> present (an app's |
| +// first ever restart request). present -> absent should not happen |
| +// (restart generations are only created and overwritten, never deleted), |
| +// but it counts as movement too — one extra root pass is cheap, and |
| +// pretending we did not see it would be a lie about the only thing this |
| +// function reports. |
| +// - The moved apps come back as a LIST, but the root request is one marker |
| +// touch however long that list is: the root pass re-reads every app's etag |
| +// itself and dispatches per app, so asking once per moved app would only |
| +// ask the same pass to run several times. The list is for the caller's own |
| +// half — which of ITS exec children to bounce. |
| +// - A read error that is not "not found" leaves that app's cache entry |
| +// exactly as it was and keeps evaluating the rest — the errors come back |
| +// for the caller to aggregate into ONE log line per pass. Clearing the |
| +// entry would turn a recovered blip into a bogus first sight or a bogus |
| +// restart. |
| +// - Apps that vanish from the route set lose their cache entries: a re-added |
| +// app is a first sight again, which is right, because a route that has |
| +// just been (re-)minted is starting its process fresh anyway. |
| +// |
| +// The cache is mutated in place — it lives across ticks in the loop, which is |
| +// the whole point (there is nowhere else to remember what "changed" means). |
| +func restartGenPass(ctx context.Context, routes []registry.Route, store release.Store, cache map[[2]string]string) (moved [][2]string, readErrs []error) { |
| + apps := restartApps(routes) |
| + live := make(map[[2]string]bool, len(apps)) |
| + for _, k := range apps { |
| + live[k] = true |
| + _, etag, err := store.Get(ctx, release.RestartGenKey(k[0], k[1])) |
| + switch { |
| + case errors.Is(err, os.ErrNotExist): |
| + etag = genAbsent |
| + case err != nil: |
| + readErrs = append(readErrs, fmt.Errorf("%s/%s: %w", k[0], k[1], err)) |
| + continue // cache entry untouched: this app is unknown this tick, not changed |
| + } |
| + prev, seen := cache[k] |
| + cache[k] = etag |
| + if seen && prev != etag { |
| + moved = append(moved, k) |
| + } |
| + } |
| + for k := range cache { |
| + if !live[k] { |
| + delete(cache, k) |
| + } |
| + } |
| + return moved, readErrs |
| +} |
| + |
| +// bounceExecRoutes cycles the agent's OWN children for the apps whose restart |
| +// generation just moved, and returns how many it actually stopped. This is the |
| +// half no root pass can perform: an exec instance is a child of THIS process, |
| +// and systemctl has no name for it. |
| +// |
| +// The route filter is the complement of what rootRestartRoute can reach — no |
| +// named unit, no unit backing — so the two halves cannot cycle the same |
| +// process twice, and every restartable route is covered by exactly one of |
| +// them. |
| +// |
| +// HIBERNATE-flagged routes are included, and that is not a contradiction of |
| +// "never wake a sleeper": Bounce provably cannot start anything (it is a no-op |
| +// unless the instance is AWAKE), so a sleeping instance is untouched and picks |
| +// its next start up naturally, while a WARM hibernator — which is what a |
| +// wedged instance usually is — gets exactly the cycle that was asked for. This |
| +// is release.RestartHibernating's argument, in the one process that can act on |
| +// it. |
| +// |
| +// A nil bounce (no activator armed on this box) is a working no-op: there are |
| +// no children to cycle, and the unit half still runs. |
| +func bounceExecRoutes(routes []registry.Route, moved [][2]string, bounce func(host string) bool) int { |
| + if bounce == nil || len(moved) == 0 { |
| + return 0 |
| + } |
| + want := make(map[[2]string]bool, len(moved)) |
| + for _, k := range moved { |
| + want[k] = true |
| + } |
| + n := 0 |
| + for _, rt := range routes { |
| + if !release.Adoptable(rt) || rt.Unit != "" || rt.Backing == registry.BackingUnit { |
| + continue |
| + } |
| + if !want[[2]string{release.RouteAccount(rt), rt.App}] { |
| + continue |
| + } |
| + if bounce(rt.Host) { |
| + n++ |
| + log.Printf("agent: restart requested: bounced exec instance %s", rt.Host) |
| + } |
| + } |
| + return n |
| +} |
| + |
| +// restartRequestMarker is the file restartGenLoop touches to ask a root pass |
| +// to run now: carlos-restart.path watches it (PathChanged) and starts |
| +// carlos-restart.service (`carlos adopt -restart-only`), which owns the half |
| +// this unprivileged process cannot do — restarting units — and only that half. |
| +// |
| +// A sibling of configSyncMarker rather than a reuse of it, because the two |
| +// relays land on different services: config's target is `carlos adopt |
| +// -config-only`, which renders /etc/carlos/apps and restarts only the routes |
| +// whose env file actually changed. A restart request changes no file, so that |
| +// pass would find nothing to do and cycle nothing at all. |
| +// |
| +// It lives in the data dir, deliberately outside the *.config.env namespace |
| +// reapConfigEnv sweeps — the reaper matches that suffix alone and must go on |
| +// ignoring this. |
| +const restartRequestMarker = "restart-requested" |
| + |
| +// requestRootRestart touches the marker. The content is a timestamp only so an |
| +// operator reading /data can see when the last request went out; systemd |
| +// watches the path, not the bytes. Errors are the caller's to log and swallow: |
| +// a request that cannot be written costs latency, not correctness, because |
| +// carlos-adopt.timer's hourly pass runs the same comparison underneath it. |
| +func requestRootRestart(dd string) error { |
| + return os.WriteFile(filepath.Join(dd, restartRequestMarker), |
| + []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644) |
| +} |
| + |
| +// restartMarksFile is the ROOT pass's memory: the last restart generation it |
| +// has already acted on, per app. It lives in the data dir beside the bake |
| +// clock, in the same JSON + temp-file-rename shape and for the same |
| +// cross-uid reason (see restartMarks.save). |
| +const restartMarksFile = "restart-marks.json" |
| + |
| +func restartMarksPath(dd string) string { return filepath.Join(dd, restartMarksFile) } |
| + |
| +// restartMarks is that memory. Apps is keyed "<account>|<app>" — "|" is |
| +// outside every character class a sqid or an app name allows, the same |
| +// separator argument internal/release's bake clock makes for its own keys. |
| +// |
| +// Entries are never pruned when a route leaves the box. A vanished app's mark |
| +// costs one map entry; DROPPING it would make the app's return look like a |
| +// brand-new object and cycle it once for a request that was already served. |
| +type restartMarks struct { |
| + Schema int `json:"schema"` // 1 |
| + Apps map[string]string `json:"apps"` |
| +} |
| + |
| +func restartMarkKey(acct, app string) string { return acct + "|" + app } |
| + |
| +// loadRestartMarks reads the marks file. `known` is the seed-don't-replay |
| +// switch and the only subtle thing here: it is FALSE when the file is missing, |
| +// unreadable, or does not parse — three different accidents with one correct |
| +// response, which is to record what the bucket currently says and restart |
| +// NOTHING this pass. |
| +// |
| +// That direction is the opposite of the bake clock's ("an unreadable clock |
| +// fails toward baking"), and deliberately so: baking is the cautious act there, |
| +// while restarting is the DISRUPTIVE act here. The cost of failing this way is |
| +// that a box whose marks file was just wiped can drop ONE restart request — |
| +// logged, and re-runnable by the person who typed it. The cost of failing the |
| +// other way is a box that cycles every app it serves the moment its state file |
| +// goes missing. |
| +func loadRestartMarks(path string) (marks *restartMarks, known bool) { |
| + m := &restartMarks{Schema: 1, Apps: map[string]string{}} |
| + raw, err := os.ReadFile(path) |
| + if err != nil { |
| + if !errors.Is(err, os.ErrNotExist) { |
| + log.Printf("adopt: restart marks %s unreadable (%v) — re-baselining, no restarts this pass", path, err) |
| + } |
| + return m, false |
| + } |
| + if err := json.Unmarshal(raw, m); err != nil { |
| + log.Printf("adopt: restart marks %s do not parse (%v) — re-baselining, no restarts this pass", path, err) |
| + return &restartMarks{Schema: 1, Apps: map[string]string{}}, false |
| + } |
| + if m.Apps == nil { |
| + m.Apps = map[string]string{} |
| + } |
| + m.Schema = 1 |
| + return m, true |
| +} |
| + |
| +// save persists the marks via temp-file-in-the-same-dir + rename, the |
| +// co-written-file pattern release.FirstSeen.Save documents at length: /data is |
| +// written by BOTH uids on this box (carlos for the agent's seed below, root |
| +// for this pass), and an in-place O_TRUNC write needs write permission on the |
| +// existing file, so whichever uid created it would lock the other out forever. |
| +// rename(2) only needs write permission on the directory, which both have. The |
| +// temp name is minted fresh per attempt so a crash between write and rename |
| +// can never leave cross-uid residue for the next save to trip over, and the |
| +// 0600 CreateTemp mints is widened to 0644 BEFORE the rename so the other uid |
| +// can still read it. |
| +func (m *restartMarks) save(path string) error { |
| + raw, err := json.MarshalIndent(m, "", " ") |
| + if err != nil { |
| + return err |
| + } |
| + raw = append(raw, '\n') |
| + f, err := os.CreateTemp(filepath.Dir(path), "restart-marks-*.json") |
| + if err != nil { |
| + return err |
| + } |
| + tmp := f.Name() |
| + _, writeErr := f.Write(raw) |
| + closeErr := f.Close() |
| + if writeErr != nil { |
| + os.Remove(tmp) |
| + return writeErr |
| + } |
| + if closeErr != nil { |
| + os.Remove(tmp) |
| + return closeErr |
| + } |
| + if err := os.Chmod(tmp, 0o644); err != nil { |
| + os.Remove(tmp) |
| + return err |
| + } |
| + if err := os.Rename(tmp, path); err != nil { |
| + os.Remove(tmp) // don't leave a stray temp file behind |
| + return err |
| + } |
| + return nil |
| +} |
| + |
| +// seedRestartMarks writes the baseline the ROOT pass compares against, ONCE, |
| +// from the agent's first sweep — create-only (O_EXCL), so it can never |
| +// overwrite a mark the root pass has already advanced. |
| +// |
| +// It exists to close a real hole in the handover between the two halves. The |
| +// agent seeds its own baseline within seconds of starting; the root pass would |
| +// otherwise not seed until it next ran, and a restart requested inside that |
| +// window would arrive at a pass with no baseline — which, by the rule above, |
| +// records and restarts nothing. The request would be silently dropped, having |
| +// told its caller "requested". Creating the file here shrinks that window to |
| +// the agent's first tick after boot, where the box has just restarted |
| +// everything anyway. |
| +// |
| +// Absent objects are NOT recorded: an app with no restart generation has |
| +// nothing to baseline, and its object's later appearance is a genuine request. |
| +func seedRestartMarks(path string, cache map[[2]string]string) error { |
| + if _, err := os.Stat(path); err == nil { |
| + return nil // the root pass (or an earlier agent) already has a baseline |
| + } else if !errors.Is(err, os.ErrNotExist) { |
| + return err |
| + } |
| + m := &restartMarks{Schema: 1, Apps: map[string]string{}} |
| + for k, etag := range cache { |
| + if etag == genAbsent { |
| + continue |
| + } |
| + m.Apps[restartMarkKey(k[0], k[1])] = etag |
| + } |
| + raw, err := json.MarshalIndent(m, "", " ") |
| + if err != nil { |
| + return err |
| + } |
| + raw = append(raw, '\n') |
| + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) |
| + if err != nil { |
| + if errors.Is(err, os.ErrExist) { |
| + return nil // lost the race to the root pass; its mark is the better one |
| + } |
| + return err |
| + } |
| + _, writeErr := f.Write(raw) |
| + if err := f.Close(); err != nil && writeErr == nil { |
| + writeErr = err |
| + } |
| + return writeErr |
| +} |
| + |
| +// restartPass is the ROOT half: for every app on this box whose restart |
| +// generation has moved since the mark this box recorded, cycle that app's |
| +// routes. Called from RunAdopt — on the marker-triggered `-restart-only` run |
| +// (seconds after the request) and on the ordinary pass (the hourly floor |
| +// underneath the relay, which can drop an event that lands while the service |
| +// is mid-run). |
| +// |
| +// The rules, each a deliberate choice: |
| +// |
| +// - An app with NO restart generation is skipped entirely and records |
| +// nothing: absent is legal forever. |
| +// - An unknown baseline (see loadRestartMarks) records and dispatches |
| +// nothing. Seed-don't-replay, at the only layer that could cycle a whole |
| +// box at once. |
| +// - A missing mark for ONE app under a known baseline IS movement: the |
| +// object exists, so somebody asked, and this box has never acted on it. |
| +// - HIBERNATING routes are skipped outright and not marked as a failure. |
| +// Restarting a process that is not running is meaningless, waking it to |
| +// restart it is worse (it would defeat hibernation for no gain), and it |
| +// will exec fresh on its next wake regardless — which is the restart. A |
| +// WARM hibernating exec instance is not left serving, though: the agent's |
| +// own bounce (bounceExecRoutes) reaches it, in the one process that can. |
| +// - A dispatch FAILURE still advances the mark, and logs. A restart request |
| +// is a point-in-time event, not a desired state: a named unit that exits |
| +// non-zero (a wrong unit name, a service that will not come back) would |
| +// otherwise be retried on every pass, forever, for a request somebody made |
| +// once. The journal line is the report; `carlos restart` again is the |
| +// retry. |
| +// - The mark is written only AFTER the dispatch, so a pass that dies |
| +// mid-cycle retries rather than swallowing the request. |
| +// |
| +// Holds, bake windows and the release ladder take no part in any of this: |
| +// nothing about which version runs changes here, so there is nothing for them |
| +// to gate. |
| +func restartPass(ctx context.Context, routes []registry.Route, store release.Store, marksPath string, restart func(host string) error) { |
| + marks, known := loadRestartMarks(marksPath) |
| + changed := false |
| + for _, k := range restartApps(routes) { |
| + _, etag, err := store.Get(ctx, release.RestartGenKey(k[0], k[1])) |
| + if errors.Is(err, os.ErrNotExist) { |
| + continue // never asked for: nothing to record, nothing to do |
| + } |
| + if err != nil { |
| + log.Printf("adopt: restart generation %s/%s: %v (skipped this pass)", k[0], k[1], err) |
| + continue |
| + } |
| + key := restartMarkKey(k[0], k[1]) |
| + prev, seen := marks.Apps[key] |
| + if !known { |
| + // Baseline only: this box has no memory it can trust, so it |
| + // records what the bucket says and cycles nothing. |
| + if !seen || prev != etag { |
| + marks.Apps[key] = etag |
| + changed = true |
| + } |
| + continue |
| + } |
| + if seen && prev == etag { |
| + continue |
| + } |
| + dispatched := 0 |
| + for _, rt := range routes { |
| + if !release.Adoptable(rt) || rt.App != k[1] || release.RouteAccount(rt) != k[0] { |
| + continue |
| + } |
| + if rt.Hibernate { |
| + log.Printf("adopt: restart requested for %s/%s: %s is hibernating — left asleep (it execs fresh on its next wake)", |
| + k[0], k[1], rt.Host) |
| + continue |
| + } |
| + if err := restart(rt.Host); err != nil { |
| + log.Printf("adopt: restart requested for %s/%s: %s did not cycle: %v", k[0], k[1], rt.Host, err) |
| + continue |
| + } |
| + dispatched++ |
| + } |
| + // One line per APP, not per route, and it says DISPATCHED rather than |
| + // cycled: an exec-backed route reaching rootRestartRoute lands on a |
| + // try-restart of a carlos-app@ unit that does not exist — a deliberate |
| + // no-op (the agent bounced that child itself) which this pass cannot |
| + // tell apart from a real one. Claiming "cycled" for it would be the |
| + // kind of lie issue #110 was made of. |
| + log.Printf("adopt: restart requested for %s/%s: dispatched to %d route(s)", k[0], k[1], dispatched) |
| + marks.Apps[key] = etag |
| + changed = true |
| + } |
| + // A first, unknown baseline is written even when nothing changed: that |
| + // write is what makes the NEXT pass a known one, and until it lands every |
| + // pass would re-baseline and drop whatever it was asked to do. |
| + if changed || !known { |
| + if err := marks.save(marksPath); err != nil { |
| + log.Printf("adopt: save restart marks %s: %v (the next pass may repeat a restart)", marksPath, err) |
| + } |
| + } |
| +} |
| + |
| +// restartGenLoop is the agent's half: every ~3s, read one small object per app |
| +// this box serves and, when one moves, bounce that app's exec children here |
| +// and touch the marker that asks a root pass to cycle everything else. The |
| +// tick's whole decision lives in restartGenPass and bounceExecRoutes, which |
| +// are tested; this is the thin plumbing around them, like adoptLoop and |
| +// configGenLoop. |
| +// |
| +// Deliberately NOT folded into configGenLoop, though it is the same shape: |
| +// that loop is gated on CARLOS_SECRETS_KEY (without the key it cannot render |
| +// an app's environment at all, so it would poll to signal no one). Restarting |
| +// needs no secrets key, no release key and no signature — so gating it on one |
| +// would mean a box that happens to carry no secrets key silently has no |
| +// restart verb, and the person typing `carlos restart` would be told |
| +// "requested" by a console that was right, about a box that would never act. |
| +// |
| +// Deliberately NOT doing here: |
| +// |
| +// - No fast first pass. The loop sleeps BEFORE its first tick and the first |
| +// sight of every app is silent (see restartGenPass): a box coming up must |
| +// not cycle everything it serves, and it has in any case just started all |
| +// of it. |
| +// - No unit restarting, and no marks bookkeeping. This process writes no |
| +// /etc and runs no systemctl: for anything unit-shaped it only touches |
| +// <dataDir>/restart-requested, which carlos-restart.path relays into a |
| +// root `carlos adopt -restart-only` run. That pass owns the marks file, |
| +// and this loop's in-memory cache is deliberately a SECOND, independent |
| +// memory — the two halves cycle disjoint sets of routes, so neither can |
| +// mark the other's work done. |
| +func restartGenLoop(reg *registry.Registry, svc *activator.Service, dd string) { |
| + ctx := context.Background() |
| + store, err := openDeployStore(ctx) |
| + if err != nil { |
| + log.Printf("agent: restart request loop disabled: %v", err) |
| + return |
| + } |
| + bounce := bounceFn(svc) |
| + // One cache for the life of the process: what "changed" means is exactly |
| + // "differs from the last tick that could read it". |
| + cache := map[[2]string]string{} |
| + seeded := false |
| + log.Printf("agent: restart request loop armed (3s + jitter)") |
| + for { |
| + time.Sleep(3*time.Second + rand.N(time.Second)) |
| + routes, err := reg.List() |
| + if err != nil { |
| + log.Printf("agent: restart poll: list routes: %v", err) |
| + continue |
| + } |
| + moved, readErrs := restartGenPass(ctx, routes, store, cache) |
| + if len(readErrs) > 0 { |
| + // ONE line per pass, never one per app: a bucket having a bad |
| + // minute would otherwise flood the journal at 20 lines a minute per |
| + // app. The count carries the scale, the first error the cause. |
| + log.Printf("agent: restart poll: %d apps unreadable: %v", len(readErrs), readErrs[0]) |
| + } |
| + if !seeded && len(readErrs) == 0 { |
| + // After the first CLEAN pass, whose sightings are the baseline by |
| + // definition. A pass that could not read some app is not a |
| + // baseline: seeding from it would leave that app unrecorded, and |
| + // an unrecorded app under a known baseline reads as a request. |
| + // Create-only; see seedRestartMarks. |
| + if err := seedRestartMarks(restartMarksPath(dd), cache); err != nil { |
| + log.Printf("agent: seed restart marks: %v", err) |
| + } |
| + seeded = true |
| + } |
| + if len(moved) == 0 { |
| + continue |
| + } |
| + // This process's own children first — they are cycled here or nowhere, |
| + // and the write below can block on a busy /data. |
| + bounceExecRoutes(routes, moved, bounce) |
| + if err := requestRootRestart(dd); err != nil { |
| + log.Printf("agent: request root restart pass: %v", err) |
| + continue |
| + } |
| + log.Printf("agent: restart requested for %d app(s) on this box, asked for a root restart pass", len(moved)) |
| + } |
| +} |
diff --git a/internal/host/restart_test.go b/internal/host/restart_test.go| new file mode 100644 |
| index 000000000..48c203d9c |
| --- /dev/null |
| +++ b/internal/host/restart_test.go |
| @@ -0,0 +1,456 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "os" |
| + "path/filepath" |
| + "slices" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// restartEtag reads an app's restart-generation etag straight from the store, |
| +// so a test asserts on what the bucket actually reports rather than a value it |
| +// made up. |
| +func restartEtag(t *testing.T, ctx context.Context, s release.Store, acct, app string) string { |
| + t.Helper() |
| + _, etag, err := s.Get(ctx, release.RestartGenKey(acct, app)) |
| + if err != nil { |
| + t.Fatalf("read restart generation %s/%s: %v", acct, app, err) |
| + } |
| + return etag |
| +} |
| + |
| +// askRestart is a member's `carlos restart` landing in the bucket: the one |
| +// write the whole feature turns on. |
| +func askRestart(t *testing.T, ctx context.Context, s release.Store, acct, app string) { |
| + t.Helper() |
| + if err := release.TouchRestartGen(ctx, s, acct, app); err != nil { |
| + t.Fatalf("TouchRestartGen %s/%s: %v", acct, app, err) |
| + } |
| +} |
| + |
| +// namedUnitRoute is the addr-backed, operator-named-unit shape issue #110 |
| +// added and #113's second case is about: a console. Only the ROOT pass can |
| +// cycle it. |
| +func namedUnitRoute(host, acct, app, unit string) registry.Route { |
| + rt := execRoute(host, acct, app) |
| + rt.Socket = "" |
| + rt.Addr = "127.0.0.1:9000" |
| + rt.Unit = unit |
| + return rt |
| +} |
| + |
| +// recordingRestart is the root pass's dispatch seam: every host it was asked |
| +// to cycle, in order, plus an optional per-host failure. |
| +type recordingRestart struct { |
| + hosts []string |
| + fail map[string]error |
| +} |
| + |
| +func (r *recordingRestart) fn(host string) error { |
| + r.hosts = append(r.hosts, host) |
| + if err, ok := r.fail[host]; ok { |
| + return err |
| + } |
| + return nil |
| +} |
| + |
| +// ---- the agent's sweep ------------------------------------------------------- |
| + |
| +func TestRestartGenPassFirstSightIsSilent(t *testing.T) { |
| + ctx, s := genKit(t) |
| + askRestart(t, ctx, s, "bab", "eleven") // requested before this box ever looked |
| + routes := []registry.Route{execRoute("a.example.com", "bab", "eleven")} |
| + cache := map[[2]string]string{} |
| + |
| + moved, errs := restartGenPass(ctx, routes, s, cache) |
| + if len(moved) != 0 { |
| + t.Fatalf("first sight moved %v — a box that has just started must not cycle everything it serves", moved) |
| + } |
| + if len(errs) != 0 { |
| + t.Fatalf("unexpected read errors: %v", errs) |
| + } |
| + if got, want := cache[[2]string{"bab", "eleven"}], restartEtag(t, ctx, s, "bab", "eleven"); got != want { |
| + t.Fatalf("cache etag = %q, want the store's %q", got, want) |
| + } |
| +} |
| + |
| +func TestRestartGenPassMovementAsksOnce(t *testing.T) { |
| + ctx, s := genKit(t) |
| + askRestart(t, ctx, s, "bab", "eleven") |
| + routes := []registry.Route{ |
| + execRoute("a.example.com", "bab", "eleven"), |
| + execRoute("b.example.com", "bab", "eleven"), // same app, two routes |
| + } |
| + cache := map[[2]string]string{} |
| + restartGenPass(ctx, routes, s, cache) // baseline |
| + |
| + askRestart(t, ctx, s, "bab", "eleven") |
| + moved, errs := restartGenPass(ctx, routes, s, cache) |
| + if len(errs) != 0 { |
| + t.Fatalf("unexpected read errors: %v", errs) |
| + } |
| + if len(moved) != 1 || moved[0] != [2]string{"bab", "eleven"} { |
| + t.Fatalf("moved = %v, want exactly one app entry (one request, one ask)", moved) |
| + } |
| + if moved, _ := restartGenPass(ctx, routes, s, cache); len(moved) != 0 { |
| + t.Fatalf("the pass after a request moved %v — a stale cache would ask forever", moved) |
| + } |
| +} |
| + |
| +func TestRestartGenPassAbsentStaysSilentAndFirstRequestMoves(t *testing.T) { |
| + ctx, s := genKit(t) |
| + routes := []registry.Route{execRoute("a.example.com", "bab", "eleven")} |
| + cache := map[[2]string]string{} |
| + |
| + // An app nobody has ever restarted: absent, legal, forever. |
| + for i := range 2 { |
| + moved, errs := restartGenPass(ctx, routes, s, cache) |
| + if len(moved) != 0 { |
| + t.Fatalf("pass %d: an app with no restart generation moved %v", i, moved) |
| + } |
| + if len(errs) != 0 { |
| + t.Fatalf("pass %d: absent is not a read error: %v", i, errs) |
| + } |
| + } |
| + if got := cache[[2]string{"bab", "eleven"}]; got != genAbsent { |
| + t.Fatalf("cache = %q, want the known-absent sentinel %q", got, genAbsent) |
| + } |
| + |
| + // Its FIRST ever request: absent -> present is movement, or the very first |
| + // `carlos restart` for an app would be the one that never works. |
| + askRestart(t, ctx, s, "bab", "eleven") |
| + moved, _ := restartGenPass(ctx, routes, s, cache) |
| + if len(moved) != 1 { |
| + t.Fatalf("absent -> present moved %v, want the app", moved) |
| + } |
| +} |
| + |
| +func TestRestartGenPassReadErrorLeavesCacheAndKeepsGoing(t *testing.T) { |
| + ctx, s := genKit(t) |
| + askRestart(t, ctx, s, "bab", "eleven") |
| + askRestart(t, ctx, s, "bab", "titogo") |
| + routes := []registry.Route{ |
| + execRoute("a.example.com", "bab", "eleven"), |
| + execRoute("b.example.com", "bab", "titogo"), |
| + } |
| + cache := map[[2]string]string{} |
| + restartGenPass(ctx, routes, s, cache) |
| + before := cache[[2]string{"bab", "eleven"}] |
| + |
| + askRestart(t, ctx, s, "bab", "titogo") // a real request on the app that CAN be read |
| + blip := &failingKeyStore{Store: s, failKey: release.RestartGenKey("bab", "eleven"), err: errors.New("503 slow down")} |
| + moved, errs := restartGenPass(ctx, routes, blip, cache) |
| + if len(errs) != 1 { |
| + t.Fatalf("read errors = %v, want exactly one", errs) |
| + } |
| + if len(moved) != 1 || moved[0] != [2]string{"bab", "titogo"} { |
| + t.Fatalf("moved = %v, want only the readable app — one bad object must not stop the pass", moved) |
| + } |
| + if got := cache[[2]string{"bab", "eleven"}]; got != before { |
| + t.Fatalf("cache for the unreadable app = %q, want it untouched at %q (a cleared entry is a bogus first sight)", got, before) |
| + } |
| +} |
| + |
| +func TestRestartGenPassDropsVanishedApps(t *testing.T) { |
| + ctx, s := genKit(t) |
| + askRestart(t, ctx, s, "bab", "eleven") |
| + both := []registry.Route{ |
| + execRoute("a.example.com", "bab", "eleven"), |
| + execRoute("b.example.com", "bab", "titogo"), |
| + } |
| + cache := map[[2]string]string{} |
| + restartGenPass(ctx, both, s, cache) |
| + |
| + restartGenPass(ctx, both[:1], s, cache) // titogo's route is gone |
| + if _, ok := cache[[2]string{"bab", "titogo"}]; ok { |
| + t.Fatal("a route that left the box kept its cache entry") |
| + } |
| + moved, _ := restartGenPass(ctx, both, s, cache) // and comes back |
| + if len(moved) != 0 { |
| + t.Fatalf("a re-added app moved %v, want a silent first sight", moved) |
| + } |
| +} |
| + |
| +// TestRestartGenPassIgnoresNonAdoptableRoutes: the membership predicate is |
| +// release.Adoptable, the same one the adopt pass moves. A platform alias |
| +// shares its canonical route's process, and a mid-provision route must not be |
| +// disturbed — neither is an app this box should be reading a restart signal |
| +// for. |
| +func TestRestartGenPassIgnoresNonAdoptableRoutes(t *testing.T) { |
| + ctx, s := genKit(t) |
| + alias := execRoute("alias.oncarlos.com", "bab", "eleven") |
| + alias.Platform = true |
| + pending := execRoute("new.example.com", "bab", "titogo") |
| + pending.Status = "creating" |
| + |
| + cache := map[[2]string]string{} |
| + restartGenPass(ctx, []registry.Route{alias, pending}, s, cache) |
| + if len(cache) != 0 { |
| + t.Fatalf("cache = %v, want empty (neither route is adoptable)", cache) |
| + } |
| +} |
| + |
| +// ---- the agent's own children ------------------------------------------------ |
| + |
| +func TestBounceExecRoutesCyclesOnlyWhatRootCannot(t *testing.T) { |
| + moved := [][2]string{{"bab", "eleven"}} |
| + routes := []registry.Route{ |
| + execRoute("exec.example.com", "bab", "eleven"), // the agent's child |
| + unitRoute("unit.example.com", "bab", "eleven"), // carlos-app@ — root's |
| + namedUnitRoute("console.example.com", "bab", "eleven", "console.service"), // #110 — root's |
| + execRoute("other.example.com", "bab", "titogo"), // a different app |
| + } |
| + var bounced []string |
| + n := bounceExecRoutes(routes, moved, func(h string) bool { |
| + bounced = append(bounced, h) |
| + return true |
| + }) |
| + if n != 1 || !slices.Equal(bounced, []string{"exec.example.com"}) { |
| + t.Fatalf("bounced %v (n=%d), want only the exec-backed route of the moved app", bounced, n) |
| + } |
| +} |
| + |
| +// TestBounceExecRoutesReachesAWarmHibernator: a hibernate-flagged route is |
| +// included on purpose. Bounce cannot start anything, so a SLEEPING instance is |
| +// untouched — but a warm one is exactly the wedged instance `carlos restart` |
| +// exists for, and leaving it serving would be the feature failing at its own |
| +// headline case. |
| +func TestBounceExecRoutesReachesAWarmHibernator(t *testing.T) { |
| + rt := execRoute("warm.example.com", "bab", "eleven") |
| + rt.Hibernate = true |
| + var bounced []string |
| + // The activator's own contract: false when not awake, true when it stopped |
| + // something. Here it is awake. |
| + bounceExecRoutes([]registry.Route{rt}, [][2]string{{"bab", "eleven"}}, func(h string) bool { |
| + bounced = append(bounced, h) |
| + return true |
| + }) |
| + if !slices.Equal(bounced, []string{"warm.example.com"}) { |
| + t.Fatalf("bounced %v, want the warm hibernator", bounced) |
| + } |
| +} |
| + |
| +func TestBounceExecRoutesWithoutAnActivator(t *testing.T) { |
| + routes := []registry.Route{execRoute("exec.example.com", "bab", "eleven")} |
| + if n := bounceExecRoutes(routes, [][2]string{{"bab", "eleven"}}, nil); n != 0 { |
| + t.Fatalf("bounced %d with no activator armed, want 0 and no panic", n) |
| + } |
| +} |
| + |
| +// ---- the root pass ----------------------------------------------------------- |
| + |
| +func marksOf(t *testing.T, path string) *restartMarks { |
| + t.Helper() |
| + raw, err := os.ReadFile(path) |
| + if err != nil { |
| + t.Fatalf("read marks: %v", err) |
| + } |
| + m := &restartMarks{} |
| + if err := json.Unmarshal(raw, m); err != nil { |
| + t.Fatalf("decode marks %s: %v", raw, err) |
| + } |
| + return m |
| +} |
| + |
| +// TestRestartPassSeedsWithoutReplaying is the expensive lesson this repo has |
| +// already paid for once: a box that has never recorded a baseline records one |
| +// and restarts NOTHING. Every app on a freshly upgraded box has a restart |
| +// generation the moment anyone has ever used the verb, and cycling them all on |
| +// the first pass would take the box out. |
| +func TestRestartPassSeedsWithoutReplaying(t *testing.T) { |
| + ctx, s := genKit(t) |
| + askRestart(t, ctx, s, "bab", "eleven") |
| + askRestart(t, ctx, s, "bab", "titogo") |
| + routes := []registry.Route{ |
| + unitRoute("a.example.com", "bab", "eleven"), |
| + unitRoute("b.example.com", "bab", "titogo"), |
| + } |
| + path := filepath.Join(t.TempDir(), "restart-marks.json") |
| + rec := &recordingRestart{} |
| + |
| + restartPass(ctx, routes, s, path, rec.fn) |
| + if len(rec.hosts) != 0 { |
| + t.Fatalf("the seeding pass cycled %v, want nothing", rec.hosts) |
| + } |
| + m := marksOf(t, path) |
| + if len(m.Apps) != 2 { |
| + t.Fatalf("marks = %v, want both apps baselined", m.Apps) |
| + } |
| + // And the very next pass is a quiet one, not a replay. |
| + restartPass(ctx, routes, s, path, rec.fn) |
| + if len(rec.hosts) != 0 { |
| + t.Fatalf("the pass after seeding cycled %v, want nothing", rec.hosts) |
| + } |
| +} |
| + |
| +// TestRestartPassCyclesOnMovement is the feature working: baseline, request, |
| +// dispatch — for every non-hibernating route of THAT app and no other. |
| +func TestRestartPassCyclesOnMovement(t *testing.T) { |
| + ctx, s := genKit(t) |
| + askRestart(t, ctx, s, "bab", "eleven") |
| + askRestart(t, ctx, s, "bab", "titogo") |
| + sleeping := unitRoute("sleep.example.com", "bab", "eleven") |
| + sleeping.Hibernate = true |
| + routes := []registry.Route{ |
| + unitRoute("a.example.com", "bab", "eleven"), |
| + namedUnitRoute("console.example.com", "bab", "eleven", "console.service"), |
| + sleeping, |
| + unitRoute("z.example.com", "bab", "titogo"), // a bystander app |
| + } |
| + path := filepath.Join(t.TempDir(), "restart-marks.json") |
| + rec := &recordingRestart{} |
| + restartPass(ctx, routes, s, path, rec.fn) // baseline |
| + |
| + askRestart(t, ctx, s, "bab", "eleven") |
| + restartPass(ctx, routes, s, path, rec.fn) |
| + |
| + slices.Sort(rec.hosts) |
| + want := []string{"a.example.com", "console.example.com"} |
| + if !slices.Equal(rec.hosts, want) { |
| + t.Fatalf("cycled %v, want %v — the hibernating route must be left asleep and another app's route untouched", |
| + rec.hosts, want) |
| + } |
| + if got, want := marksOf(t, path).Apps[restartMarkKey("bab", "eleven")], restartEtag(t, ctx, s, "bab", "eleven"); got != want { |
| + t.Fatalf("mark = %q, want the etag just acted on %q", got, want) |
| + } |
| + // Idempotent: the same request is not served twice. |
| + rec.hosts = nil |
| + restartPass(ctx, routes, s, path, rec.fn) |
| + if len(rec.hosts) != 0 { |
| + t.Fatalf("a second pass on the same generation cycled %v, want nothing", rec.hosts) |
| + } |
| +} |
| + |
| +// TestRestartPassActsOnAnAppsFirstEverRequest: under a KNOWN baseline, an app |
| +// with no mark at all is movement, not a first sight. It has to be: the object |
| +// only comes into existence when somebody asks, so treating "no mark yet" as |
| +// something to seed would drop the first `carlos restart` for every app, |
| +// forever. |
| +func TestRestartPassActsOnAnAppsFirstEverRequest(t *testing.T) { |
| + ctx, s := genKit(t) |
| + routes := []registry.Route{unitRoute("a.example.com", "bab", "eleven")} |
| + path := filepath.Join(t.TempDir(), "restart-marks.json") |
| + rec := &recordingRestart{} |
| + |
| + restartPass(ctx, routes, s, path, rec.fn) // baseline: the app has no object at all |
| + if m := marksOf(t, path); len(m.Apps) != 0 { |
| + t.Fatalf("marks = %v, want no entry for an app with no restart generation", m.Apps) |
| + } |
| + |
| + askRestart(t, ctx, s, "bab", "eleven") |
| + restartPass(ctx, routes, s, path, rec.fn) |
| + if !slices.Equal(rec.hosts, []string{"a.example.com"}) { |
| + t.Fatalf("cycled %v, want the app's route on its first ever request", rec.hosts) |
| + } |
| +} |
| + |
| +// TestRestartPassAdvancesTheMarkOnAFailedDispatch: a restart request is a |
| +// point-in-time event, not a desired state. A named unit that will not come |
| +// back must be reported, not retried on every pass until the end of time. |
| +func TestRestartPassAdvancesTheMarkOnAFailedDispatch(t *testing.T) { |
| + ctx, s := genKit(t) |
| + askRestart(t, ctx, s, "bab", "eleven") |
| + routes := []registry.Route{namedUnitRoute("console.example.com", "bab", "eleven", "wrong.service")} |
| + path := filepath.Join(t.TempDir(), "restart-marks.json") |
| + rec := &recordingRestart{fail: map[string]error{"console.example.com": errors.New("Unit wrong.service not loaded")}} |
| + restartPass(ctx, routes, s, path, rec.fn) // baseline |
| + |
| + askRestart(t, ctx, s, "bab", "eleven") |
| + restartPass(ctx, routes, s, path, rec.fn) |
| + if len(rec.hosts) != 1 { |
| + t.Fatalf("dispatched %v, want one attempt", rec.hosts) |
| + } |
| + rec.hosts = nil |
| + restartPass(ctx, routes, s, path, rec.fn) |
| + if len(rec.hosts) != 0 { |
| + t.Fatalf("a failed dispatch was retried (%v) — the request must not become a standing order", rec.hosts) |
| + } |
| +} |
| + |
| +// TestRestartPassUnreadableMarksReBaseline: a wiped or corrupt marks file |
| +// fails toward NOT restarting. The cost is one dropped request, logged and |
| +// re-runnable; the alternative is a box that cycles everything it serves the |
| +// moment a state file goes missing. |
| +func TestRestartPassUnreadableMarksReBaseline(t *testing.T) { |
| + ctx, s := genKit(t) |
| + askRestart(t, ctx, s, "bab", "eleven") |
| + routes := []registry.Route{unitRoute("a.example.com", "bab", "eleven")} |
| + path := filepath.Join(t.TempDir(), "restart-marks.json") |
| + if err := os.WriteFile(path, []byte("{not json"), 0o644); err != nil { |
| + t.Fatal(err) |
| + } |
| + rec := &recordingRestart{} |
| + |
| + restartPass(ctx, routes, s, path, rec.fn) |
| + if len(rec.hosts) != 0 { |
| + t.Fatalf("a corrupt marks file cycled %v, want nothing", rec.hosts) |
| + } |
| + if got, want := marksOf(t, path).Apps[restartMarkKey("bab", "eleven")], restartEtag(t, ctx, s, "bab", "eleven"); got != want { |
| + t.Fatalf("re-baselined mark = %q, want %q", got, want) |
| + } |
| +} |
| + |
| +// ---- the handover between the two halves ------------------------------------- |
| + |
| +// TestSeedRestartMarksIsCreateOnly: the agent writes the baseline within |
| +// seconds of starting so a request made before the root pass has ever run is |
| +// not silently dropped — but it must never overwrite a mark the root pass has |
| +// already advanced, or a served request would be served again. |
| +func TestSeedRestartMarksIsCreateOnly(t *testing.T) { |
| + path := filepath.Join(t.TempDir(), "restart-marks.json") |
| + cache := map[[2]string]string{ |
| + {"bab", "eleven"}: "etag-1", |
| + {"bab", "titogo"}: genAbsent, // never asked for: nothing to baseline |
| + } |
| + if err := seedRestartMarks(path, cache); err != nil { |
| + t.Fatalf("seed: %v", err) |
| + } |
| + m := marksOf(t, path) |
| + if len(m.Apps) != 1 || m.Apps[restartMarkKey("bab", "eleven")] != "etag-1" { |
| + t.Fatalf("seeded marks = %v, want only the app that has an object", m.Apps) |
| + } |
| + |
| + cache[[2]string{"bab", "eleven"}] = "etag-2" |
| + if err := seedRestartMarks(path, cache); err != nil { |
| + t.Fatalf("second seed: %v", err) |
| + } |
| + if got := marksOf(t, path).Apps[restartMarkKey("bab", "eleven")]; got != "etag-1" { |
| + t.Fatalf("mark = %q after a second seed, want the original %q — seeding must never overwrite", got, "etag-1") |
| + } |
| +} |
| + |
| +// TestSeededBaselineIsWhatTheRootPassTrusts closes the handover: the agent |
| +// seeds, a member then asks, and the root pass — running for the FIRST time, |
| +// with no memory of its own — still cycles. Without the seed that pass would |
| +// baseline instead and the request would vanish having been answered |
| +// "requested". |
| +func TestSeededBaselineIsWhatTheRootPassTrusts(t *testing.T) { |
| + ctx, s := genKit(t) |
| + dd := t.TempDir() |
| + routes := []registry.Route{unitRoute("a.example.com", "bab", "eleven")} |
| + askRestart(t, ctx, s, "bab", "eleven") // an older request, already served |
| + |
| + // The agent's first tick: baseline in memory, baseline on disk. |
| + cache := map[[2]string]string{} |
| + restartGenPass(ctx, routes, s, cache) |
| + if err := seedRestartMarks(restartMarksPath(dd), cache); err != nil { |
| + t.Fatalf("seed: %v", err) |
| + } |
| + |
| + askRestart(t, ctx, s, "bab", "eleven") // the member types `carlos restart` |
| + moved, _ := restartGenPass(ctx, routes, s, cache) |
| + if len(moved) != 1 { |
| + t.Fatalf("the agent missed the request: moved = %v", moved) |
| + } |
| + rec := &recordingRestart{} |
| + restartPass(ctx, routes, s, restartMarksPath(dd), rec.fn) |
| + if !slices.Equal(rec.hosts, []string{"a.example.com"}) { |
| + t.Fatalf("the first root pass cycled %v, want the requested route", rec.hosts) |
| + } |
| +} |
diff --git a/internal/host/units/carlos-restart.path b/internal/host/units/carlos-restart.path| new file mode 100644 |
| index 000000000..e735d94db |
| --- /dev/null |
| +++ b/internal/host/units/carlos-restart.path |
| @@ -0,0 +1,27 @@ |
| +# Member-requested process restarts (issue #113) — carlos-adopt.path's twin |
| +# for the third half the sandboxed agent cannot do itself. The agent's |
| +# restartGenLoop sees an app's restart generation move within seconds and |
| +# bounces its OWN exec children immediately, but it cannot restart a systemd |
| +# unit: not a carlos-app@ tenant, and not the bespoke unit an addr-backed route |
| +# names (issue #110's consoles). Touching /data/restart-requested relays that |
| +# to a root pass. |
| +# |
| +# It starts carlos-restart.service — `carlos adopt -restart-only` — |
| +# deliberately NOT carlos-adopt.service or carlos-config-sync.service. A |
| +# restart request changes no version and writes no config, so a full pass would |
| +# stage releases and nag about canaries on a per-request trigger, and the |
| +# config pass would find no changed env file and cycle nothing at all. Same |
| +# split, and the same reason, as carlos-stage.path having its own service. |
| +# |
| +# /data is hardcoded like every path in these unit files — a box with a |
| +# nonstandard data dir simply never fast-triggers and keeps the hourly floor |
| +# (carlos-adopt.timer runs the same comparison). |
| +[Unit] |
| +Description=CARLOS instance restart on agent request (carlos restart) |
| + |
| +[Path] |
| +PathChanged=/data/restart-requested |
| +Unit=carlos-restart.service |
| + |
| +[Install] |
| +WantedBy=multi-user.target |
diff --git a/internal/host/units/carlos-restart.service b/internal/host/units/carlos-restart.service| new file mode 100644 |
| index 000000000..fe55e1440 |
| --- /dev/null |
| +++ b/internal/host/units/carlos-restart.service |
| @@ -0,0 +1,53 @@ |
| +# One RESTART-REQUEST pass (`carlos restart`, issue #113): cycle the units of |
| +# every app whose restart generation has moved since this box last acted, and |
| +# nothing else. Started only by carlos-restart.path, which fires when the agent |
| +# touches /data/restart-requested; it has no [Install] section because nothing |
| +# else should ever want it. |
| +# |
| +# Deliberately NOT carlos-adopt.service: that unit also runs release.Pass, the |
| +# capability converge and the canary-TTL nag, so pointing a per-request trigger |
| +# at it would let one tenant's `carlos restart` cut over ANOTHER app's pending |
| +# release within seconds, and would nag on every request instead of hourly — |
| +# the same split carlos-config-sync.service and carlos-stage.service already |
| +# make. Same root + hardening shape as those two, which is what `systemctl |
| +# try-restart` requires. |
| +# |
| +# It needs neither CARLOS_RELEASE_PUBKEY nor CARLOS_SECRETS_KEY: a restart |
| +# verifies no signature and renders no config. The EnvironmentFile is still |
| +# read for CARLOS_DEPLOYMENT_BUCKET/_DIR, which is where the request itself is. |
| +[Unit] |
| +Description=carlos instance restart (on agent request) |
| +After=network-online.target |
| +Wants=network-online.target |
| +# A path unit re-triggers as often as members ask; the default start rate limit |
| +# would latch this unit into start-limit-hit after a burst and then ignore every |
| +# later trigger until an operator reset it. ([Unit], not [Service] — systemd |
| +# moved these two keys in v230.) |
| +StartLimitIntervalSec=0 |
| + |
| +[Service] |
| +Type=oneshot |
| +# CARLOS_DEPLOYMENT_BUCKET/_DIR lives here. |
| +EnvironmentFile=-/etc/carlos/host.env |
| +ExecStart=/opt/carlos/carlos adopt -restart-only |
| + |
| +# --- hardening --- |
| +# Runs as ROOT deliberately, exactly like carlos-adopt.service: this pass |
| +# restarts carlos-app@ tenants and operator-named units via systemctl. |
| +# NoNewPrivileges and the syscall filter are omitted for the systemctl child; |
| +# the rest applies. |
| +ProtectHome=yes |
| +PrivateTmp=yes |
| +PrivateDevices=yes |
| +ProtectClock=yes |
| +ProtectHostname=yes |
| +ProtectKernelTunables=yes |
| +ProtectKernelModules=yes |
| +ProtectKernelLogs=yes |
| +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX |
| +RestrictNamespaces=yes |
| +RestrictRealtime=yes |
| +LockPersonality=yes |
| +MemoryDenyWriteExecute=yes |
| +MemoryMax=256M |
| +TasksMax=64 |