| index 31927673a..96c1881ae 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -1,6 +1,6 @@ |
| // The `carlos agent` verb — the single-box role: edge (TLS + host routing) + |
| -// hibernation activator + hourly release adoption (plus a canary/*-only |
| -// fast-adopt sweep every few seconds, canaryAdoptLoop) + the fleet status |
| +// hibernation activator + hourly release adoption (plus a 2s fast-converge |
| +// loop, fastConvergeLoop: serial poll + canary/unstaged sweep) + the fleet status |
| // page's 5-minute self-report composed in one process. The composition is |
| // new here, but each part is a port: the edge from Eleven's router.go, the |
| // activator from lchat activator.go, the adopt loop is release.Pass (or, |
| @@ -158,7 +158,7 @@ func RunAgent(args []string) { |
| } |
| |
| go adoptLoop(reg, svc) |
| - go canaryAdoptLoop(reg, svc) |
| + go fastConvergeLoop(reg, svc) |
| go statusLoop(reg) |
| go domainsLoop(reg, peekAcmeCache(args)) |
| go instancesLoop(reg) |
| @@ -218,6 +218,14 @@ func peekRegistryPath(args []string, dd string) string { |
| // Store and key are resolved ONCE, at agent start, exactly like adoptLoop's: |
| // a wake is on a request's critical path, and re-reading env plus re-opening |
| // an S3 client per cold wake would put that work there for nothing. |
| +// |
| +// OnStageBlocked is wired so a cold wake on a hardened box (this sandboxed |
| +// process cannot write /opt/carlos) relays the stage it cannot perform to |
| +// the root pass (carlos-stage.path) instead of just failing the wake. NO |
| +// RestartHibernating here, deliberately: the host is cold, restart stays |
| +// nil — the activator exec'ing the freshly-pointed binary IS the restart. |
| +// The requester is built once with the hook, so its once-a-minute budget is |
| +// the hook's own, same shape as the loops'. |
| func materializeHook(reg *registry.Registry, dd string) func(context.Context, string) error { |
| store, err := openDeployStore(context.Background()) |
| if err != nil { |
| @@ -230,8 +238,10 @@ func materializeHook(reg *registry.Registry, dd string) func(context.Context, st |
| return nil |
| } |
| optDir := release.OptDir() |
| + stageReq := stageRequester(dd) |
| return func(ctx context.Context, host string) error { |
| - return release.MaterializeHost(ctx, reg, store, pub, optDir, dd, host) |
| + return release.MaterializeHost(ctx, reg, store, pub, optDir, dd, host, |
| + release.OnStageBlocked(stageReq)) |
| } |
| } |
| |
| @@ -521,7 +531,7 @@ const stageRequestEvery = time.Minute |
| |
| // stageRequester returns a release.OnStageBlocked callback that touches the |
| // marker at most once per stageRequestEvery. The closure holds the clock, so |
| -// each loop gets its own budget: the canary loop asking does not silence the |
| +// each loop gets its own budget: the fast loop asking does not silence the |
| // hourly one, and neither can starve the other. |
| // |
| // Failures are logged and swallowed, like requestRootSync's: a request that |
| @@ -547,7 +557,7 @@ func stageRequester(dd string) func(host, ver string, err error) { |
| } |
| } |
| |
| -// configGenLoop is canaryAdoptLoop's shape applied to issue #44: a config |
| +// configGenLoop is the fast-converge shape applied to issue #44: a config |
| // set should converge like a canary adopts — seconds, not the next hour. |
| // Every ~3s it reads one small object per config-carrying app (the |
| // generation object every config write touches) and, on movement, delivers |
| @@ -568,10 +578,11 @@ func stageRequester(dd string) func(host, ver string, err error) { |
| // /etc/carlos/host.env, so the case is theoretical — and the hourly timer |
| // floor stands regardless. |
| // |
| -// Cardinality: unlike canaryAdoptLoop, which is bounded to the deliberately |
| -// rare canary/* routes, this watches EVERY config-carrying app on the box. |
| -// That is accepted knowingly — one GET of the smallest object in the bucket |
| -// per app per 3s, against the store canaryAdoptLoop already hits every 2s. What |
| +// Cardinality: unlike fastConvergeLoop's sweep, which is bounded to the |
| +// deliberately rare canary/* routes plus still-unstaged ones, this watches |
| +// EVERY config-carrying app on the box. That is accepted knowingly — one GET |
| +// of the smallest object in the bucket |
| +// per app per 3s, against the store fastConvergeLoop already hits every 2s. What |
| // keeps it bounded is that the pass reads generation objects only: no |
| // manifests, no pointers, no per-environment resolution. All of that |
| // happens inside the SyncExec a real change triggers. |
| @@ -660,22 +671,239 @@ func configGenLoop(reg *registry.Registry, svc *activator.Service, dd string) { |
| } |
| } |
| |
| -// canaryAdoptLoop mirrors adoptLoop but sweeps only routes following a |
| -// canary/* channel, every few seconds instead of hourly — "I promoted a |
| -// canary, why is my box still on the old version for the next 40 minutes" |
| -// with a real fix instead of an operator forcing `carlos adopt` by hand. |
| -// Bounded deliberately to canary/*: those channels are individually |
| -// created and inherently low-cardinality (one deliberate rollout at a |
| -// time, never the whole fleet — see release.PassFilter's doc comment and |
| -// bake.go's "canary/* has no window"), so this can't grow into a |
| -// fleet-wide poll storm the way sweeping every route this fast would. |
| -// Stable/beta routes stay on adoptLoop's hourly cadence: they're |
| -// bake-gated for a day or three regardless, so faster polling would buy |
| -// nothing there. |
| -// Takes the activator for the same reason adoptLoop does: a promoted canary |
| -// on a warm exec instance must bounce onto its new binary, not wait for |
| -// hibernation. Nil is fine. |
| -func canaryAdoptLoop(reg *registry.Registry, svc *activator.Service) { |
| +// Fast-converge cadence and clamps (carlos-deploy spec §3-§4). The tick is |
| +// the serial poll's latency promise (~2s from console bump to pass); the |
| +// clamp bounds what a burst of console writes can cost (one full pass set |
| +// per 5s, with the last bump deferred, never dropped); the backoff bounds |
| +// what a permanently-unstageable route can cost (one look per 5min at the |
| +// cap, instead of one per tick forever). |
| +const ( |
| + fastTickEvery = 2 * time.Second |
| + fastFullClamp = 5 * time.Second |
| + fastBackoffBase = 2 * time.Second |
| + fastBackoffCap = 5 * time.Minute |
| +) |
| + |
| +// fastDeps is one fastConvergeLoop's state and seams — a struct so the tick |
| +// is testable without goroutines, env vars, or sleeps. |
| +type fastDeps struct { |
| + reg *registry.Registry |
| + store release.Store |
| + pub ed25519.PublicKey |
| + optDir string |
| + restart func(string) error |
| + passOpts []release.PassOption // fastPassOpts — every sweep pass carries all of them |
| + fullPass func(ctx context.Context) // reconcile → adopt → config sync, or a test hook |
| + lastSerial int64 |
| + seeded bool |
| + lastFull time.Time |
| + notBefore map[string]time.Time |
| + step map[string]time.Duration |
| + serialBroken bool // last ReadSerial failed and was logged; reset on success |
| +} |
| + |
| +// fastPassOpts is the option set EVERY fast-converge pass carries — the |
| +// serial-triggered full release.Pass and the sweep's release.PassFilter |
| +// both, built in one place so neither call site can drift from the other. |
| +// |
| +// DO NOT add release.NagExpiredCanaries(...) to this set, ever. This |
| +// is THE loop release.NagExpiredCanaries' doc comment warns about by |
| +// name: every 2-3 seconds, canary/* always in the set. An expired canary sitting |
| +// still would nag on every single tick — a journal-flooding poll storm |
| +// out of what is supposed to be an hourly, easy-to-skim log line. The |
| +// nag belongs to RunAdopt (carlos-adopt.service) alone. |
| +// |
| +// DO NOT add release.WithCapabilities(...) to this set, ever. Same |
| +// sandbox as adoptLoop above (ProtectSystem=strict, |
| +// ReadWritePaths=/data /run/carlos), and this one ticks every 2-3 |
| +// SECONDS: arming the converge here would write nothing and log a |
| +// permission denial forever. The accepted consequence, ruled on in |
| +// the 2026-08-03 review of the capability design: a freshly promoted |
| +// canary runs its new binary against the unit shape the previous |
| +// version left until the hourly root pass converges it — up to an |
| +// hour. release.adoptHost prints one line per such repoint so the |
| +// window is visible in the journal rather than silent. |
| +// |
| +// OnStageBlocked IS wired here, and this is the loop it matters most |
| +// for: a canary is promoted to be picked up in seconds, and staging is |
| +// the one step of that pickup this process cannot perform. Throttled |
| +// inside stageRequester — this tick is 2-3 seconds, the request is at |
| +// most one a minute (the requester is built ONCE per loop, so the budget |
| +// is the loop's, not one tick's). |
| +// |
| +// RestartHibernating IS wired here too, and for the same reason as on |
| +// adoptLoop: this process owns the exec children. It matters more on |
| +// this loop than on any other — a promotion is bumped or fast-swept |
| +// precisely to be picked up in seconds, and the routes this loop reaches |
| +// are the warm ones somebody is watching. Pinned by |
| +// TestFastSweepBouncesAWarmHibernatingCanary: dropping it would restore |
| +// a dead-branch production bug without failing any other test. |
| +func fastPassOpts(dd string, stageReq func(host, ver string, err error), restart func(string) error) []release.PassOption { |
| + return []release.PassOption{ |
| + release.BakeDir(dd), |
| + release.OnStageBlocked(stageReq), |
| + release.RestartHibernating(restart), |
| + } |
| +} |
| + |
| +// fastSweepable mirrors release's adoptable predicate (adopt.go): the routes |
| +// a pass would consider at all. Anything else — platform aliases, non-ready |
| +// routes, routes with no app — must not count as outstanding work, or a |
| +// route the pass refuses on shape would make every tick sweep forever. |
| +func fastSweepable(rt registry.Route) bool { |
| + return rt.Kind == "instance" && rt.App != "" && !rt.Platform && |
| + (rt.Status == "" || rt.Status == "ready") |
| +} |
| + |
| +// fastMembers computes the fast set for one tick — the routes with a reason |
| +// to be looked at FASTER than the hourly floor: |
| +// |
| +// - canary/* routes, always: a direct-bucket operator promote |
| +// (scripts/promote-app.sh, or `carlos promote` against the bucket) |
| +// never bumps the serial, so the sweep is the only thing keeping the |
| +// "canary adopts in seconds" promise (spec §4 correction). |
| +// - unstaged routes (no live symlink), once their backoff window has |
| +// passed: a route was just minted, or its stage failed; the sweep |
| +// picks it up as soon as there is something to stage. |
| +// |
| +// Computed BEFORE any store work so an empty set costs zero store reads — |
| +// the idle box pays one serial GET per tick and nothing else. A registry |
| +// list failure returns the empty set (a broken box, logged per tick by |
| +// statusLoop's identical call — not worth a second line every 2s here). |
| +func (d *fastDeps) fastMembers(now time.Time) []string { |
| + routes, err := d.reg.List() |
| + if err != nil { |
| + return nil |
| + } |
| + var members []string |
| + seen := map[string]bool{} |
| + for _, rt := range routes { |
| + if !fastSweepable(rt) || seen[rt.Host] { |
| + continue |
| + } |
| + if isCanaryRoute(rt) { |
| + seen[rt.Host] = true |
| + members = append(members, rt.Host) |
| + continue |
| + } |
| + // A LiveVersion error reads as unstaged: the pass is the thing that |
| + // can repair a broken link, and the backoff bounds the retry cost. |
| + if v, err := release.LiveVersion(d.optDir, rt.Host); err == nil && v != "" { |
| + continue // staged: nothing outstanding at this cadence |
| + } |
| + if nb, ok := d.notBefore[rt.Host]; ok && now.Before(nb) { |
| + continue // still inside this host's backoff window |
| + } |
| + seen[rt.Host] = true |
| + members = append(members, rt.Host) |
| + } |
| + return members |
| +} |
| + |
| +// noteUnstaged records that host came through a sweep still unstaged: |
| +// double its backoff (base 2s, capped at 5min) and push its next look out. |
| +// Held routes cost almost nothing this way: the pass's own bake logic |
| +// refuses them and the backoff keeps re-checks rare; the serial-triggered |
| +// and hourly passes re-notice them when the window expires. |
| +func (d *fastDeps) noteUnstaged(host string, now time.Time) { |
| + if d.step == nil { |
| + d.step = map[string]time.Duration{} |
| + d.notBefore = map[string]time.Time{} |
| + } |
| + s := d.step[host] |
| + if s == 0 { |
| + s = fastBackoffBase |
| + } else if s *= 2; s > fastBackoffCap { |
| + s = fastBackoffCap |
| + } |
| + d.step[host] = s |
| + d.notBefore[host] = now.Add(s) |
| +} |
| + |
| +// backoffFor reports host's current backoff step (zero when none recorded). |
| +func (d *fastDeps) backoffFor(host string) time.Duration { return d.step[host] } |
| + |
| +// fastTick is one fast-converge tick — two independent jobs (spec §3-§4): |
| +// |
| +// 1. Serial check: one ReadSerial. The FIRST successful sight of the |
| +// object only seeds lastSerial — boot must not replay history |
| +// (adoptLoop's 1-minute first pass covers boot). Afterward any CHANGE |
| +// (not just increments — a reset must not wedge the box) runs the full |
| +// pass set, clamped to one per fastFullClamp: a bump inside the window |
| +// is deferred by NOT recording the acted serial, so the next tick past |
| +// the window re-fires. Read errors are tolerated silently (one log |
| +// line per state change, never per tick). |
| +// 2. Fast sweep: fastMembers, and only when non-empty one PassFilter over |
| +// exactly those hosts — an empty sweep costs zero store reads. After |
| +// the sweep, members still unstaged back off; staged members clear |
| +// their backoff state. |
| +func fastTick(ctx context.Context, d *fastDeps, now time.Time) { |
| + doc, etag, err := release.ReadSerial(ctx, d.store) |
| + switch { |
| + case err != nil: |
| + if !d.serialBroken { |
| + log.Printf("agent: fast converge serial read: %v", err) |
| + d.serialBroken = true |
| + } |
| + case etag == "": |
| + // No serial object yet (pre-first-bump): nothing to seed — seeding |
| + // the zero doc here would make the first real bump replay as a |
| + // "change" that predates this box's interest. |
| + d.serialBroken = false |
| + case !d.seeded: |
| + d.lastSerial, d.seeded = doc.Serial, true |
| + d.serialBroken = false |
| + default: |
| + d.serialBroken = false |
| + if doc.Serial != d.lastSerial && now.Sub(d.lastFull) >= fastFullClamp { |
| + d.fullPass(ctx) |
| + d.lastSerial = doc.Serial |
| + d.lastFull = now |
| + } |
| + } |
| + |
| + members := d.fastMembers(now) |
| + if len(members) == 0 { |
| + return // zero store reads: no PassFilter, no bake-clock write |
| + } |
| + member := map[string]bool{} |
| + for _, h := range members { |
| + member[h] = true |
| + } |
| + if err := release.PassFilter(ctx, d.reg, d.store, d.pub, d.optDir, d.restart, |
| + func(rt registry.Route) bool { return member[rt.Host] }, d.passOpts...); err != nil { |
| + log.Printf("agent: fast converge sweep: %v", err) |
| + } |
| + for _, h := range members { |
| + if v, err := release.LiveVersion(d.optDir, h); err == nil && v != "" { |
| + delete(d.notBefore, h) |
| + delete(d.step, h) |
| + } else { |
| + d.noteUnstaged(h, now) |
| + } |
| + } |
| +} |
| + |
| +// fastConvergeLoop replaces the canary-only fast adopt loop with the |
| +// carlos-deploy fast set (spec §3-§4): a 2s serial poll that turns a console |
| +// bump (promote, instance create, config write) into the ordinary verified |
| +// pass set — reconcile → adopt → exec config sync — within seconds, plus a |
| +// fast sweep of the routes with genuinely outstanding work (canary/* always; |
| +// unstaged routes until they drain, with per-host backoff). Everything it |
| +// triggers is the same signature-verified, bake-gated work the hourly loops |
| +// run; the serial carries no authority, only earliness. |
| +// |
| +// Setup mirrors adoptLoop's: missing deployment config or pubkey disables |
| +// the loop silently (adoptLoop already logged the reason). The config-sync |
| +// leg degrades exactly as configSyncLoop does — no CARLOS_SECRETS_KEY, or a |
| +// broken one, and the full pass set simply omits that leg (configSyncLoop |
| +// already logged that too). Takes the activator for the same reason |
| +// adoptLoop does: a fresh version on a warm exec instance must bounce onto |
| +// its new binary, not wait for hibernation. Nil is fine. |
| +// |
| +// Thin and untested by convention, like adoptLoop/statusLoop — the tick's |
| +// whole decision lives in fastTick/fastMembers, which are tested. |
| +func fastConvergeLoop(reg *registry.Registry, svc *activator.Service) { |
| ctx := context.Background() |
| store, err := openDeployStore(ctx) |
| if err != nil { |
| @@ -685,45 +913,43 @@ func canaryAdoptLoop(reg *registry.Registry, svc *activator.Service) { |
| if err != nil { |
| return // ditto |
| } |
| + var priv *ecdh.PrivateKey |
| + if raw := os.Getenv("CARLOS_SECRETS_KEY"); raw != "" { |
| + if p, err := appconfig.ParsePrivateKey(raw, "CARLOS_SECRETS_KEY"); err == nil { |
| + priv = p |
| + } |
| + } |
| + dd := dataDir() |
| optDir := release.OptDir() |
| restart := makeRestartRoute(reg, systemctlTryRestart, bounceFn(svc)) |
| - log.Printf("agent: canary adopt loop armed (2s + jitter, canary/* routes only)") |
| + // Built once, used by both pass shapes below — see fastPassOpts for the |
| + // DO-NOT list and why the stage requester is per-loop, not per-tick. |
| + opts := fastPassOpts(dd, stageRequester(dd), restart) |
| + d := &fastDeps{ |
| + reg: reg, store: store, pub: pub, optDir: optDir, |
| + restart: restart, passOpts: opts, |
| + // The serial-triggered full pass set, in spec order: mint routes |
| + // (reconcile), then stage them (adopt), then deliver their config. |
| + // Each error is one log line, never fatal — the hourly floors stand |
| + // underneath every leg. |
| + fullPass: func(ctx context.Context) { |
| + if err := provision.Reconcile(ctx, store, reg, provisionTickConfig(os.Getenv)); err != nil { |
| + log.Printf("agent: fast converge reconcile: %v", err) |
| + } |
| + if err := release.Pass(ctx, reg, store, pub, optDir, restart, opts...); err != nil { |
| + log.Printf("agent: fast converge adopt pass: %v", err) |
| + } |
| + if priv != nil { |
| + if err := syncExecSerialized(ctx, reg, store, priv, dd, svc); err != nil { |
| + log.Printf("agent: fast converge config sync: %v", err) |
| + } |
| + } |
| + }, |
| + } |
| + log.Printf("agent: fast converge loop armed (2s + jitter: serial poll + canary/* + unstaged routes)") |
| for { |
| - time.Sleep(2*time.Second + rand.N(time.Second)) |
| - // DO NOT add release.NagExpiredCanaries(...) to this call, ever. This |
| - // is THE loop release.NagExpiredCanaries' doc comment warns about by |
| - // name: every 2-3 seconds, canary/* only. An expired canary sitting |
| - // still would nag on every single tick — a journal-flooding poll storm |
| - // out of what is supposed to be an hourly, easy-to-skim log line. The |
| - // nag belongs to RunAdopt (carlos-adopt.service) alone. |
| - // |
| - // DO NOT add release.WithCapabilities(...) to this call, ever. Same |
| - // sandbox as adoptLoop above (ProtectSystem=strict, |
| - // ReadWritePaths=/data /run/carlos), and this one ticks every 2-3 |
| - // SECONDS: arming the converge here would write nothing and log a |
| - // permission denial forever. The accepted consequence, ruled on in |
| - // the 2026-08-03 review of the capability design: a freshly promoted |
| - // canary runs its new binary against the unit shape the previous |
| - // version left until the hourly root pass converges it — up to an |
| - // hour. release.adoptHost prints one line per such repoint so the |
| - // window is visible in the journal rather than silent. |
| - // |
| - // OnStageBlocked IS wired here, and this is the loop it matters most |
| - // for: a canary is promoted to be picked up in seconds, and staging is |
| - // the one step of that pickup this process cannot perform. Throttled |
| - // inside stageRequester — this tick is 2-3 seconds, the request is at |
| - // most one a minute. |
| - // |
| - // RestartHibernating IS wired here too, and for the same reason as on |
| - // adoptLoop: this process owns the exec children. It matters more on |
| - // this loop than on any other — a canary is promoted precisely to be |
| - // picked up in seconds, and the routes following canary/* are the warm |
| - // ones somebody is watching. |
| - if err := release.PassFilter(ctx, reg, store, pub, optDir, restart, isCanaryRoute, |
| - release.BakeDir(dataDir()), release.OnStageBlocked(stageRequester(dataDir())), |
| - release.RestartHibernating(restart)); err != nil { |
| - log.Printf("agent: canary adopt pass: %v", err) |
| - } |
| + time.Sleep(fastTickEvery + rand.N(time.Second)) |
| + fastTick(ctx, d, time.Now()) |
| } |
| } |
| |
| @@ -1427,9 +1653,9 @@ func RunAdopt(args []string) { |
| // NagExpiredCanaries(os.Stdout) is set HERE ONLY (2026-08-03 review) — the |
| // canary-TTL nag (spec §5) belongs to the hourly root pass |
| // (carlos-adopt.service, this verb's ExecStart), the same split PR #9's |
| - // config sync above uses. adoptLoop and canaryAdoptLoop in this same |
| - // package call release.Pass/PassFilter without the nag option (they pass |
| - // BakeDir only), so an expired canary nags once an hour from here, never |
| + // config sync above uses. adoptLoop and fastConvergeLoop in this same |
| + // package call release.Pass/PassFilter without the nag option, so an |
| + // expired canary nags once an hour from here, never |
| // every 2-3 seconds from the in-process fast loop. |
| // |
| // WithCapabilities(caps.Ops{}) is likewise set HERE ONLY, and the zero Ops |
| @@ -1437,8 +1663,8 @@ func RunAdopt(args []string) { |
| // useradd). This verb runs as root out of carlos-adopt.service; the |
| // in-process loops in this same file run inside carlos-edge.service's |
| // ProtectSystem=strict sandbox and cannot write unit configuration at all, |
| - // which is why they call Pass/PassFilter with BakeDir only and why both |
| - // carry a comment saying never to add this one. |
| + // which is why adoptLoop and fastPassOpts each carry a comment saying |
| + // never to add this one. |
| // |
| // -stage-only drops the nag and keeps everything else. It is what |
| // carlos-stage.service runs when the agent relays a version it could not |
amadan