| 1 | // The housekeeper: the ONE election for this deployment's global duties |
| 2 | // (spec 2026-08-20-multi-region-ingress-design.md, Part 1). A deployment now |
| 3 | // has N platform boxes converging one control bucket, and the rule the writer |
| 4 | // map states is "data that is per-box gets per-box keys; duties that are |
| 5 | // deployment-global get a lease". The releases retention prune is the first |
| 6 | // such duty to move behind it — five concurrent pruners racing a promote is a |
| 7 | // bug class the spec refuses to have — and Part 2's cert issuance is the next. |
| 8 | // |
| 9 | // The mechanism is deliberately NOT new. It is internal/lease.Acquire, the |
| 10 | // same conditional-write lock (litestream's lss3.Leaser writing |
| 11 | // <prefix>/lock.json) the status publisher has elected with since the fleet |
| 12 | // status page shipped — see statusLoop's lease.Acquire call in agent.go and |
| 13 | // the bucket-writer audit's §1. The spec's own words: "The status publisher's |
| 14 | // CAS-lease is the one election mechanism, reused; no second election pattern |
| 15 | // may be introduced." So there is no store-CAS lease here, no |
| 16 | // control/housekeeper.json record, and no bespoke TTL. |
| 17 | // |
| 18 | // Two things DO differ from the status election, both on purpose: |
| 19 | // |
| 20 | // - It is long-lived, not per-tick. The status loop acquires and releases |
| 21 | // inside one 5-minute tick because publishing is a single act; the |
| 22 | // housekeeper's duties are continuous and Task 4's edge cert gate has to |
| 23 | // be answerable at any instant, so this box acquires once and RENEWS |
| 24 | // (lease.Acquire starts the renewer itself) for as long as it can, and |
| 25 | // housekeeperHeld() is a non-blocking read of that state. |
| 26 | // - It leases in the DEPLOYMENT bucket at a fixed key — |
| 27 | // control/housekeeper-lease/lock.json — not under CARLOS_STATUS_PREFIX. |
| 28 | // The duties act on the deployment bucket's own control plane |
| 29 | // (releases/**, later control/certs/**), and that bucket is the one |
| 30 | // linearizable home the spec pins the control plane to. |
| 31 | // |
| 32 | // The accepted price is lease.LeaseTTL, 15 minutes: a HARD-crashed |
| 33 | // housekeeper strands the lease until expiry, so handover can stall up to |
| 34 | // ~15m. Nothing here cares. The prune is idempotent and not urgent (a policy |
| 35 | // written now is promised "within ~10 minutes", and the loop is a 5-minute |
| 36 | // tick anyway), and cert renewals run ~30 days early. |
| 37 | // |
| 38 | // A GRACEFUL stand-down releases immediately, and that path has to be |
| 39 | // REACHABLE, not merely written: a `systemctl restart carlos-agent` is not a |
| 40 | // crash, and if it left the lock behind, every ordinary platform roll would |
| 41 | // cost the deployment up to ~15m (+ a retry) with no housekeeper — including |
| 42 | // a single-box deployment, which would be inert against its own lock, and |
| 43 | // from Part 2 that gap is "no cert issuer" after every deploy. So |
| 44 | // startHousekeeper returns a stop func and RunAgent calls it in the same |
| 45 | // post-drain block that shuts the activator's children down, beside the |
| 46 | // comment about not orphaning leases until TTL. That is the same principle. |
| 47 | // |
| 48 | // The hold owns a context, not just a flag. Every duty runs under a context |
| 49 | // derived from the holding, cancelled the instant the hold ends — a prune is |
| 50 | // a full list+delete sweep and Part 2's ACME order takes seconds to minutes, |
| 51 | // so a duty that kept running after a lost renewal would stretch the window |
| 52 | // where two boxes act on deployment-global state from housekeeperPoll to |
| 53 | // housekeeperPoll plus however long the duty runs. Duties must therefore |
| 54 | // honour their context; the ones here already do (every store call takes it). |
| 55 | package host |
| 56 | |
| 57 | import ( |
| 58 | "context" |
| 59 | "errors" |
| 60 | "log" |
| 61 | "math/rand/v2" |
| 62 | "os" |
| 63 | "sync/atomic" |
| 64 | "time" |
| 65 | |
| 66 | "github.com/carlosframework/platform/internal/edge" |
| 67 | "github.com/carlosframework/platform/internal/lease" |
| 68 | ) |
| 69 | |
| 70 | // housekeeperLeasePrefix is the deployment-bucket prefix the election's lock |
| 71 | // lands under: the leaser writes <prefix>/lock.json, so the object is |
| 72 | // control/housekeeper-lease/lock.json. Under control/ because it is control |
| 73 | // plane, and a fixed key because there is exactly one housekeeper per |
| 74 | // deployment — the same "the bucket IS the scope" property the status-lease |
| 75 | // relies on (one control bucket per deployment, forever). |
| 76 | const housekeeperLeasePrefix = "control/housekeeper-lease" |
| 77 | |
| 78 | // housekeeperRetry spaces a losing contender's attempts. Each attempt is one |
| 79 | // conditional PUT against the lock (a GET first when a lock exists), so this |
| 80 | // is the same request-cost trade internal/lease's LeaseTTL comment weighs: |
| 81 | // 5 minutes is 288 attempts/day/box, in step with the status election's own |
| 82 | // per-tick contention, and bounds takeover after a hard crash at roughly the |
| 83 | // 15m TTL plus one retry. |
| 84 | const housekeeperRetry = 5 * time.Minute |
| 85 | |
| 86 | // housekeeperStopWait bounds how long a graceful stand-down may take before |
| 87 | // the agent stops waiting for it. The release itself gets 5s |
| 88 | // (releaseHousekeeperLease) and a holder's poll tick is instant, so this is |
| 89 | // generous; it exists only so a wedged S3 call cannot hold a restart open. |
| 90 | const housekeeperStopWait = 10 * time.Second |
| 91 | |
| 92 | // housekeeperPoll is how often a HOLDER checks that renewal is still winning. |
| 93 | // Renewal itself runs inside internal/lease (TTL/3 = 5m); this only decides |
| 94 | // how quickly a box that has LOST the lease stops believing it holds it — |
| 95 | // which is exactly how long a stale holder could keep running duties after a |
| 96 | // takeover. 30 seconds is cheap (a mutexed field read, no I/O at all). |
| 97 | const housekeeperPoll = 30 * time.Second |
| 98 | |
| 99 | // heldLease is the slice of *lease.Held this file uses: renewal health and |
| 100 | // release. An interface only so the election transport is injectable — the |
| 101 | // lock's own semantics are internal/lease's to test, against a real |
| 102 | // S3-compatible store. |
| 103 | type heldLease interface { |
| 104 | Err() error |
| 105 | Release(ctx context.Context) error |
| 106 | } |
| 107 | |
| 108 | // leaseAcquirer is lease.Acquire's shape, as a seam. |
| 109 | type leaseAcquirer func(ctx context.Context, bucket, prefix, region string) (heldLease, error) |
| 110 | |
| 111 | // acquireDeploymentLease is the production acquirer. The region is |
| 112 | // deliberately empty: internal/lease's resolveRegion then answers from the |
| 113 | // AWS default chain (AWS_REGION / profile), which is precisely what |
| 114 | // release.NewS3Store signs for against this same bucket — a housekeeper lease |
| 115 | // signing for a different region than the store it guards would be a split |
| 116 | // waiting to happen. No new env var: the deployment bucket has never had a |
| 117 | // region variable of its own, and inventing one here would be a second answer |
| 118 | // to a question the chain already answers. |
| 119 | // |
| 120 | // The nil-into-interface hazard is why this is a func and not a method value: |
| 121 | // lease.Acquire returns a typed nil *lease.Held alongside its error, which |
| 122 | // would arrive as a NON-nil heldLease if handed straight back. |
| 123 | func acquireDeploymentLease(ctx context.Context, bucket, prefix, region string) (heldLease, error) { |
| 124 | h, err := lease.Acquire(ctx, bucket, prefix, region) |
| 125 | if err != nil { |
| 126 | return nil, err |
| 127 | } |
| 128 | return h, nil |
| 129 | } |
| 130 | |
| 131 | // housekeeperMode is this box's relationship to the election, decided once |
| 132 | // from the environment. |
| 133 | type housekeeperMode int |
| 134 | |
| 135 | const ( |
| 136 | // housekeeperNever: this box must never run deployment-global duties — |
| 137 | // a fleet box (a customer's machine; the platform's releases are not |
| 138 | // its business, and the store door would refuse the deletes anyway), or |
| 139 | // a box with no deployment store at all. |
| 140 | housekeeperNever housekeeperMode = iota |
| 141 | // housekeeperSole: a CARLOS_DEPLOYMENT_DIR deployment. There is no |
| 142 | // bucket to hold a lock in, and a directory store is one box by |
| 143 | // construction, so this box IS the housekeeper — anything else would |
| 144 | // silently stop dev/test deployments pruning at all. Such a deployment |
| 145 | // publishes no status, so retention reads this box's own build-state |
| 146 | // report in its place (settings spec rev 11 §4.4, ruling R-T9-SOLE): |
| 147 | // one box, so its own report is every box's. |
| 148 | housekeeperSole |
| 149 | // housekeeperElect: a bucket-backed platform box. Contend. |
| 150 | housekeeperElect |
| 151 | ) |
| 152 | |
| 153 | // holding is one continuous stretch of this box being the housekeeper. Its |
| 154 | // context is what every duty runs under, so ending the holding ends the |
| 155 | // duties: there is no state where this box believes it does not hold the |
| 156 | // lease while work it started as holder is still running. |
| 157 | type holding struct { |
| 158 | ctx context.Context |
| 159 | cancel context.CancelFunc |
| 160 | } |
| 161 | |
| 162 | // housekeeper is the long-lived acquire/renew/stand-down lifecycle plus the |
| 163 | // state every gated caller reads. |
| 164 | type housekeeper struct { |
| 165 | mode housekeeperMode |
| 166 | reason string // the one line run() logs when it arms |
| 167 | bucket string |
| 168 | acquire leaseAcquirer |
| 169 | retry time.Duration |
| 170 | poll time.Duration |
| 171 | |
| 172 | // cur is the current holding, or nil when this box is not the |
| 173 | // housekeeper. ONE atomic, deliberately: a separate "held" flag beside |
| 174 | // it would have an instant where the two disagree, and both the duty |
| 175 | // gate and Task 4's cert gate read this on live paths. |
| 176 | cur atomic.Pointer[holding] |
| 177 | } |
| 178 | |
| 179 | // newHousekeeper decides the mode from the environment. Nothing is acquired |
| 180 | // and no holding is begun here — start() does both. |
| 181 | func newHousekeeper(getenv func(string) string, acquire leaseAcquirer) *housekeeper { |
| 182 | hk := &housekeeper{acquire: acquire, retry: housekeeperRetry, poll: housekeeperPoll} |
| 183 | switch { |
| 184 | // Deliberately the bare CARLOS_FLEET test that retentionLoop and |
| 185 | // instancesLoop already use, for the same reason retentionLoop states: |
| 186 | // a fleet box whose LABEL is misconfigured is still a fleet box. |
| 187 | case shouldSkipPublisherElection(getenv): |
| 188 | hk.mode = housekeeperNever |
| 189 | hk.reason = "fleet box — deployment-global duties are a platform-box job; never contending" |
| 190 | case getenv("CARLOS_DEPLOYMENT_BUCKET") != "": |
| 191 | // The bucket openDeployStore already adopts from — the duties act on |
| 192 | // its contents, so the lock guarding them belongs in it. |
| 193 | hk.mode = housekeeperElect |
| 194 | hk.bucket = getenv("CARLOS_DEPLOYMENT_BUCKET") |
| 195 | case getenv("CARLOS_DEPLOYMENT_DIR") != "": |
| 196 | hk.mode = housekeeperSole |
| 197 | hk.reason = "directory-backed deployment — this box is its own housekeeper (no bucket to elect in)" |
| 198 | default: |
| 199 | hk.mode = housekeeperNever |
| 200 | hk.reason = "no deployment store configured — never contending" |
| 201 | } |
| 202 | return hk |
| 203 | } |
| 204 | |
| 205 | // held reports whether this box currently holds the lease. Cheap and |
| 206 | // non-blocking by contract: it is read on a duty tick and, from Part 2, on |
| 207 | // the edge's certificate path. |
| 208 | func (hk *housekeeper) held() bool { |
| 209 | _, ok := hk.gate() |
| 210 | return ok |
| 211 | } |
| 212 | |
| 213 | // gate is held() plus the context the caller's work must run under: valid |
| 214 | // only while the holding lasts, cancelled the moment it ends. A duty that |
| 215 | // takes this context cannot outlive the authority it started under. |
| 216 | func (hk *housekeeper) gate() (context.Context, bool) { |
| 217 | if hk == nil { |
| 218 | return nil, false |
| 219 | } |
| 220 | h := hk.cur.Load() |
| 221 | if h == nil { |
| 222 | return nil, false |
| 223 | } |
| 224 | return h.ctx, true |
| 225 | } |
| 226 | |
| 227 | // beginHolding marks this box the housekeeper for as long as parent lives. |
| 228 | func (hk *housekeeper) beginHolding(parent context.Context) { |
| 229 | ctx, cancel := context.WithCancel(parent) |
| 230 | hk.cur.Store(&holding{ctx: ctx, cancel: cancel}) |
| 231 | } |
| 232 | |
| 233 | // endHolding stops this box being the housekeeper: the gate closes FIRST (no |
| 234 | // new duty can start) and only then is the context cancelled (whatever is |
| 235 | // already running unwinds). Idempotent — the stop func calls it after the |
| 236 | // loop has usually done it already. |
| 237 | func (hk *housekeeper) endHolding() { |
| 238 | if h := hk.cur.Swap(nil); h != nil { |
| 239 | h.cancel() |
| 240 | } |
| 241 | } |
| 242 | |
| 243 | // start arms this housekeeper and returns the stand-down: cancel the loop, |
| 244 | // wait for it to release and exit, and close the gate. Calling it is what |
| 245 | // makes the graceful release reachable — see the file header. |
| 246 | func (hk *housekeeper) start(parent context.Context) (stop func()) { |
| 247 | ctx, cancel := context.WithCancel(parent) |
| 248 | // A lone box is the housekeeper from the instant it is configured, not |
| 249 | // from whenever a goroutine is scheduled: no acquisition is involved, so |
| 250 | // there is nothing to wait for. |
| 251 | if hk.mode == housekeeperSole { |
| 252 | hk.beginHolding(ctx) |
| 253 | } |
| 254 | done := make(chan struct{}) |
| 255 | go func() { |
| 256 | defer close(done) |
| 257 | hk.run(ctx) |
| 258 | }() |
| 259 | return func() { |
| 260 | // Close the gate BEFORE cancelling: cancel() only reaches the |
| 261 | // holding's context by propagation (it is a child of this ctx via |
| 262 | // beginHolding), so between cancel() and the run loop noticing and |
| 263 | // calling endHolding() itself there is a window where hk.cur is |
| 264 | // still non-nil over an already-cancelled context — held() reads |
| 265 | // true against dead work. Calling endHolding() first closes hk.cur |
| 266 | // synchronously, before ctx is even touched, so that window never |
| 267 | // opens. The run path's own endHolding() call (in hold()) stays and |
| 268 | // is idempotent — this is belt and braces for a path that doesn't |
| 269 | // go through hold() (housekeeperSole) or that races it. |
| 270 | hk.endHolding() |
| 271 | cancel() |
| 272 | select { |
| 273 | case <-done: |
| 274 | case <-time.After(housekeeperStopWait): |
| 275 | // Never wedge a shutdown on this: the lock expires on its own |
| 276 | // TTL, which is exactly the crash behaviour we already accept. |
| 277 | log.Printf("agent: housekeeper: stand-down did not complete within %s — the lease will expire on its TTL instead", housekeeperStopWait) |
| 278 | } |
| 279 | hk.endHolding() |
| 280 | } |
| 281 | } |
| 282 | |
| 283 | // run is the election loop; it returns only when ctx ends (or immediately, on |
| 284 | // a box that never contends). |
| 285 | // |
| 286 | // NOT holding the lease is a normal, quiet state — it is what N-1 boxes are |
| 287 | // in, permanently. Contention (*lease.HeldError) is therefore never logged, |
| 288 | // exactly as statusLoop swallows it: a line per retry per box, forever, would |
| 289 | // bury the log. A real acquire failure (credentials, a broken bucket) IS |
| 290 | // logged, same as any other tick failure. |
| 291 | func (hk *housekeeper) run(ctx context.Context) { |
| 292 | if hk.mode != housekeeperElect { |
| 293 | log.Printf("agent: housekeeper: %s", hk.reason) |
| 294 | return |
| 295 | } |
| 296 | log.Printf("agent: housekeeper: contending for s3://%s/%s/lock.json (lease TTL %s)", |
| 297 | hk.bucket, housekeeperLeasePrefix, lease.LeaseTTL) |
| 298 | for ctx.Err() == nil { |
| 299 | held, err := hk.acquire(ctx, hk.bucket, housekeeperLeasePrefix, "") |
| 300 | if err != nil { |
| 301 | var heldErr *lease.HeldError |
| 302 | if !errors.As(err, &heldErr) && ctx.Err() == nil { |
| 303 | log.Printf("agent: housekeeper: lease acquire: %v", err) |
| 304 | } |
| 305 | if !sleepCtx(ctx, hk.retry) { |
| 306 | return |
| 307 | } |
| 308 | continue |
| 309 | } |
| 310 | log.Printf("agent: housekeeper: lease acquired — this box runs the deployment's global duties") |
| 311 | hk.beginHolding(ctx) |
| 312 | hk.hold(ctx, held) |
| 313 | } |
| 314 | } |
| 315 | |
| 316 | // hold blocks while this box is the housekeeper, and ends the holding BEFORE |
| 317 | // it gives the lease up either way — closing the gate so no duty starts |
| 318 | // against a lease this box has decided to let go of, and cancelling any duty |
| 319 | // already in flight so none outlives the authority it began under. |
| 320 | // |
| 321 | // A failed renewal means leadership is lost (internal/lease stops renewing on |
| 322 | // the first failure, precisely so a re-try can't fence a claimant who took |
| 323 | // over after expiry). The lease is then NOT released: our etag is stale, so |
| 324 | // the release would just fail a precondition, and the TTL is what hands the |
| 325 | // lock over. A graceful stand-down (ctx ended: process shutting down) DOES |
| 326 | // release, so the next box takes over in seconds instead of ~15 minutes. |
| 327 | func (hk *housekeeper) hold(ctx context.Context, held heldLease) { |
| 328 | for { |
| 329 | if !sleepCtx(ctx, hk.poll) { |
| 330 | hk.endHolding() |
| 331 | releaseHousekeeperLease(held) |
| 332 | return |
| 333 | } |
| 334 | if err := held.Err(); err != nil { |
| 335 | hk.endHolding() |
| 336 | log.Printf("agent: housekeeper: lease renewal failed — standing down: %v", err) |
| 337 | return |
| 338 | } |
| 339 | } |
| 340 | } |
| 341 | |
| 342 | // releaseHousekeeperLease hands the lock back on a graceful stand-down. The |
| 343 | // release gets its OWN budget, never the caller's: by the time this runs the |
| 344 | // agent's context is already dead, and a release under a dead context leaves |
| 345 | // the lock stranded until TTL — the exact trap internal/lease's |
| 346 | // Replicator.Close names, where it cost a wake up to a full TTL of 503s. |
| 347 | func releaseHousekeeperLease(held heldLease) { |
| 348 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) |
| 349 | defer cancel() |
| 350 | if err := held.Release(ctx); err != nil { |
| 351 | log.Printf("agent: housekeeper: lease release: %v", err) |
| 352 | } |
| 353 | } |
| 354 | |
| 355 | // sleepCtx waits for d, reporting false if ctx ended first. |
| 356 | func sleepCtx(ctx context.Context, d time.Duration) bool { |
| 357 | t := time.NewTimer(d) |
| 358 | defer t.Stop() |
| 359 | select { |
| 360 | case <-ctx.Done(): |
| 361 | return false |
| 362 | case <-t.C: |
| 363 | return true |
| 364 | } |
| 365 | } |
| 366 | |
| 367 | // theHousekeeper is the process's one election, published by |
| 368 | // startHousekeeper. A pointer rather than a bare flag so the loop owns its own |
| 369 | // state and nothing else can set "held" from the side; nil (never armed) |
| 370 | // reads as not held. |
| 371 | var theHousekeeper atomic.Pointer[housekeeper] |
| 372 | |
| 373 | // housekeeperHeld reports whether THIS box currently holds the deployment's |
| 374 | // housekeeper lease. Cheap, non-blocking, and safe before (or without) |
| 375 | // startHousekeeper: false. This is the read for callers that only need the |
| 376 | // answer — Part 2's edge cert gate. Anything that then goes and DOES work |
| 377 | // wants housekeeperGate instead, so the work is bound to the holding. |
| 378 | func housekeeperHeld() bool { return theHousekeeper.Load().held() } |
| 379 | |
| 380 | // housekeeperGate is housekeeperHeld plus the context the duty must run |
| 381 | // under — the gate every deployment-global duty consults. |
| 382 | func housekeeperGate() (context.Context, bool) { return theHousekeeper.Load().gate() } |
| 383 | |
| 384 | // housekeeperContends reports whether this box takes part in the election at |
| 385 | // all — mode sole or elect, decided once from the environment. The edge's |
| 386 | // cert-backfill eligibility (edge.WithIssuerContender, #318): a fleet box |
| 387 | // grandfathered with the bucket env is agent-run and gated, but mode |
| 388 | // "never" — its disk holds a customer's certs, never this deployment's to |
| 389 | // publish. |
| 390 | func housekeeperContends() bool { |
| 391 | // Nil-safe like held()/gate(): several agent loops start before |
| 392 | // startHousekeeper, and a caller from one of them should read "does not |
| 393 | // contend", not panic. |
| 394 | hk := theHousekeeper.Load() |
| 395 | return hk != nil && hk.mode != housekeeperNever |
| 396 | } |
| 397 | |
| 398 | // startHousekeeper arms the election for this process and returns its |
| 399 | // stand-down. Called once from RunAgent, before the loops whose duties it |
| 400 | // gates; the returned func MUST be called on the way out (RunAgent's |
| 401 | // post-drain block) or a restart strands the lock until its TTL. |
| 402 | func startHousekeeper() (stop func()) { |
| 403 | hk := newHousekeeper(os.Getenv, acquireDeploymentLease) |
| 404 | theHousekeeper.Store(hk) |
| 405 | return hk.start(context.Background()) |
| 406 | } |
| 407 | |
| 408 | // housekeeperDuty is one deployment-global job: a name for the log and the |
| 409 | // work. Today's set is the releases retention prune; Part 2 adds certificate |
| 410 | // issuance. |
| 411 | type housekeeperDuty struct { |
| 412 | name string |
| 413 | run func(context.Context) error |
| 414 | } |
| 415 | |
| 416 | // runHousekeeperDuties runs one tick of the given duties — but only on the |
| 417 | // box holding the lease, and only under the context of THAT holding. A |
| 418 | // non-holder does nothing and says nothing: that is the normal state of every |
| 419 | // box but one, on every tick, forever. |
| 420 | // |
| 421 | // The gate is consulted once per tick, not once per duty: within a tick the |
| 422 | // context is what carries a mid-flight stand-down, and re-reading the gate |
| 423 | // between duties would just replace one cancellation mechanism with two. |
| 424 | // |
| 425 | // Per-duty isolation, the shape retentionPass already uses for per-app |
| 426 | // errors: one duty's failure is logged and the next still runs. |
| 427 | func runHousekeeperDuties(gate func() (context.Context, bool), duties ...housekeeperDuty) { |
| 428 | ctx, ok := gate() |
| 429 | if !ok { |
| 430 | return |
| 431 | } |
| 432 | for _, d := range duties { |
| 433 | // The holding can end mid-tick. What is left is not this box's to |
| 434 | // run, and the duty that was interrupted stood down — it did not |
| 435 | // fail, so it gets no error line either. |
| 436 | if ctx.Err() != nil { |
| 437 | return |
| 438 | } |
| 439 | if err := d.run(ctx); err != nil && ctx.Err() == nil { |
| 440 | log.Printf("agent: housekeeper: %s: %v", d.name, err) |
| 441 | } |
| 442 | } |
| 443 | } |
| 444 | |
| 445 | // mintMeshKeyLoop mints the deployment's mesh key (multi-region backhaul |
| 446 | // spec, Part 3 Task 2) — a housekeeper duty, but deliberately its OWN loop |
| 447 | // rather than a duty folded into retentionLoop's runHousekeeperDuties call |
| 448 | // (agent.go's retentionLoop, gated on prune's cadence). |
| 449 | // |
| 450 | // The plan first drafted the mint as riding retentionLoop's tick; review |
| 451 | // caught the trap: retentionLoop returns early — never arming its loop at |
| 452 | // all, so runHousekeeperDuties is never even called — on a box missing |
| 453 | // CARLOS_RELEASE_PUBKEY (release-signature verification, something wholly |
| 454 | // unrelated to the mesh key). Critically, a box in that state still WINS |
| 455 | // the housekeeper election (newHousekeeper/hk.run don't consult |
| 456 | // CARLOS_RELEASE_PUBKEY at all), so the one box holding the lease would |
| 457 | // simply never mint, and no other box would ever try either — the mesh key |
| 458 | // would silently never exist, and Tasks 3/4's hop auth would never arm, |
| 459 | // fleet-wide, with no error anywhere. This loop is gated on nothing but |
| 460 | // deployment-store presence (shouldSkipPublisherElection, openDeployStore), |
| 461 | // mirroring edgesfile.go's edgesFileLoop shape exactly, so minting is |
| 462 | // available on every housekeeper independent of every other loop's |
| 463 | // preconditions. |
| 464 | // |
| 465 | // mintMeshKey itself is create-only and idempotent (mesh.go's own comment): |
| 466 | // every tick after the deployment's first successful mint finds the key |
| 467 | // already there and does nothing, silently. |
| 468 | func mintMeshKeyLoop() { |
| 469 | ctx := context.Background() |
| 470 | if shouldSkipPublisherElection(os.Getenv) { |
| 471 | log.Printf("agent: mesh key mint loop is a platform-box job — skipped on a fleet box") |
| 472 | return |
| 473 | } |
| 474 | store, err := openDeployStore(ctx) |
| 475 | if err != nil { |
| 476 | log.Printf("agent: mesh key mint loop disabled: %v", err) |
| 477 | return |
| 478 | } |
| 479 | log.Printf("agent: mesh key mint loop armed (5min + jitter) — mints only while this box holds the housekeeper lease") |
| 480 | mint := housekeeperDuty{"mesh key mint", func(ctx context.Context) error { |
| 481 | return edge.MintMeshKey(ctx, store) |
| 482 | }} |
| 483 | delay := time.Minute |
| 484 | for { |
| 485 | time.Sleep(delay) |
| 486 | delay = 5*time.Minute + rand.N(time.Minute) |
| 487 | runHousekeeperDuties(housekeeperGate, mint) |
| 488 | } |
| 489 | } |
| 490 | |
amadan