amadan
host: housekeeper via the one lease mechanism; prune single-holder
The deployment's global duties get a single elected box (spec 2026-08-20-multi-region-ingress-design.md, Part 1), and the election is internal/lease.Acquire — the same conditional-write lock the status publisher has used since the fleet status page shipped. No second election pattern: no store-CAS lease, no control/housekeeper.json, no bespoke TTL. The lock lands at control/housekeeper-lease/lock.json in the deployment bucket, held long-term and renewed by internal/lease, with housekeeperHeld() a cheap atomic read for the duty ticks (and, from Part 2, the edge's cert-issuance gate). The releases retention prune is the first duty to move behind it: N boxes pruning releases/** concurrently is the bug class the spec refuses to have. Fleet boxes never contend (the shouldSkipPublisherElection predicate the retention loop already used); a CARLOS_DEPLOYMENT_DIR deployment is its own housekeeper, so dev/test boxes keep pruning; losing the election is silent, exactly as the status tick swallows contention. domains.Reconcile's every-box PurgeApp is deliberately untouched (audit finding §3).
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
3 files changed,
+737
−4
internal/host/agent.go+30 −4internal/host/housekeeper.go+303 −0internal/host/housekeeper_test.go+404 −0
diff --git a/internal/host/agent.go b/internal/host/agent.go| index b7498b778..ec245302d 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -97,6 +97,16 @@ |
| // page's entire config surface is documented in one place rather than |
| // split across this file and the tofu module. |
| // |
| +// The status-lease above is one of TWO elections this process runs, and the |
| +// only per-tick one. The other is the HOUSEKEEPER lease (housekeeper.go): |
| +// the same internal/lease mechanism, held long-term, at the fixed |
| +// deployment-bucket key control/housekeeper-lease/lock.json, electing the one |
| +// box that performs this deployment's global duties — today the releases |
| +// retention prune (retentionLoop), from Part 2 of the multi-region spec |
| +// certificate issuance. It reads CARLOS_DEPLOYMENT_BUCKET (or |
| +// CARLOS_DEPLOYMENT_DIR, which makes the box its own housekeeper) and the |
| +// same CARLOS_FLEET guard, and adds no env var of its own. |
| +// |
| // Domains config surface (read by domainsLoop / domainsTickConfig): |
| // |
| // - PLATFORM_APPS_DOMAIN — the deployment's platform apps domain |
| @@ -303,6 +313,11 @@ func RunAgent(args []string) { |
| go statusLoop(reg) |
| go domainsLoop(reg, peekAcmeCache(args)) |
| go instancesLoop(reg) |
| + // The deployment-global election, armed BEFORE the loops it gates so a |
| + // single-box deployment is holding well before their first tick (a |
| + // minute away) — see housekeeper.go. Not a `go` of its own: it publishes |
| + // the process's housekeeper and spawns its loop itself. |
| + startHousekeeper() |
| go retentionLoop(reg) |
| go registrySnapshotLoop(reg) |
| // The sweep reads the registry only through this closure — the stamped |
| @@ -1546,6 +1561,16 @@ func instancesLoop(reg *registry.Registry) { |
| // deployment config or a missing/unparseable release pubkey disables the |
| // loop with one log line — fail closed, domainsLoop's auto-purge shape: a |
| // box that cannot verify pointers must not delete anything. |
| +// |
| +// Since the multi-region writer map (spec |
| +// 2026-08-20-multi-region-ingress-design.md, Part 1) the pass itself is a |
| +// HOUSEKEEPER DUTY: it runs only on the one box holding the deployment's |
| +// housekeeper lease. releases/** is deployment-global state, and N boxes |
| +// pruning it concurrently is the bug class the spec refuses to have. Every |
| +// other box still arms this loop and still ticks — it just prunes nothing, |
| +// silently, until the day it wins the lease (housekeeper.go). On a |
| +// single-box deployment the holder is always this box, so the behaviour is |
| +// unchanged. |
| func retentionLoop(reg *registry.Registry) { |
| ctx := context.Background() |
| // Platform boxes only (spec F6). Retention prunes the PLATFORM's release |
| @@ -1576,14 +1601,15 @@ func retentionLoop(reg *registry.Registry) { |
| log.Printf("agent: retention loop disabled: %v — retention policies will not be enforced by this box", err) |
| return |
| } |
| - log.Printf("agent: retention loop armed (5min + jitter)") |
| + log.Printf("agent: retention loop armed (5min + jitter) — prunes only while this box holds the housekeeper lease") |
| + prune := housekeeperDuty{"retention pass", func(ctx context.Context) error { |
| + return retentionPass(ctx, store, pub, reg) |
| + }} |
| delay := time.Minute |
| for { |
| time.Sleep(delay) |
| delay = 5*time.Minute + rand.N(time.Minute) |
| - if err := retentionPass(ctx, store, pub, reg); err != nil { |
| - log.Printf("agent: retention pass: %v", err) |
| - } |
| + runHousekeeperDuties(ctx, housekeeperHeld, prune) |
| } |
| } |
diff --git a/internal/host/housekeeper.go b/internal/host/housekeeper.go| new file mode 100644 |
| index 000000000..9ff5cf7ec |
| --- /dev/null |
| +++ b/internal/host/housekeeper.go |
| @@ -0,0 +1,303 @@ |
| +// The housekeeper: the ONE election for this deployment's global duties |
| +// (spec 2026-08-20-multi-region-ingress-design.md, Part 1). A deployment now |
| +// has N platform boxes converging one control bucket, and the rule the writer |
| +// map states is "data that is per-box gets per-box keys; duties that are |
| +// deployment-global get a lease". The releases retention prune is the first |
| +// such duty to move behind it — five concurrent pruners racing a promote is a |
| +// bug class the spec refuses to have — and Part 2's cert issuance is the next. |
| +// |
| +// The mechanism is deliberately NOT new. It is internal/lease.Acquire, the |
| +// same conditional-write lock (litestream's lss3.Leaser writing |
| +// <prefix>/lock.json) the status publisher has elected with since the fleet |
| +// status page shipped — see statusLoop's lease.Acquire call in agent.go and |
| +// the bucket-writer audit's §1. The spec's own words: "The status publisher's |
| +// CAS-lease is the one election mechanism, reused; no second election pattern |
| +// may be introduced." So there is no store-CAS lease here, no |
| +// control/housekeeper.json record, and no bespoke TTL. |
| +// |
| +// Two things DO differ from the status election, both on purpose: |
| +// |
| +// - It is long-lived, not per-tick. The status loop acquires and releases |
| +// inside one 5-minute tick because publishing is a single act; the |
| +// housekeeper's duties are continuous and Task 4's edge cert gate has to |
| +// be answerable at any instant, so this box acquires once and RENEWS |
| +// (lease.Acquire starts the renewer itself) for as long as it can, and |
| +// housekeeperHeld() is a non-blocking read of that state. |
| +// - It leases in the DEPLOYMENT bucket at a fixed key — |
| +// control/housekeeper-lease/lock.json — not under CARLOS_STATUS_PREFIX. |
| +// The duties act on the deployment bucket's own control plane |
| +// (releases/**, later control/certs/**), and that bucket is the one |
| +// linearizable home the spec pins the control plane to. |
| +// |
| +// The accepted price is lease.LeaseTTL, 15 minutes: a HARD-crashed |
| +// housekeeper strands the lease until expiry, so handover can stall up to |
| +// ~15m. Nothing here cares. The prune is idempotent and not urgent (a policy |
| +// written now is promised "within ~10 minutes", and the loop is a 5-minute |
| +// tick anyway), and cert renewals run ~30 days early. A GRACEFUL stand-down |
| +// releases immediately. |
| +package host |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "log" |
| + "os" |
| + "sync/atomic" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/lease" |
| +) |
| + |
| +// housekeeperLeasePrefix is the deployment-bucket prefix the election's lock |
| +// lands under: the leaser writes <prefix>/lock.json, so the object is |
| +// control/housekeeper-lease/lock.json. Under control/ because it is control |
| +// plane, and a fixed key because there is exactly one housekeeper per |
| +// deployment — the same "the bucket IS the scope" property the status-lease |
| +// relies on (one control bucket per deployment, forever). |
| +const housekeeperLeasePrefix = "control/housekeeper-lease" |
| + |
| +// housekeeperRetry spaces a losing contender's attempts. Each attempt is one |
| +// conditional PUT against the lock (a GET first when a lock exists), so this |
| +// is the same request-cost trade internal/lease's LeaseTTL comment weighs: |
| +// 5 minutes is 288 attempts/day/box, in step with the status election's own |
| +// per-tick contention, and bounds takeover after a hard crash at roughly the |
| +// 15m TTL plus one retry. |
| +const housekeeperRetry = 5 * time.Minute |
| + |
| +// housekeeperPoll is how often a HOLDER checks that renewal is still winning. |
| +// Renewal itself runs inside internal/lease (TTL/3 = 5m); this only decides |
| +// how quickly a box that has LOST the lease stops believing it holds it — |
| +// which is exactly how long a stale holder could keep running duties after a |
| +// takeover. 30 seconds is cheap (a mutexed field read, no I/O at all). |
| +const housekeeperPoll = 30 * time.Second |
| + |
| +// heldLease is the slice of *lease.Held this file uses: renewal health and |
| +// release. An interface only so the election transport is injectable — the |
| +// lock's own semantics are internal/lease's to test, against a real |
| +// S3-compatible store. |
| +type heldLease interface { |
| + Err() error |
| + Release(ctx context.Context) error |
| +} |
| + |
| +// leaseAcquirer is lease.Acquire's shape, as a seam. |
| +type leaseAcquirer func(ctx context.Context, bucket, prefix, region string) (heldLease, error) |
| + |
| +// acquireDeploymentLease is the production acquirer. The region is |
| +// deliberately empty: internal/lease's resolveRegion then answers from the |
| +// AWS default chain (AWS_REGION / profile), which is precisely what |
| +// release.NewS3Store signs for against this same bucket — a housekeeper lease |
| +// signing for a different region than the store it guards would be a split |
| +// waiting to happen. No new env var: the deployment bucket has never had a |
| +// region variable of its own, and inventing one here would be a second answer |
| +// to a question the chain already answers. |
| +// |
| +// The nil-into-interface hazard is why this is a func and not a method value: |
| +// lease.Acquire returns a typed nil *lease.Held alongside its error, which |
| +// would arrive as a NON-nil heldLease if handed straight back. |
| +func acquireDeploymentLease(ctx context.Context, bucket, prefix, region string) (heldLease, error) { |
| + h, err := lease.Acquire(ctx, bucket, prefix, region) |
| + if err != nil { |
| + return nil, err |
| + } |
| + return h, nil |
| +} |
| + |
| +// housekeeperMode is this box's relationship to the election, decided once |
| +// from the environment. |
| +type housekeeperMode int |
| + |
| +const ( |
| + // housekeeperNever: this box must never run deployment-global duties — |
| + // a fleet box (a customer's machine; the platform's releases are not |
| + // its business, and the store door would refuse the deletes anyway), or |
| + // a box with no deployment store at all. |
| + housekeeperNever housekeeperMode = iota |
| + // housekeeperSole: a CARLOS_DEPLOYMENT_DIR deployment. There is no |
| + // bucket to hold a lock in, and a directory store is one box by |
| + // construction, so this box IS the housekeeper — anything else would |
| + // silently stop dev/test deployments pruning at all. |
| + housekeeperSole |
| + // housekeeperElect: a bucket-backed platform box. Contend. |
| + housekeeperElect |
| +) |
| + |
| +// housekeeper is the long-lived acquire/renew/stand-down lifecycle plus the |
| +// flag every gated caller reads. |
| +type housekeeper struct { |
| + mode housekeeperMode |
| + reason string // the one line run() logs when it arms |
| + bucket string |
| + acquire leaseAcquirer |
| + retry time.Duration |
| + poll time.Duration |
| + |
| + flag atomic.Bool |
| +} |
| + |
| +// newHousekeeper decides the mode from the environment. Nothing is acquired |
| +// here — run() does the work — EXCEPT the sole-housekeeper case, which is a |
| +// pure fact about the deployment shape and is therefore true from the moment |
| +// the box is configured, not from whenever a goroutine happens to be |
| +// scheduled. |
| +func newHousekeeper(getenv func(string) string, acquire leaseAcquirer) *housekeeper { |
| + hk := &housekeeper{acquire: acquire, retry: housekeeperRetry, poll: housekeeperPoll} |
| + switch { |
| + // Deliberately the bare CARLOS_FLEET test that retentionLoop and |
| + // instancesLoop already use, for the same reason retentionLoop states: |
| + // a fleet box whose LABEL is misconfigured is still a fleet box. |
| + case shouldSkipPublisherElection(getenv): |
| + hk.mode = housekeeperNever |
| + hk.reason = "fleet box — deployment-global duties are a platform-box job; never contending" |
| + case getenv("CARLOS_DEPLOYMENT_BUCKET") != "": |
| + // The bucket openDeployStore already adopts from — the duties act on |
| + // its contents, so the lock guarding them belongs in it. |
| + hk.mode = housekeeperElect |
| + hk.bucket = getenv("CARLOS_DEPLOYMENT_BUCKET") |
| + case getenv("CARLOS_DEPLOYMENT_DIR") != "": |
| + hk.mode = housekeeperSole |
| + hk.reason = "directory-backed deployment — this box is its own housekeeper (no bucket to elect in)" |
| + hk.flag.Store(true) |
| + default: |
| + hk.mode = housekeeperNever |
| + hk.reason = "no deployment store configured — never contending" |
| + } |
| + return hk |
| +} |
| + |
| +// held reports whether this box currently holds the lease. Cheap and |
| +// non-blocking by contract: it is read on a duty tick and, from Part 2, on |
| +// the edge's certificate path. |
| +func (hk *housekeeper) held() bool { |
| + if hk == nil { |
| + return false |
| + } |
| + return hk.flag.Load() |
| +} |
| + |
| +// run is the election loop; it returns only when ctx ends (or immediately, on |
| +// a box that never contends). |
| +// |
| +// NOT holding the lease is a normal, quiet state — it is what N-1 boxes are |
| +// in, permanently. Contention (*lease.HeldError) is therefore never logged, |
| +// exactly as statusLoop swallows it: a line per retry per box, forever, would |
| +// bury the log. A real acquire failure (credentials, a broken bucket) IS |
| +// logged, same as any other tick failure. |
| +func (hk *housekeeper) run(ctx context.Context) { |
| + if hk.mode != housekeeperElect { |
| + log.Printf("agent: housekeeper: %s", hk.reason) |
| + return |
| + } |
| + log.Printf("agent: housekeeper: contending for s3://%s/%s/lock.json (lease TTL %s)", |
| + hk.bucket, housekeeperLeasePrefix, lease.LeaseTTL) |
| + for ctx.Err() == nil { |
| + held, err := hk.acquire(ctx, hk.bucket, housekeeperLeasePrefix, "") |
| + if err != nil { |
| + var heldErr *lease.HeldError |
| + if !errors.As(err, &heldErr) && ctx.Err() == nil { |
| + log.Printf("agent: housekeeper: lease acquire: %v", err) |
| + } |
| + if !sleepCtx(ctx, hk.retry) { |
| + return |
| + } |
| + continue |
| + } |
| + log.Printf("agent: housekeeper: lease acquired — this box runs the deployment's global duties") |
| + hk.flag.Store(true) |
| + hk.hold(ctx, held) |
| + } |
| +} |
| + |
| +// hold blocks while this box is the housekeeper, and clears the flag BEFORE |
| +// it gives the lease up either way — a duty must never start against a lease |
| +// this box has already decided to let go of. |
| +// |
| +// A failed renewal means leadership is lost (internal/lease stops renewing on |
| +// the first failure, precisely so a re-try can't fence a claimant who took |
| +// over after expiry). The lease is then NOT released: our etag is stale, so |
| +// the release would just fail a precondition, and the TTL is what hands the |
| +// lock over. A graceful stand-down (ctx ended: process shutting down) DOES |
| +// release, so the next box takes over in seconds instead of ~15 minutes. |
| +func (hk *housekeeper) hold(ctx context.Context, held heldLease) { |
| + for { |
| + if !sleepCtx(ctx, hk.poll) { |
| + hk.flag.Store(false) |
| + releaseHousekeeperLease(held) |
| + return |
| + } |
| + if err := held.Err(); err != nil { |
| + hk.flag.Store(false) |
| + log.Printf("agent: housekeeper: lease renewal failed — standing down: %v", err) |
| + return |
| + } |
| + } |
| +} |
| + |
| +// releaseHousekeeperLease hands the lock back on a graceful stand-down. The |
| +// release gets its OWN budget, never the caller's: by the time this runs the |
| +// agent's context is already dead, and a release under a dead context leaves |
| +// the lock stranded until TTL — the exact trap internal/lease's |
| +// Replicator.Close names, where it cost a wake up to a full TTL of 503s. |
| +func releaseHousekeeperLease(held heldLease) { |
| + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) |
| + defer cancel() |
| + if err := held.Release(ctx); err != nil { |
| + log.Printf("agent: housekeeper: lease release: %v", err) |
| + } |
| +} |
| + |
| +// sleepCtx waits for d, reporting false if ctx ended first. |
| +func sleepCtx(ctx context.Context, d time.Duration) bool { |
| + t := time.NewTimer(d) |
| + defer t.Stop() |
| + select { |
| + case <-ctx.Done(): |
| + return false |
| + case <-t.C: |
| + return true |
| + } |
| +} |
| + |
| +// theHousekeeper is the process's one election, published by |
| +// startHousekeeper. A pointer rather than a bare flag so the loop owns its own |
| +// state and nothing else can set "held" from the side; nil (never armed) |
| +// reads as not held. |
| +var theHousekeeper atomic.Pointer[housekeeper] |
| + |
| +// housekeeperHeld reports whether THIS box currently holds the deployment's |
| +// housekeeper lease — the gate every deployment-global duty consults. Cheap, |
| +// non-blocking, and safe before (or without) startHousekeeper: false. |
| +func housekeeperHeld() bool { return theHousekeeper.Load().held() } |
| + |
| +// startHousekeeper arms the election for this process. Called once from |
| +// RunAgent, before the loops whose duties it gates. |
| +func startHousekeeper() { |
| + hk := newHousekeeper(os.Getenv, acquireDeploymentLease) |
| + theHousekeeper.Store(hk) |
| + go hk.run(context.Background()) |
| +} |
| + |
| +// housekeeperDuty is one deployment-global job: a name for the log and the |
| +// work. Today's set is the releases retention prune; Part 2 adds certificate |
| +// issuance. |
| +type housekeeperDuty struct { |
| + name string |
| + run func(context.Context) error |
| +} |
| + |
| +// runHousekeeperDuties runs one tick of the given duties — but only on the |
| +// box holding the lease. A non-holder does nothing and says nothing: that is |
| +// the normal state of every box but one, on every tick, forever. |
| +// |
| +// Per-duty isolation, the shape retentionPass already uses for per-app |
| +// errors: one duty's failure is logged and the next still runs. |
| +func runHousekeeperDuties(ctx context.Context, held func() bool, duties ...housekeeperDuty) { |
| + if !held() { |
| + return |
| + } |
| + for _, d := range duties { |
| + if err := d.run(ctx); err != nil { |
| + log.Printf("agent: housekeeper: %s: %v", d.name, err) |
| + } |
| + } |
| +} |
diff --git a/internal/host/housekeeper_test.go b/internal/host/housekeeper_test.go| new file mode 100644 |
| index 000000000..69aa4176b |
| --- /dev/null |
| +++ b/internal/host/housekeeper_test.go |
| @@ -0,0 +1,404 @@ |
| +package host |
| + |
| +// What is testable here without a live S3 is the GATING — who contends, who |
| +// runs duties, and how the box behaves when it loses or never wins. The lock |
| +// semantics themselves (a second claimant gets a *lease.HeldError; a dark |
| +// holder cannot reclaim after failover) belong to internal/lease and are |
| +// covered there against a real S3-compatible store — internal/lease's own |
| +// fake (endpoint_test.go's newFakeS3) answers every GET with NoSuchKey, so it |
| +// proves where the traffic goes and nothing about contention. This file |
| +// deliberately invents no second fake S3: the election transport is injected |
| +// as a seam instead. |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "fmt" |
| + "io" |
| + "log" |
| + "os" |
| + "strings" |
| + "sync" |
| + "sync/atomic" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/lease" |
| +) |
| + |
| +// ---- helpers --------------------------------------------------------------- |
| + |
| +// lockedBuffer is an io.Writer safe for the election goroutine and the test |
| +// body to touch at once — log output is the assertion in the "stays quiet" |
| +// tests, so it has to be read while the loop is still running. |
| +type lockedBuffer struct { |
| + mu sync.Mutex |
| + b strings.Builder |
| +} |
| + |
| +func (l *lockedBuffer) Write(p []byte) (int, error) { |
| + l.mu.Lock() |
| + defer l.mu.Unlock() |
| + return l.b.Write(p) |
| +} |
| + |
| +func (l *lockedBuffer) String() string { |
| + l.mu.Lock() |
| + defer l.mu.Unlock() |
| + return l.b.String() |
| +} |
| + |
| +// captureRacyLog redirects the standard logger for one test, through a |
| +// mutexed buffer: unlike this package's other captureLog, the writer here is |
| +// shared with a live election goroutine. |
| +func captureRacyLog(t *testing.T) *lockedBuffer { |
| + t.Helper() |
| + buf := &lockedBuffer{} |
| + log.SetOutput(buf) |
| + t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| + return buf |
| +} |
| + |
| +// quietLog silences the standard logger for a test that doesn't assert on it. |
| +func quietLog(t *testing.T) { |
| + t.Helper() |
| + log.SetOutput(io.Discard) |
| + t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| +} |
| + |
| +// fakeLease stands in for *lease.Held: renewal health is a func the test |
| +// flips, and releases are counted. |
| +type fakeLease struct { |
| + mu sync.Mutex |
| + err error |
| + released int |
| +} |
| + |
| +func (f *fakeLease) fail(err error) { |
| + f.mu.Lock() |
| + defer f.mu.Unlock() |
| + f.err = err |
| +} |
| + |
| +func (f *fakeLease) Err() error { |
| + f.mu.Lock() |
| + defer f.mu.Unlock() |
| + return f.err |
| +} |
| + |
| +func (f *fakeLease) Release(context.Context) error { |
| + f.mu.Lock() |
| + defer f.mu.Unlock() |
| + f.released++ |
| + return nil |
| +} |
| + |
| +func (f *fakeLease) releases() int { |
| + f.mu.Lock() |
| + defer f.mu.Unlock() |
| + return f.released |
| +} |
| + |
| +type acquireCall struct{ bucket, prefix, region string } |
| + |
| +// acquirer records every acquisition attempt and answers from a per-attempt |
| +// func, so a test can script "contended, contended, won". |
| +type acquirer struct { |
| + mu sync.Mutex |
| + calls []acquireCall |
| + answer func(n int) (heldLease, error) |
| +} |
| + |
| +func (a *acquirer) acquire(_ context.Context, bucket, prefix, region string) (heldLease, error) { |
| + a.mu.Lock() |
| + a.calls = append(a.calls, acquireCall{bucket, prefix, region}) |
| + n := len(a.calls) |
| + answer := a.answer |
| + a.mu.Unlock() |
| + return answer(n) |
| +} |
| + |
| +func (a *acquirer) count() int { |
| + a.mu.Lock() |
| + defer a.mu.Unlock() |
| + return len(a.calls) |
| +} |
| + |
| +func (a *acquirer) first() acquireCall { |
| + a.mu.Lock() |
| + defer a.mu.Unlock() |
| + return a.calls[0] |
| +} |
| + |
| +// contended is what a losing contender sees, wrapped exactly as |
| +// lease.Acquire wraps it. |
| +func contended() error { |
| + return fmt.Errorf("acquire lease: %w", &lease.HeldError{Owner: "other-box", ExpiresAt: time.Now().Add(time.Minute)}) |
| +} |
| + |
| +func waitFor(t *testing.T, what string, cond func() bool) { |
| + t.Helper() |
| + deadline := time.Now().Add(3 * time.Second) |
| + for time.Now().Before(deadline) { |
| + if cond() { |
| + return |
| + } |
| + time.Sleep(time.Millisecond) |
| + } |
| + t.Fatalf("timed out waiting for %s", what) |
| +} |
| + |
| +// runElection starts hk.run in the background with fast timings and returns a |
| +// stop func that cancels it and waits for the loop to exit. |
| +func runElection(t *testing.T, hk *housekeeper) func() { |
| + t.Helper() |
| + ctx, cancel := context.WithCancel(context.Background()) |
| + done := make(chan struct{}) |
| + go func() { |
| + defer close(done) |
| + hk.run(ctx) |
| + }() |
| + stop := func() { |
| + cancel() |
| + select { |
| + case <-done: |
| + case <-time.After(3 * time.Second): |
| + t.Error("election loop did not exit after cancel") |
| + } |
| + } |
| + t.Cleanup(stop) |
| + return stop |
| +} |
| + |
| +func fastHousekeeper(getenv func(string) string, a *acquirer) *housekeeper { |
| + hk := newHousekeeper(getenv, a.acquire) |
| + hk.retry = time.Millisecond |
| + hk.poll = time.Millisecond |
| + return hk |
| +} |
| + |
| +// ---- duty gating ----------------------------------------------------------- |
| + |
| +func TestRunHousekeeperDutiesRunsEachDutyOncePerTickWhenHeld(t *testing.T) { |
| + quietLog(t) |
| + a, b := 0, 0 |
| + runHousekeeperDuties(context.Background(), func() bool { return true }, |
| + housekeeperDuty{"a", func(context.Context) error { a++; return nil }}, |
| + housekeeperDuty{"b", func(context.Context) error { b++; return nil }}, |
| + ) |
| + if a != 1 || b != 1 { |
| + t.Fatalf("duty runs = (%d, %d), want (1, 1)", a, b) |
| + } |
| +} |
| + |
| +func TestRunHousekeeperDutiesSkipsQuietlyWhenNotHeld(t *testing.T) { |
| + buf := captureRacyLog(t) |
| + ran := 0 |
| + runHousekeeperDuties(context.Background(), func() bool { return false }, |
| + housekeeperDuty{"retention pass", func(context.Context) error { ran++; return nil }}, |
| + ) |
| + if ran != 0 { |
| + t.Fatalf("duty ran %d times on a non-holder, want 0", ran) |
| + } |
| + // Not holding the lease is the NORMAL state on every box but one, every |
| + // tick, forever: a log line here would bury the log. |
| + if out := buf.String(); out != "" { |
| + t.Fatalf("non-holder logged %q, want silence", out) |
| + } |
| +} |
| + |
| +func TestRunHousekeeperDutiesLogsOneDutysErrorAndRunsTheRest(t *testing.T) { |
| + buf := captureRacyLog(t) |
| + later := 0 |
| + runHousekeeperDuties(context.Background(), func() bool { return true }, |
| + housekeeperDuty{"retention pass", func(context.Context) error { return errors.New("boom") }}, |
| + housekeeperDuty{"later duty", func(context.Context) error { later++; return nil }}, |
| + ) |
| + if later != 1 { |
| + t.Fatalf("later duty ran %d times, want 1 — one duty's error must not stop the next", later) |
| + } |
| + out := buf.String() |
| + if !strings.Contains(out, "retention pass") || !strings.Contains(out, "boom") { |
| + t.Fatalf("log = %q, want the failing duty's name and error", out) |
| + } |
| +} |
| + |
| +// ---- who contends ---------------------------------------------------------- |
| + |
| +func TestHousekeeperFleetBoxNeverAttemptsAcquisition(t *testing.T) { |
| + quietLog(t) |
| + a := &acquirer{answer: func(int) (heldLease, error) { return &fakeLease{}, nil }} |
| + hk := fastHousekeeper(envFunc(map[string]string{ |
| + "CARLOS_FLEET": "acme", |
| + "CARLOS_DEPLOYMENT_BUCKET": "dep-bucket", |
| + }), a) |
| + |
| + // run returns immediately on a fleet box — nothing to loop over. |
| + hk.run(context.Background()) |
| + |
| + if a.count() != 0 { |
| + t.Fatalf("fleet box attempted acquisition %d times, want 0", a.count()) |
| + } |
| + if hk.held() { |
| + t.Fatal("fleet box reports holding the housekeeper lease") |
| + } |
| +} |
| + |
| +func TestHousekeeperWithNoDeploymentStoreNeverHolds(t *testing.T) { |
| + quietLog(t) |
| + a := &acquirer{answer: func(int) (heldLease, error) { return &fakeLease{}, nil }} |
| + hk := fastHousekeeper(envFunc(map[string]string{}), a) |
| + |
| + hk.run(context.Background()) |
| + |
| + if a.count() != 0 { |
| + t.Fatalf("storeless box attempted acquisition %d times, want 0", a.count()) |
| + } |
| + if hk.held() { |
| + t.Fatal("storeless box reports holding the housekeeper lease") |
| + } |
| +} |
| + |
| +func TestHousekeeperDirDeploymentIsItsOwnHousekeeper(t *testing.T) { |
| + quietLog(t) |
| + a := &acquirer{answer: func(int) (heldLease, error) { return &fakeLease{}, nil }} |
| + hk := fastHousekeeper(envFunc(map[string]string{"CARLOS_DEPLOYMENT_DIR": t.TempDir()}), a) |
| + |
| + hk.run(context.Background()) |
| + |
| + if a.count() != 0 { |
| + t.Fatalf("dir deployment attempted acquisition %d times, want 0 — there is no bucket to elect in", a.count()) |
| + } |
| + if !hk.held() { |
| + t.Fatal("dir deployment does not hold: a directory-backed deployment is one box by construction and must keep pruning") |
| + } |
| +} |
| + |
| +// ---- the election ---------------------------------------------------------- |
| + |
| +func TestHousekeeperAcquiresTheDeploymentLeaseAndHolds(t *testing.T) { |
| + quietLog(t) |
| + a := &acquirer{answer: func(int) (heldLease, error) { return &fakeLease{}, nil }} |
| + hk := fastHousekeeper(envFunc(map[string]string{"CARLOS_DEPLOYMENT_BUCKET": "dep-bucket"}), a) |
| + runElection(t, hk) |
| + |
| + waitFor(t, "the lease to be held", hk.held) |
| + |
| + got := a.first() |
| + want := acquireCall{bucket: "dep-bucket", prefix: housekeeperLeasePrefix, region: ""} |
| + if got != want { |
| + t.Fatalf("acquire call = %+v, want %+v", got, want) |
| + } |
| + if housekeeperLeasePrefix != "control/housekeeper-lease" { |
| + t.Fatalf("lease prefix = %q, want control/housekeeper-lease (lock.json lands under it)", housekeeperLeasePrefix) |
| + } |
| +} |
| + |
| +func TestHousekeeperLosingTheElectionIsQuietAndKeepsTrying(t *testing.T) { |
| + buf := captureRacyLog(t) |
| + a := &acquirer{answer: func(int) (heldLease, error) { return nil, contended() }} |
| + hk := fastHousekeeper(envFunc(map[string]string{"CARLOS_DEPLOYMENT_BUCKET": "dep-bucket"}), a) |
| + // The arming line is not what this test is about; drop everything logged |
| + // before the loop starts contending. |
| + runElection(t, hk) |
| + |
| + waitFor(t, "several acquisition attempts", func() bool { return a.count() >= 3 }) |
| + |
| + if hk.held() { |
| + t.Fatal("a losing contender reports holding the lease") |
| + } |
| + // Contention is normal on every tick a box doesn't win: an error loop |
| + // here would be a line every retry, forever, on N-1 boxes. |
| + for _, line := range strings.Split(strings.TrimSpace(buf.String()), "\n") { |
| + if strings.Contains(line, "lease acquire") { |
| + t.Fatalf("contention logged as a failure: %q", line) |
| + } |
| + } |
| +} |
| + |
| +func TestHousekeeperLogsARealAcquireFailure(t *testing.T) { |
| + buf := captureRacyLog(t) |
| + a := &acquirer{answer: func(int) (heldLease, error) { return nil, errors.New("no credentials") }} |
| + hk := fastHousekeeper(envFunc(map[string]string{"CARLOS_DEPLOYMENT_BUCKET": "dep-bucket"}), a) |
| + runElection(t, hk) |
| + |
| + waitFor(t, "the failure to be logged", func() bool { return strings.Contains(buf.String(), "no credentials") }) |
| + |
| + if hk.held() { |
| + t.Fatal("a box that could not acquire reports holding the lease") |
| + } |
| +} |
| + |
| +func TestHousekeeperStandsDownWhenRenewalFails(t *testing.T) { |
| + quietLog(t) |
| + first := &fakeLease{} |
| + third := &fakeLease{} |
| + // The takeover, scripted: this box holds, loses renewal, and finds the |
| + // lock already taken by whoever won it after expiry — for as long as the |
| + // test says so. That middle state is what makes "stood down" observable |
| + // at all: if a lease were waiting on the very next attempt, the box would |
| + // be holding again within a retry and a housekeeper that never cleared |
| + // its flag would look identical to one that did. |
| + var freed atomic.Bool |
| + a := &acquirer{answer: func(n int) (heldLease, error) { |
| + switch { |
| + case n == 1: |
| + return first, nil |
| + case !freed.Load(): |
| + return nil, contended() |
| + default: |
| + return third, nil |
| + } |
| + }} |
| + hk := fastHousekeeper(envFunc(map[string]string{"CARLOS_DEPLOYMENT_BUCKET": "dep-bucket"}), a) |
| + runElection(t, hk) |
| + |
| + waitFor(t, "the first lease to be held", hk.held) |
| + |
| + // Renewal fails: leadership is lost, so this box must stop reporting |
| + // itself as the housekeeper — a stale holder still running duties (or, |
| + // from Part 2, still issuing certs) against a lease another box now owns |
| + // is the whole reason this check exists. |
| + first.fail(errors.New("renew: precondition failed")) |
| + waitFor(t, "the stand-down", func() bool { return !hk.held() }) |
| + waitFor(t, "a fresh acquisition attempt", func() bool { return a.count() > 1 }) |
| + |
| + // A lost lease is NOT released: our etag is stale, and the TTL is what |
| + // hands it over. |
| + if first.releases() != 0 { |
| + t.Fatalf("released a lost lease %d times, want 0", first.releases()) |
| + } |
| + |
| + // And it recovers on its own once the lock frees up again. |
| + freed.Store(true) |
| + waitFor(t, "the lease to be won back", hk.held) |
| + if third.releases() != 0 { |
| + t.Fatalf("released the recovered lease %d times, want 0 while still holding it", third.releases()) |
| + } |
| +} |
| + |
| +func TestHousekeeperReleasesTheLeaseOnShutdown(t *testing.T) { |
| + quietLog(t) |
| + held := &fakeLease{} |
| + a := &acquirer{answer: func(int) (heldLease, error) { return held, nil }} |
| + hk := fastHousekeeper(envFunc(map[string]string{"CARLOS_DEPLOYMENT_BUCKET": "dep-bucket"}), a) |
| + stop := runElection(t, hk) |
| + |
| + waitFor(t, "the lease to be held", hk.held) |
| + stop() |
| + |
| + if hk.held() { |
| + t.Fatal("still reports holding the lease after shutdown") |
| + } |
| + if held.releases() != 1 { |
| + t.Fatalf("releases = %d, want 1 — a graceful stand-down hands the lease over immediately", held.releases()) |
| + } |
| +} |
| + |
| +// ---- the process-wide flag ------------------------------------------------- |
| + |
| +func TestHousekeeperHeldIsFalseBeforeAnyElectionIsArmed(t *testing.T) { |
| + if housekeeperHeld() { |
| + t.Fatal("housekeeperHeld() is true with no housekeeper armed") |
| + } |
| +} |