amadan
host: the domains reconcile is a housekeeper duty — one writer per deployment
Since multi-region every platform box carries PLATFORM_APPS_DOMAIN (the per-account wildcards and the steering probe need it), which also armed the domains loop on every box: N reconcilers racing over the same custom-domain/pointing/purge state, each judging 'pointing' against its OWN PLATFORM_EDGE_IPS. Found live on the first three-region flagship roll (2026-08-24). The pass now runs through runHousekeeperDuties like steering and the mesh-key mint: single-box deployments are unchanged (the one box always holds the lease); a box that never contends never reconciles, which is right for a platform-box job.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2 files changed,
+79
−4
internal/host/agent.go+25 −4internal/host/domains_duty_test.go+54 −0
diff --git a/internal/host/agent.go b/internal/host/agent.go| index fab5d5121..4f12d4a93 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -1629,7 +1629,7 @@ func domainsLoop(reg *registry.Registry, acmeCacheFlag string) { |
| log.Printf("agent: domains loop disabled: %v", err) |
| return |
| } |
| - log.Printf("agent: domains loop armed (5min + jitter), apps domain %s, edge IPs %v", cfg.AppsDomain, cfg.EdgeIPs) |
| + log.Printf("agent: domains loop armed (5min + jitter), apps domain %s, edge IPs %v — reconciles only while this box holds the housekeeper lease", cfg.AppsDomain, cfg.EdgeIPs) |
| // One line, and it must name the REAL cause: a typo'd key and an absent |
| // one are different problems with different fixes, and this is the only |
| // place an operator learns that purging is not happening. |
| @@ -1638,16 +1638,37 @@ func domainsLoop(reg *registry.Registry, acmeCacheFlag string) { |
| } else if cfg.ReleasePub == nil { |
| log.Printf("agent: domains loop auto-purge disabled: CARLOS_RELEASE_PUBKEY is not set — trashed apps will be hidden but never destroyed") |
| } |
| + duty := domainsDuty(func(ctx context.Context) error { |
| + return domains.Reconcile(ctx, store, reg, cfg) |
| + }) |
| delay := time.Minute |
| for { |
| time.Sleep(delay) |
| delay = 5*time.Minute + rand.N(time.Minute) |
| - if err := domains.Reconcile(ctx, store, reg, cfg); err != nil { |
| - log.Printf("agent: domains pass: %v", err) |
| - } |
| + runHousekeeperDuties(housekeeperGate, duty) |
| } |
| } |
| +// domainsDuty wraps one domains reconcile pass as a housekeeper duty. The |
| +// pass is deployment-wide state — custom-domain `pointing` stamps, cert |
| +// expiry stamps, purging trashed apps — with one correct writer, and since |
| +// multi-region (spec 2026-08-20) every platform box in a deployment carries |
| +// PLATFORM_APPS_DOMAIN (the per-account wildcards and the steering probe |
| +// need it), so an ungated loop would be N reconcilers racing over the same |
| +// records, each judging `pointing` against its OWN edge-IP list. Gating on |
| +// the lease is what makes the env safe to set fleet-wide. Single-box |
| +// deployments are unchanged: the one box always holds the lease. A box that |
| +// never contends (a fleet box) never runs the pass — which is right, the |
| +// pass is a platform-box job. |
| +func domainsDuty(pass func(context.Context) error) housekeeperDuty { |
| + return housekeeperDuty{"domains reconcile", func(ctx context.Context) error { |
| + if pass == nil { |
| + return nil |
| + } |
| + return pass(ctx) |
| + }} |
| +} |
| + |
| // instancesLoop runs the instance-provisioning reconcile pass forever on the |
| // same cadence as domainsLoop (5min + jitter, first tick after 1 minute) — |
| // turning bucket instance records into registry routes for THIS agent's |
diff --git a/internal/host/domains_duty_test.go b/internal/host/domains_duty_test.go| new file mode 100644 |
| index 000000000..9c84d1823 |
| --- /dev/null |
| +++ b/internal/host/domains_duty_test.go |
| @@ -0,0 +1,54 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "testing" |
| +) |
| + |
| +// The domains pass is a housekeeper duty since multi-region: N platform boxes |
| +// carry PLATFORM_APPS_DOMAIN, and only the lease holder may reconcile |
| +// deployment-wide domain state (agent.go, domainsDuty). |
| + |
| +func TestDomainsDutyRunsThePassAndPropagatesItsError(t *testing.T) { |
| + quietLog(t) |
| + ran := 0 |
| + boom := errors.New("reconcile failed") |
| + duty := domainsDuty(func(context.Context) error { ran++; return boom }) |
| + |
| + if err := duty.run(context.Background()); !errors.Is(err, boom) { |
| + t.Fatalf("duty error = %v, want the pass's error propagated to the housekeeper log", err) |
| + } |
| + if ran != 1 { |
| + t.Fatalf("pass ran %d times, want 1", ran) |
| + } |
| + if duty.name != "domains reconcile" { |
| + t.Fatalf("duty name = %q, want the housekeeper log label", duty.name) |
| + } |
| +} |
| + |
| +func TestDomainsDutyWithNoPassDeclines(t *testing.T) { |
| + quietLog(t) |
| + if err := domainsDuty(nil).run(context.Background()); err != nil { |
| + t.Fatalf("duty with a nil pass returned %v, want nil (and no panic)", err) |
| + } |
| +} |
| + |
| +// The gate pin: a box that does not hold the lease never runs the pass — |
| +// that is what makes PLATFORM_APPS_DOMAIN safe to set on every box. |
| +func TestDomainsDutyNeverRunsWhenLeaseNotHeld(t *testing.T) { |
| + quietLog(t) |
| + ran := 0 |
| + duty := domainsDuty(func(context.Context) error { ran++; return nil }) |
| + notHeld := func() (context.Context, bool) { return nil, false } |
| + held := func() (context.Context, bool) { return context.Background(), true } |
| + |
| + runHousekeeperDuties(notHeld, duty) |
| + if ran != 0 { |
| + t.Fatalf("pass ran %d times without the lease, want 0", ran) |
| + } |
| + runHousekeeperDuties(held, duty) |
| + if ran != 1 { |
| + t.Fatalf("pass ran %d times with the lease, want 1", ran) |
| + } |
| +} |