amadan
domains: the per-box health stamps run on every box again (#303)
#289 gated the whole domains pass to the housekeeper because sections 1-3 (materialization from claims, trashed-app purge, platform-domain minting, canary hosts) are deployment-global. Sections 4-5 — the DNS pointing/elsewhere sweep and the cert_expires_at stamp — write THIS box's registry from this box's own resolver and cache, and had no business behind the lease: since the d6cd0e9 roll every non-housekeeper box's route and alias health froze. StampHealth is now its own pass. The agent runs it on every box every tick, then the gated duty; Reconcile still ends with it, so a single-box deployment and the tests that pin its stamps are unchanged.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
3 files changed,
+77
−1
internal/domains/reconcile.go+30 −0internal/domains/reconcile_test.go+38 −0internal/host/agent.go+9 −1
diff --git a/internal/domains/reconcile.go b/internal/domains/reconcile.go| index 235b09a5e..46252c86f 100644 |
| --- a/internal/domains/reconcile.go |
| +++ b/internal/domains/reconcile.go |
| @@ -406,6 +406,35 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg |
| } |
| routes = kept |
| + // Sections 4 and 5 — the per-box health stamps — live in StampHealth |
| + // (#303): they write THIS box's registry and must run on every box, not |
| + // only on the housekeeper this pass is gated to. Reconcile still runs |
| + // them, so a single-box deployment's behaviour and the tests that pin |
| + // it are unchanged; the agent runs StampHealth on every tick besides. |
| + return StampHealth(ctx, reg, cfg, routes) |
| +} |
| + |
| +// StampHealth is the per-box half of the domains pass: DNS health (section |
| +// 4, "pointing"/"elsewhere"/"" on routes and aliases) and TLS health |
| +// (section 5, cert_expires_at). Both stamp the LOCAL registry from a local |
| +// view — this box's resolver, this box's autocert cache — so they are |
| +// every box's job on every tick, with no lease involved: gating them to |
| +// the housekeeper (#289) froze every other box's stamps (#303), and the |
| +// edges-file publisher's "pointing" bound (#287) needs the HOME box's own |
| +// row stamped. |
| +// |
| +// routes may be nil, in which case the registry is listed here; Reconcile |
| +// passes the rows it already holds. |
| +func StampHealth(ctx context.Context, reg *registry.Registry, cfg Config, routes []registry.Route) error { |
| + if cfg.Now == nil { |
| + cfg.Now = time.Now |
| + } |
| + if routes == nil { |
| + var err error |
| + if routes, err = reg.List(); err != nil { |
| + return fmt.Errorf("domains: list routes: %w", err) |
| + } |
| + } |
| // aliases are the extra hostnames route_aliases holds (Task 2/#151), |
| // health-stamped by sections 4 and 5 below exactly like the routes they |
| // point at. Listed ONCE, above both sections, and guarded the same way |
| @@ -413,6 +442,7 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg |
| // AcmeDir configured pays no aliases query at all. |
| var aliases []registry.RouteAlias |
| if cfg.Resolve != nil || cfg.AcmeDir != "" { |
| + var err error |
| aliases, err = reg.ListAliases() |
| if err != nil { |
| return fmt.Errorf("domains: list aliases: %w", err) |
diff --git a/internal/domains/reconcile_test.go b/internal/domains/reconcile_test.go| index ab3b8b652..b5b475b2f 100644 |
| --- a/internal/domains/reconcile_test.go |
| +++ b/internal/domains/reconcile_test.go |
| @@ -1183,3 +1183,41 @@ func TestReconcileDoesNotDropRoutesWhenTheProductionFlagIsUnreadable(t *testing. |
| t.Errorf("no refusal logged; log was:\n%s", logged.String()) |
| } |
| } |
| + |
| +// #303: the health stamps are a per-box pass with no lease involved — |
| +// callable on their own, stamping routes AND aliases from this box's view, |
| +// and a no-op with nothing configured. |
| +func TestStampHealthStandsAlone(t *testing.T) { |
| + reg := regWith(t, registry.Route{Host: "app.acct.oncarlos.test", Kind: "service", Addr: "10.0.0.1:1", Status: "ready"}) |
| + if err := reg.UpsertAlias(registry.RouteAlias{Alias: "www.customer.test", Host: "app.acct.oncarlos.test", Account: "acct", App: "app"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + cfg := domains.Config{ |
| + AppsDomain: "oncarlos.test", |
| + EdgeIPs: []string{"203.0.113.7"}, |
| + Resolve: func(host string) ([]string, error) { |
| + if host == "www.customer.test" { |
| + return []string{"203.0.113.7"}, nil |
| + } |
| + return []string{"198.51.100.1"}, nil |
| + }, |
| + } |
| + if err := domains.StampHealth(context.Background(), reg, cfg, nil); err != nil { |
| + t.Fatal(err) |
| + } |
| + if got := mustGet(t, reg, "app.acct.oncarlos.test"); got.DNSStatus != "elsewhere" { |
| + t.Errorf("route dns_status = %q; want elsewhere", got.DNSStatus) |
| + } |
| + aliases, err := reg.ListAliases() |
| + if err != nil || len(aliases) != 1 || aliases[0].DNSStatus != "pointing" { |
| + t.Errorf("alias dns_status: %+v, %v; want pointing", aliases, err) |
| + } |
| + // Nothing configured: nothing stamped, no error. |
| + reg2 := regWith(t, registry.Route{Host: "x.test", Kind: "service", Addr: "10.0.0.1:1", Status: "ready"}) |
| + if err := domains.StampHealth(context.Background(), reg2, domains.Config{AppsDomain: "oncarlos.test"}, nil); err != nil { |
| + t.Fatal(err) |
| + } |
| + if got := mustGet(t, reg2, "x.test"); got.DNSStatus != "" || got.DNSCheckedAt != 0 { |
| + t.Errorf("unconfigured pass stamped %+v", got) |
| + } |
| +} |
diff --git a/internal/host/agent.go b/internal/host/agent.go| index 4f12d4a93..cfe329fc6 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -1629,7 +1629,7 @@ func domainsLoop(reg *registry.Registry, acmeCacheFlag string) { |
| log.Printf("agent: domains loop disabled: %v", err) |
| return |
| } |
| - log.Printf("agent: domains loop armed (5min + jitter), apps domain %s, edge IPs %v — reconciles only while this box holds the housekeeper lease", cfg.AppsDomain, cfg.EdgeIPs) |
| + log.Printf("agent: domains loop armed (5min + jitter), apps domain %s, edge IPs %v — health stamps every tick; claims reconcile only while this box holds the housekeeper lease", cfg.AppsDomain, cfg.EdgeIPs) |
| // One line, and it must name the REAL cause: a typo'd key and an absent |
| // one are different problems with different fixes, and this is the only |
| // place an operator learns that purging is not happening. |
| @@ -1645,6 +1645,14 @@ func domainsLoop(reg *registry.Registry, acmeCacheFlag string) { |
| for { |
| time.Sleep(delay) |
| delay = 5*time.Minute + rand.N(time.Minute) |
| + // The per-box half first, on EVERY box (#303): DNS and TLS health |
| + // stamp this box's own registry from this box's own view, and no |
| + // lease is involved. Then the deployment-global half, gated. The |
| + // housekeeper stamps twice a tick (Reconcile runs the same pass |
| + // last) — idempotent, and cheaper than a special case. |
| + if err := domains.StampHealth(ctx, reg, cfg, nil); err != nil { |
| + log.Printf("agent: domains health: %v", err) |
| + } |
| runHousekeeperDuties(housekeeperGate, duty) |
| } |
| } |