amadan

carlos / platform Public

Merge pull request #304 from carlosframework/domains-health-per-box

domains: the per-box health stamps run on every box again (#303)
Paul Campbell pushed by paul@keymail.dev 6340c9e1e742cb4f8d426b1a50dad7e3be6785e3 parent e9628b7parent a7c6763
4 files changed, +190 −10
  • e2e_test.go +7 −2
  • internal/domains/reconcile.go +51 −0
  • internal/domains/reconcile_test.go +109 −0
  • internal/host/agent.go +23 −8
diff --git a/e2e_test.go b/e2e_test.go
index 9e63d7cb3..e0928d1b6 100644
--- a/e2e_test.go
+++ b/e2e_test.go
@@ -418,14 +418,19 @@ func TestEndToEndSmoke(t *testing.T) {
// echoapp was shipped under ops above, so ops is where it mints; the
// two-account case is TestMultiAccountEndToEnd.
acctSqid := release.OpsAccount
- if err := domains.Reconcile(ctx, store, reg, domains.Config{
+ dcfg := domains.Config{
AppsDomain: "oncarlos.test",
EdgeIPs: []string{"203.0.113.7"},
Resolve: func(host string) ([]string, error) { return []string{"203.0.113.7"}, nil },
Now: time.Now,
- }); err != nil {
+ }
+ if err := domains.Reconcile(ctx, store, reg, dcfg); err != nil {
t.Fatalf("domains.Reconcile: %v", err)
}
+ // The per-box health pass follows the reconcile, as domainsLoop runs it (#303).
+ if err := domains.StampHealth(ctx, reg, dcfg, nil); err != nil {
+ t.Fatalf("domains.StampHealth: %v", err)
+ }
platformHost := "echoapp." + acctSqid + ".oncarlos.test"
prt, err := reg.Get(platformHost)
if err != nil {
diff --git a/internal/domains/reconcile.go b/internal/domains/reconcile.go
index 235b09a5e..dae29e6bc 100644
--- a/internal/domains/reconcile.go
+++ b/internal/domains/reconcile.go
@@ -406,6 +406,43 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
}
routes = kept
+ // Sections 4 and 5 — the per-box health stamps — live in StampHealth
+ // (#303): they write THIS box's registry and must run on every box, not
+ // only on the housekeeper this pass is gated to. The agent runs it once
+ // per tick after this pass, on every box; this pass does not, so the
+ // housekeeper stamps once like everyone else.
+ return nil
+}
+
+// StampHealth is the per-box half of the domains pass: DNS health (section
+// 4, "pointing"/"elsewhere"/"" on routes and aliases) and TLS health
+// (section 5, cert_expires_at). Both stamp the LOCAL registry from a local
+// view — this box's resolver, this box's autocert cache — so they are
+// every box's job on every tick, with no lease involved: gating them to
+// the housekeeper (#289) froze every other box's stamps (#303), and the
+// edges-file publisher's "pointing" bound (#287) needs the HOME box's own
+// row stamped.
+//
+// Housekeeper or not, the stamps are what the console shows: a home box's
+// status snapshot carries them, and a non-housekeeper home box would show
+// every attached hostname as un-pointing forever (#303) — and the edges-file
+// publisher's "pointing" bound (#287) reads the home box's own row.
+//
+// routes may be nil, in which case the registry is listed here. Inert with
+// no AppsDomain, per Config's contract.
+func StampHealth(ctx context.Context, reg *registry.Registry, cfg Config, routes []registry.Route) error {
+ if cfg.AppsDomain == "" {
+ return nil
+ }
+ if cfg.Now == nil {
+ cfg.Now = time.Now
+ }
+ if routes == nil {
+ var err error
+ if routes, err = reg.List(); err != nil {
+ return fmt.Errorf("domains: list routes: %w", err)
+ }
+ }
// aliases are the extra hostnames route_aliases holds (Task 2/#151),
// health-stamped by sections 4 and 5 below exactly like the routes they
// point at. Listed ONCE, above both sections, and guarded the same way
@@ -413,6 +450,7 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
// AcmeDir configured pays no aliases query at all.
var aliases []registry.RouteAlias
if cfg.Resolve != nil || cfg.AcmeDir != "" {
+ var err error
aliases, err = reg.ListAliases()
if err != nil {
return fmt.Errorf("domains: list aliases: %w", err)
@@ -426,14 +464,27 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
edge[ip] = true
}
now := cfg.Now().Unix()
+ // A resolver failure is UNKNOWN (""), and unknown never overwrites a
+ // verdict: one bad DNS tick must not flip a working domain to
+ // "never checked" — the edges-file publisher gates on "pointing"
+ // (#287), so that flip would unpublish every alias fleet-wide for a
+ // cadence. The stamp time is left alone too, so staleness stays
+ // visible (dns_checked_at stops advancing) rather than being
+ // laundered into a fresh unknown.
for _, rt := range routes {
status := dnsStatus(cfg.Resolve, rt.Host, edge)
+ if status == "" && rt.DNSStatus != "" {
+ continue
+ }
if err := reg.SetDNSStatus(rt.Host, status, now); err != nil {
log.Printf("domains: stamp dns on %s: %v", rt.Host, err)
}
}
for _, a := range aliases {
status := dnsStatus(cfg.Resolve, a.Alias, edge)
+ if status == "" && a.DNSStatus != "" {
+ continue
+ }
if err := reg.SetAliasDNSStatus(a.Alias, status, now); err != nil {
log.Printf("domains: stamp dns on alias %s: %v", a.Alias, err)
}
diff --git a/internal/domains/reconcile_test.go b/internal/domains/reconcile_test.go
index ab3b8b652..f90fcdaab 100644
--- a/internal/domains/reconcile_test.go
+++ b/internal/domains/reconcile_test.go
@@ -541,6 +541,9 @@ func TestReconcileStampsDNS(t *testing.T) {
if err := domains.Reconcile(ctx, s, reg, c); err != nil {
t.Fatal(err)
}
+ if err := domains.StampHealth(ctx, reg, c, nil); err != nil { // as domainsLoop does after the pass
+ t.Fatal(err)
+ }
now := c.Now().Unix()
for host, want := range map[string]string{
"points.test": "pointing",
@@ -612,6 +615,9 @@ func TestReconcileStampsCertExpiry(t *testing.T) {
if err := domains.Reconcile(ctx, s, reg, c); err != nil {
t.Fatal(err)
}
+ if err := domains.StampHealth(ctx, reg, c, nil); err != nil { // as domainsLoop does after the pass
+ t.Fatal(err)
+ }
if got := mustGet(t, reg, "kass.training"); got.CertExpiresAt != notAfter.Unix() {
t.Errorf("kass.training cert_expires_at = %d, want %d", got.CertExpiresAt, notAfter.Unix())
}
@@ -664,6 +670,9 @@ func TestReconcileStampsAliasDNS(t *testing.T) {
if err := domains.Reconcile(ctx, s, reg, c); err != nil {
t.Fatal(err)
}
+ if err := domains.StampHealth(ctx, reg, c, nil); err != nil { // as domainsLoop does after the pass
+ t.Fatal(err)
+ }
now := c.Now().Unix()
for alias, want := range map[string]string{
"points-alias.test": "pointing",
@@ -696,6 +705,9 @@ func TestReconcileStampsAliasCert(t *testing.T) {
if err := domains.Reconcile(ctx, s, reg, c); err != nil {
t.Fatal(err)
}
+ if err := domains.StampHealth(ctx, reg, c, nil); err != nil { // as domainsLoop does after the pass
+ t.Fatal(err)
+ }
if got := aliasByName(t, reg, "kass-alias.training"); got.CertExpiresAt != notAfter.Unix() {
t.Errorf("kass-alias.training cert_expires_at = %d, want %d", got.CertExpiresAt, notAfter.Unix())
}
@@ -1183,3 +1195,100 @@ func TestReconcileDoesNotDropRoutesWhenTheProductionFlagIsUnreadable(t *testing.
t.Errorf("no refusal logged; log was:\n%s", logged.String())
}
}
+
+// #303: the health stamps are a per-box pass with no lease involved —
+// callable on their own, stamping routes AND aliases from this box's view,
+// and a no-op with nothing configured.
+func TestStampHealthStandsAlone(t *testing.T) {
+ reg := regWith(t, registry.Route{Host: "app.acct.oncarlos.test", Kind: "service", Addr: "10.0.0.1:1", Status: "ready"})
+ if err := reg.UpsertAlias(registry.RouteAlias{Alias: "www.customer.test", Host: "app.acct.oncarlos.test", Account: "acct", App: "app"}); err != nil {
+ t.Fatal(err)
+ }
+ cfg := domains.Config{
+ AppsDomain: "oncarlos.test",
+ EdgeIPs: []string{"203.0.113.7"},
+ Resolve: func(host string) ([]string, error) {
+ if host == "www.customer.test" {
+ return []string{"203.0.113.7"}, nil
+ }
+ return []string{"198.51.100.1"}, nil
+ },
+ }
+ if err := domains.StampHealth(context.Background(), reg, cfg, nil); err != nil {
+ t.Fatal(err)
+ }
+ if got := mustGet(t, reg, "app.acct.oncarlos.test"); got.DNSStatus != "elsewhere" {
+ t.Errorf("route dns_status = %q; want elsewhere", got.DNSStatus)
+ }
+ aliases, err := reg.ListAliases()
+ if err != nil || len(aliases) != 1 || aliases[0].DNSStatus != "pointing" {
+ t.Errorf("alias dns_status: %+v, %v; want pointing", aliases, err)
+ }
+ // Nothing configured: nothing stamped, no error.
+ reg2 := regWith(t, registry.Route{Host: "x.test", Kind: "service", Addr: "10.0.0.1:1", Status: "ready"})
+ if err := domains.StampHealth(context.Background(), reg2, domains.Config{AppsDomain: "oncarlos.test"}, nil); err != nil {
+ t.Fatal(err)
+ }
+ if got := mustGet(t, reg2, "x.test"); got.DNSStatus != "" || got.DNSCheckedAt != 0 {
+ t.Errorf("unconfigured pass stamped %+v", got)
+ }
+}
+
+// A resolver failure never overwrites a verdict: the last known verdict
+// and its stamp time stay, so a bad DNS tick cannot unpublish a working
+// custom domain (#287's gate) or launder staleness into "never checked".
+func TestStampHealthHoldsLastVerdictOnResolverFailure(t *testing.T) {
+ reg := regWith(t, registry.Route{Host: "app.acct.oncarlos.test", Kind: "service", Addr: "10.0.0.1:1", Status: "ready"})
+ if err := reg.UpsertAlias(registry.RouteAlias{Alias: "www.customer.test", Host: "app.acct.oncarlos.test", Account: "acct", App: "app"}); err != nil {
+ t.Fatal(err)
+ }
+ good := domains.Config{AppsDomain: "oncarlos.test", EdgeIPs: []string{"203.0.113.7"},
+ Resolve: func(string) ([]string, error) { return []string{"203.0.113.7"}, nil },
+ Now: func() time.Time { return time.Unix(1000, 0) }}
+ if err := domains.StampHealth(context.Background(), reg, good, nil); err != nil {
+ t.Fatal(err)
+ }
+ broken := good
+ broken.Resolve = func(string) ([]string, error) { return nil, errors.New("SERVFAIL") }
+ broken.Now = func() time.Time { return time.Unix(2000, 0) }
+ if err := domains.StampHealth(context.Background(), reg, broken, nil); err != nil {
+ t.Fatal(err)
+ }
+ if got := mustGet(t, reg, "app.acct.oncarlos.test"); got.DNSStatus != "pointing" || got.DNSCheckedAt != 1000 {
+ t.Errorf("route after resolver failure = %q @%d; want pointing @1000", got.DNSStatus, got.DNSCheckedAt)
+ }
+ aliases, _ := reg.ListAliases()
+ if len(aliases) != 1 || aliases[0].DNSStatus != "pointing" || aliases[0].DNSCheckedAt != 1000 {
+ t.Errorf("alias after resolver failure = %+v; want pointing @1000", aliases)
+ }
+ // A never-checked row still records unknown, so "unknown" stays honest.
+ fresh := regWith(t, registry.Route{Host: "new.acct.oncarlos.test", Kind: "service", Addr: "10.0.0.1:1", Status: "ready"})
+ if err := domains.StampHealth(context.Background(), fresh, broken, nil); err != nil {
+ t.Fatal(err)
+ }
+ if got := mustGet(t, fresh, "new.acct.oncarlos.test"); got.DNSStatus != "" || got.DNSCheckedAt != 2000 {
+ t.Errorf("fresh row = %q @%d; want \"\" @2000", got.DNSStatus, got.DNSCheckedAt)
+ }
+}
+
+// Back-to-back passes are idempotent: a second StampHealth with the same
+// view changes nothing but dns_checked_at.
+func TestStampHealthIsIdempotent(t *testing.T) {
+ reg := regWith(t, registry.Route{Host: "app.acct.oncarlos.test", Kind: "service", Addr: "10.0.0.1:1", Status: "ready"})
+ c := cfg(t)
+ for i := 0; i < 2; i++ {
+ if err := domains.StampHealth(context.Background(), reg, c, nil); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if got := mustGet(t, reg, "app.acct.oncarlos.test"); got.DNSStatus != "pointing" || got.DNSCheckedAt != c.Now().Unix() {
+ t.Errorf("after two passes: %q @%d", got.DNSStatus, got.DNSCheckedAt)
+ }
+ // And inert with no apps domain, per Config's contract.
+ if err := domains.StampHealth(context.Background(), reg, domains.Config{Resolve: c.Resolve, Now: func() time.Time { return time.Unix(9, 0) }}, nil); err != nil {
+ t.Fatal(err)
+ }
+ if got := mustGet(t, reg, "app.acct.oncarlos.test"); got.DNSCheckedAt == 9 {
+ t.Error("a pass with no AppsDomain wrote a stamp")
+ }
+}
diff --git a/internal/host/agent.go b/internal/host/agent.go
index 4f12d4a93..298afdb68 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -1629,7 +1629,7 @@ func domainsLoop(reg *registry.Registry, acmeCacheFlag string) {
log.Printf("agent: domains loop disabled: %v", err)
return
}
- log.Printf("agent: domains loop armed (5min + jitter), apps domain %s, edge IPs %v — reconciles only while this box holds the housekeeper lease", cfg.AppsDomain, cfg.EdgeIPs)
+ log.Printf("agent: domains loop armed (5min + jitter), apps domain %s, edge IPs %v — health stamps every tick; claims reconcile only while this box holds the housekeeper lease", cfg.AppsDomain, cfg.EdgeIPs)
// One line, and it must name the REAL cause: a typo'd key and an absent
// one are different problems with different fixes, and this is the only
// place an operator learns that purging is not happening.
@@ -1641,22 +1641,37 @@ func domainsLoop(reg *registry.Registry, acmeCacheFlag string) {
duty := domainsDuty(func(ctx context.Context) error {
return domains.Reconcile(ctx, store, reg, cfg)
})
+ fleetBox := shouldSkipPublisherElection(os.Getenv)
delay := time.Minute
for {
time.Sleep(delay)
delay = 5*time.Minute + rand.N(time.Minute)
+ // The deployment-global half, gated; then the per-box half on EVERY
+ // platform box (#303): DNS and TLS health stamp this box's own
+ // registry from this box's own view, no lease involved, ONCE per
+ // tick — after the duties so a host section 3 just minted is
+ // stamped in the same tick it appeared. Never on a fleet box: it
+ // never contends for the lease for the same reason (housekeeperNever),
+ // and its EdgeIPs fall back to its own address, which would judge
+ // every row "elsewhere".
runHousekeeperDuties(housekeeperGate, duty)
+ if !fleetBox {
+ if err := domains.StampHealth(ctx, reg, cfg, nil); err != nil {
+ log.Printf("agent: domains health: %v", err)
+ }
+ }
}
}
// domainsDuty wraps one domains reconcile pass as a housekeeper duty. The
-// pass is deployment-wide state — custom-domain `pointing` stamps, cert
-// expiry stamps, purging trashed apps — with one correct writer, and since
-// multi-region (spec 2026-08-20) every platform box in a deployment carries
-// PLATFORM_APPS_DOMAIN (the per-account wildcards and the steering probe
-// need it), so an ungated loop would be N reconcilers racing over the same
-// records, each judging `pointing` against its OWN edge-IP list. Gating on
-// the lease is what makes the env safe to set fleet-wide. Single-box
+// pass is deployment-wide state — materializing routes from claims, minting
+// platform-domain and canary hosts, purging trashed apps — with one correct
+// writer, and since multi-region (spec 2026-08-20) every platform box in a
+// deployment carries PLATFORM_APPS_DOMAIN (the per-account wildcards and the
+// steering probe need it), so an ungated loop would be N reconcilers racing
+// over the same records. Gating on the lease is what makes the env safe to
+// set fleet-wide. (The DNS/TLS health stamps are NOT in this pass: they are
+// per-box, domains.StampHealth, run by domainsLoop on every box — #303.) Single-box
// deployments are unchanged: the one box always holds the lease. A box that
// never contends (a fleet box) never runs the pass — which is right, the
// pass is a platform-box job.