amadan
domains: name the refusal that made a cross-box flip a silent no-op
ReconcileAliases resolves a prefix's target through byHost — this box's routes. Backhaul fires only on a HOST miss, never for a path target, so a target homed on another box derives no route_paths row on any box. `ti.to` on one box and `go.tito.io` on another is the likely shape for two apps as different as Tito classic and Tito Go, and it was also the ONLY refusal in the path block that logged nothing: publish, reconcile, and route nothing, forever, with no signal. The feature has no production consumer for months, which is exactly how long that silence would last. We are not making cross-box path targets work — that needs a backhaul equivalent for path targets and is a design change. This fixes the silence: - One aggregated line per host per pass, naming the count and the first few unresolved targets. Aggregated because this runs every ~2s and a 10k-entry map must not write 10k lines every two seconds. - The comment names three distinct causes with three different remedies: the target does not exist, the target is homed elsewhere, the target is local but not ready. Conflating the first two sends an operator hunting for a deleted route. - Targets are gated on Status == "ready", which spec §4 already required and the code did not do. The edge filtered unready routes out of its map anyway, so traffic fell through safely — but `carlos routes` would print the path row as if it were serving, and that is the one surface built for diagnosing a flip. Tenancy is still checked first, so a cross-tenant target is named as cross-tenant rather than swept into the aggregate. TestReconcileCapsPathRows now asserts WHICH prefixes survive the cap, not just how many: the sort is what makes every box derive the identical subset, and deleting it left the old assertion green.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2 files changed,
+154
−9
internal/domains/aliases.go+59 −8internal/domains/aliases_paths_test.go+95 −1
diff --git a/internal/domains/aliases.go b/internal/domains/aliases.go| index 6a6776d6b..e4d315418 100644 |
| --- a/internal/domains/aliases.go |
| +++ b/internal/domains/aliases.go |
| @@ -13,7 +13,9 @@ package domains |
| import ( |
| "context" |
| "log" |
| + "slices" |
| "sort" |
| + "strings" |
| "time" |
| "github.com/carlosframework/platform/internal/registry" |
| @@ -123,6 +125,14 @@ func ReconcileAliases(ctx context.Context, s release.Store, reg *registry.Regist |
| // for the same reason: the prefix map belongs to that route, so a box |
| // that does not serve the route derives nothing. |
| // |
| + // The TARGET is resolved through the same local byHost, which makes |
| + // co-location a hard constraint of the feature: a prefix's target |
| + // route must be homed on the same box as the hostname's own route. |
| + // There is no backhaul equivalent for a path target — backhaul fires |
| + // on a host miss, and the host resolves fine here — so a target on |
| + // another box derives no row anywhere at all. See the unresolved log |
| + // below, and design spec §6. |
| + // |
| // Ordered BEFORE the wildcard section on purpose: Paths is a third |
| // claim capability independent of Wildcard (claims.go's comment on |
| // the field), and the wildcard section below `continue`s for every |
| @@ -150,6 +160,14 @@ func ReconcileAliases(ctx context.Context, s release.Store, reg *registry.Regist |
| c.Host, len(prefixes), MaxPathsPerClaim, len(prefixes)-MaxPathsPerClaim) |
| prefixes = prefixes[:MaxPathsPerClaim] |
| } |
| + // Unresolved targets are counted and named ONCE per host per |
| + // pass, not logged per prefix: this pass runs every ~2s on |
| + // every box, and a 10k-entry map whose targets are all |
| + // elsewhere would otherwise write 10k lines every two |
| + // seconds. The count is the operator's signal; the first few |
| + // names are enough to identify which map is wrong. |
| + unresolved := 0 |
| + var unresolvedEg []string |
| for _, prefix := range prefixes { |
| target := c.Paths[prefix] |
| if !validPathPrefix(prefix) { |
| @@ -157,27 +175,60 @@ func ReconcileAliases(ctx context.Context, s release.Store, reg *registry.Regist |
| continue |
| } |
| tgtRt, ok := byHost[target] |
| - if !ok { |
| - // Target is not a route on this box. No row, so the |
| - // prefix falls through to the host's own route — the |
| - // right failure direction for a migration: a missing |
| - // destination serves the ORIGIN, never a 404. |
| - continue |
| - } |
| // The authorization rule this feature adds: a prefix must |
| // never point another tenant's traffic at your backend. |
| // Account only — an account routing between its own apps |
| // (classic and Go ARE two apps) is the whole use case. |
| - if release.RouteAccount(tgtRt) != c.Account { |
| + // Checked before readiness so a cross-tenant target is |
| + // named as cross-tenant rather than swept into the |
| + // unresolved count below. |
| + if ok && release.RouteAccount(tgtRt) != c.Account { |
| log.Printf("domains: path %s/%s refused: target %s belongs to %s, claim says %s", |
| c.Host, prefix, target, release.RouteAccount(tgtRt), c.Account) |
| continue |
| } |
| + if !ok || tgtRt.Status != "ready" { |
| + // The target is not a local READY route, which has |
| + // three distinct causes with three different |
| + // remedies, and the log below cannot tell them apart |
| + // from here: |
| + // |
| + // 1. The target route does not exist anywhere — a |
| + // typo in the published map, or a route deleted |
| + // out from under it. Remedy: fix the map, or |
| + // re-create the route. |
| + // 2. The target exists but is HOMED ON ANOTHER BOX. |
| + // byHost holds this box's routes only, and there |
| + // is no backhaul equivalent for a path target |
| + // (backhaul fires on a HOST miss), so nothing |
| + // derives the row anywhere. Remedy: co-locate — |
| + // a prefix's target must be homed on the same box |
| + // as the hostname's own route (design spec §6). |
| + // 3. The target is local but not ready yet — a |
| + // route mid-provision. Self-resolving; the row |
| + // appears on the pass after it goes ready. |
| + // |
| + // No row either way, so the prefix falls through to |
| + // the host's own route — the right failure direction |
| + // for a migration: a missing destination serves the |
| + // ORIGIN, never a 404. Silent traffic is the danger |
| + // here, not broken traffic, which is why this is |
| + // logged at all. |
| + unresolved++ |
| + if len(unresolvedEg) < 3 && !slices.Contains(unresolvedEg, target) { |
| + unresolvedEg = append(unresolvedEg, target) |
| + } |
| + continue |
| + } |
| desiredPaths[c.Host+"\x00"+prefix] = registry.RoutePath{ |
| Host: c.Host, Prefix: prefix, Target: target, |
| Account: c.Account, App: c.App, |
| } |
| } |
| + if unresolved > 0 { |
| + log.Printf("domains: paths on %s: %d prefix(es) name no local ready route (e.g. %s) — the target is missing, or it is homed on another box; a path target must be homed on the SAME box as %s. Those prefixes serve %s's own route.", |
| + c.Host, unresolved, strings.Join(unresolvedEg, ", "), c.Host, c.Host) |
| + } |
| } |
| } |
diff --git a/internal/domains/aliases_paths_test.go b/internal/domains/aliases_paths_test.go| index 802cee431..db02d65cc 100644 |
| --- a/internal/domains/aliases_paths_test.go |
| +++ b/internal/domains/aliases_paths_test.go |
| @@ -1,8 +1,12 @@ |
| package domains |
| import ( |
| + "bytes" |
| "context" |
| + "log" |
| + "sort" |
| "strconv" |
| + "strings" |
| "testing" |
| "github.com/carlosframework/platform/internal/registry" |
| @@ -169,7 +173,97 @@ func TestReconcileCapsPathRows(t *testing.T) { |
| if err := ReconcileAliases(ctx, s, reg, aliasNow(), nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| - if got := pathsByPrefix(t, reg, "ti.to"); len(got) != MaxPathsPerClaim { |
| + got := pathsByPrefix(t, reg, "ti.to") |
| + if len(got) != MaxPathsPerClaim { |
| t.Fatalf("derived %d rows, want the cap of %d", len(got), MaxPathsPerClaim) |
| } |
| + // WHICH prefixes survive, not just how many. The count alone passes with |
| + // the sort deleted, and the sort is the safety-critical half: every box |
| + // folds the same claim independently, so a truncation that varied with |
| + // Go's map iteration order would let the same URL resolve to different |
| + // backends depending on which edge answered. |
| + want := make([]string, 0, len(big)) |
| + for p := range big { |
| + want = append(want, p) |
| + } |
| + sort.Strings(want) |
| + want = want[:MaxPathsPerClaim] |
| + for _, p := range want { |
| + if _, ok := got[p]; !ok { |
| + t.Fatalf("prefix %q was dropped; the surviving set must be the first %d in SORTED order, identically on every box", p, MaxPathsPerClaim) |
| + } |
| + } |
| +} |
| + |
| +// TestReconcileSkipsUnreadyTarget: a target route that exists on this box but |
| +// is not ready yet contributes no row (design spec §4, "a target that is not |
| +// a local READY route contributes no row"). The edge filters unready routes |
| +// out of its map anyway, so traffic already fell through safely — what this |
| +// gate buys is that `carlos routes` and the edge agree about which prefixes |
| +// are actually serving, and that surface is the one built for diagnosing a |
| +// flip. |
| +func TestReconcileSkipsUnreadyTarget(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + reg := aliasRegWith(t) |
| + addRoute(t, reg, "ti.to", "tito", "classic") |
| + if err := reg.Upsert(registry.Route{ |
| + Host: "go.tito.io", App: "classic", Account: "tito", Kind: "instance", Status: "provisioning", |
| + }); err != nil { |
| + t.Fatal(err) |
| + } |
| + putClaim(t, s, Claim{ |
| + Host: "ti.to", Account: "tito", App: "classic", Route: "ti.to", |
| + CreatedAt: "2026-09-03T00:00:00Z", |
| + Paths: map[string]string{"paulca": "go.tito.io"}, |
| + }) |
| + |
| + if err := ReconcileAliases(ctx, s, reg, aliasNow(), nil, nil); err != nil { |
| + t.Fatal(err) |
| + } |
| + if got := pathsByPrefix(t, reg, "ti.to"); len(got) != 0 { |
| + t.Fatalf("route_paths = %v, want no row for a target that is not ready", got) |
| + } |
| +} |
| + |
| +// TestReconcileLogsUnresolvedTargetsOncePerHost is the critical-silence fix. |
| +// A target homed on ANOTHER BOX is the likeliest real-world refusal in this |
| +// feature — `ti.to` and `go.tito.io` are two very different apps — and it |
| +// used to be the ONLY refusal in the path block that said nothing at all, so |
| +// a migration could be published, accepted, and never route, forever, with |
| +// no signal. It must log; and because this pass runs every ~2s against a map |
| +// that may hold 10k entries, it must log ONE line per host per pass. |
| +func TestReconcileLogsUnresolvedTargetsOncePerHost(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + reg := aliasRegWith(t) |
| + addRoute(t, reg, "ti.to", "tito", "classic") |
| + putClaim(t, s, Claim{ |
| + Host: "ti.to", Account: "tito", App: "classic", Route: "ti.to", |
| + CreatedAt: "2026-09-03T00:00:00Z", |
| + Paths: map[string]string{ |
| + "paulca": "elsewhere.example", |
| + "bigcorp": "elsewhere.example", |
| + "acme": "also-elsewhere.example", |
| + }, |
| + }) |
| + |
| + var buf bytes.Buffer |
| + old := log.Writer() |
| + log.SetOutput(&buf) |
| + defer log.SetOutput(old) |
| + if err := ReconcileAliases(ctx, s, reg, aliasNow(), nil, nil); err != nil { |
| + t.Fatal(err) |
| + } |
| + log.SetOutput(old) |
| + |
| + out := buf.String() |
| + if n := strings.Count(out, "name no local ready route"); n != 1 { |
| + t.Fatalf("logged the unresolved-target line %d times, want exactly one per host per pass:\n%s", n, out) |
| + } |
| + for _, want := range []string{"ti.to", "3 prefix(es)", "elsewhere.example", "homed on another box"} { |
| + if !strings.Contains(out, want) { |
| + t.Fatalf("unresolved-target log is missing %q:\n%s", want, out) |
| + } |
| + } |
| } |