amadan

carlos / platform Public

registry: fleet placement and box restamp move the fingerprint

Found by the two-box e2e (next commit): Route.FleetBox is written by an
UNATTENDED failover — ReconcileFleetBindings stamps the surviving box —
but neither fleet_box nor region appears in Fingerprint's routes
aggregate, so the write left the fingerprint byte-identical, the edge's
2s Watch poll never fired, and its fleet proxy kept the DEAD box's label
captured. Rule 1 then dispatched straight back to that box the moment it
re-dialed, serving from the one box that does NOT hold the lease.

Adding LENGTH(fleet_box) would not have fixed it: box labels are
overwhelmingly the same length as each other (box-a/box-b, pi-1/pi-2),
which is exactly the same-length blind spot the token_hash rev column
exists for. So the fingerprint gains a fourth component carrying the
VALUES — rowid=region>fleet_box per row — restricted to fleet-relevant
rows, which keeps it empty on every non-fleet deployment today and
bounded by the fleet's own route count on the rest. Every transition is
visible: entering placement, leaving it, and a restamp between two
same-length labels.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Paul Campbell dce3fe949a683cefa5601ccdd54e011039e109fd parent 3e11df0
2 files changed, +125 −0
  • internal/registry/fleets_test.go +95 −0
  • internal/registry/registry.go +30 −0
diff --git a/internal/registry/fleets_test.go b/internal/registry/fleets_test.go
index 486b972df..6c3012419 100644
--- a/internal/registry/fleets_test.go
+++ b/internal/registry/fleets_test.go
@@ -382,6 +382,101 @@ func TestFingerprintChangesOnTokenRotation(t *testing.T) {
}
}
+// TestFingerprintChangesOnFleetBoxRestamp is the Task 10 regression pin —
+// the defect the two-box e2e (internal/edge/fleet_e2e_test.go) surfaced. A
+// failover restamps routes.fleet_box (ReconcileFleetBindings -> SetFleetBox)
+// between two labels of the SAME length, which is the common case for fleet
+// box labels ("box-a"/"box-b", "pi-1"/"pi-2"). Before the fleet-placement
+// projection was added to Fingerprint, that write changed nothing the
+// aggregate looked at — fleet_box is not in the routes weighted sum at all —
+// so the edge never reloaded, kept the dead box's label captured in its
+// fleet proxy, and dispatched straight back to it the moment it re-dialed.
+// Unlike an operator's route edit, a restamp is unattended: there is nobody
+// there to SIGHUP.
+func TestFingerprintChangesOnFleetBoxRestamp(t *testing.T) {
+ reg := openTestRegistry(t)
+ if err := reg.Upsert(Route{Host: "app.example", Kind: "instance", Region: PlacementTarget("pi"), FleetBox: "box-a"}); err != nil {
+ t.Fatal(err)
+ }
+ before, err := reg.Fingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := reg.SetFleetBox("app.example", "box-b"); err != nil {
+ t.Fatal(err)
+ }
+ after, err := reg.Fingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if before == after {
+ t.Fatalf("fingerprint unchanged across a same-length fleet_box restamp: %q — the edge would keep routing to the box that died", after)
+ }
+}
+
+// TestFingerprintChangesOnPlacement: the same projection covers the OTHER
+// column fleet dispatch reads — Region. Placing an existing route on a fleet
+// (`carlos apps -place`, a different PROCESS from the edge, so the
+// fingerprint poll is the only path in) must be visible too.
+func TestFingerprintChangesOnPlacement(t *testing.T) {
+ reg := openTestRegistry(t)
+ if err := reg.Upsert(Route{Host: "app.example", Kind: "instance"}); err != nil {
+ t.Fatal(err)
+ }
+ before, err := reg.Fingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := reg.SetPlacement("app.example", PlacementTarget("pi")); err != nil {
+ t.Fatal(err)
+ }
+ mid, err := reg.Fingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if before == mid {
+ t.Fatalf("fingerprint unchanged when a route was placed on a fleet: %q", mid)
+ }
+ // ...and clearing it again is equally visible (the row drops out of the
+ // projection rather than lingering as a stale entry).
+ if err := reg.SetPlacement("app.example", ""); err != nil {
+ t.Fatal(err)
+ }
+ after, err := reg.Fingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if after == mid {
+ t.Fatalf("fingerprint unchanged when a route left fleet placement: %q", after)
+ }
+}
+
+// A registry with no fleet-placed routes at all — every deployment in the
+// fleet today — must produce a STABLE fingerprint across repeated polls: the
+// projection is a GROUP_CONCAT, and one that flapped (or that grew with
+// unrelated routes) would make the edge reload every 2s forever.
+func TestFingerprintStableWithoutFleetRoutes(t *testing.T) {
+ reg := openTestRegistry(t)
+ for _, h := range []string{"one.example", "two.example", "three.example"} {
+ if err := reg.Upsert(Route{Host: h, Kind: "service", Addr: "127.0.0.1:1", Region: "eu"}); err != nil {
+ t.Fatal(err)
+ }
+ }
+ first, err := reg.Fingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ for i := 0; i < 5; i++ {
+ again, err := reg.Fingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if again != first {
+ t.Fatalf("fingerprint flapped with no writes at all: %q then %q", first, again)
+ }
+ }
+}
+
// TestFleetPlacement pins the "fleet/<name>" grammar Route.Region carries
// when a route is bound to a customer fleet: the prefix alone is not enough,
// the name half must also pass ValidTunnelName, and a bare region code (a
diff --git a/internal/registry/registry.go b/internal/registry/registry.go
index c7095cc10..1c136de22 100644
--- a/internal/registry/registry.go
+++ b/internal/registry/registry.go
@@ -1087,6 +1087,30 @@ WHERE kind = 'instance' AND next_due > 0 AND next_due <= ? ORDER BY host`, now)
// rev is what makes a fixed-length-value rewrite visible, the same job
// COUNT/MAX(rowid) do for inserts/deletes and LENGTH does for
// variable-length rewrites.
+//
+// The FOURTH component is the fleet-placement projection, and it is a
+// verbatim value list rather than a weighted length sum for a reason the
+// two-box e2e (internal/edge/fleet_e2e_test.go) found the hard way: the
+// routes aggregate above covers neither `region` nor `fleet_box`, so a
+// failover restamp (ReconcileFleetBindings -> SetFleetBox, box-a -> box-b)
+// left the fingerprint byte-identical and the edge kept dispatching to the
+// box that had died — rule 1 pointing at a corpse the moment it re-dialed,
+// with no operator around to SIGHUP, because a restamp is UNATTENDED where
+// an operator's `carlos route` edit is not. Adding LENGTH(fleet_box) would
+// not have fixed it either: fleet box labels are overwhelmingly the same
+// length as each other ("pi-1"/"pi-2", "box-a"/"box-b"), which is exactly
+// the blind spot the paragraph above documents. So this component carries
+// the VALUES — rowid=region>fleet_box per row — and is restricted to
+// fleet-relevant rows (a fleet placement target, or a stamped box), which
+// keeps it empty on every non-fleet deployment and bounded by the fleet's
+// own route count on the rest. Every transition is visible: a route
+// entering fleet placement appears, one leaving drops out, a restamp
+// changes its entry.
+//
+// The ORDER BY rowid in the subquery is what makes GROUP_CONCAT's result
+// deterministic across polls; a hypothetical reordering would only cost a
+// redundant reload (idempotent), never a missed change — the failure mode
+// worth engineering against is the false NEGATIVE.
func (r *Registry) Fingerprint() (string, error) {
db, err := sql.Open("sqlite", r.path)
if err != nil {
@@ -1113,6 +1137,12 @@ SELECT
COALESCE(SUM(rowid*31 + LENGTH(fleet)*37 + LENGTH(label)*41
+ LENGTH(token_hash)*43 + LENGTH(status)*47 + rev*59),0)
FROM fleet_boxes)
+ || ':' ||
+ (SELECT COALESCE(GROUP_CONCAT(v, ','), '')
+ FROM (SELECT rowid || '=' || region || '>' || fleet_box AS v
+ FROM routes
+ WHERE fleet_box <> '' OR region LIKE 'fleet/%'
+ ORDER BY rowid))
`).Scan(&fp)
return fp, err
}