amadan

carlos / platform Public

edge, host: a sidecar row is refused on kind, at both ends of the fleet chain

Paul Campbell pushed by paul@keymail.dev 04f3fd9f3fc9bdb22abcbabf4c7269f8d412fb13 parent 6c5efcb
4 files changed, +149 −5
  • internal/edge/edge.go +17 −0
  • internal/edge/edge_test.go +65 −0
  • internal/host/fleetbind.go +27 −5
  • internal/host/fleetbind_test.go +40 −0
diff --git a/internal/edge/edge.go b/internal/edge/edge.go
index 151be4f35..c3826cb2e 100644
--- a/internal/edge/edge.go
+++ b/internal/edge/edge.go
@@ -311,6 +311,23 @@ func (e *Edge) Reload() error {
// retires activator children for hosts absent from `managed`.
probeSpecs := make(map[string]probeSpec)
for _, t := range routes {
+ // A sidecar is a member that serves nothing: it binds no socket, and
+ // nothing — inside the box or outside it — may ever dial it. Refused
+ // here on KIND, as the first act of the loop and ahead of every other
+ // check, rather than left to fall through the dispatch switch for want
+ // of a Socket/Addr/Tunnel.
+ //
+ // That omission is not a guarantee. The fleet arm below binds on
+ // placement alone (`isFleetPlaced && e.tunnels != nil`, no kind, no
+ // backend column), and a provisioned Eleven sidecar IS fleet-placed
+ // (region "fleet/<name>") — so the row this feature creates reaches a
+ // working internet-facing proxy by shape. This refusal is the
+ // structural guarantee that survives a future arm, column or
+ // placement finding the row: no map entry, so no proxy; and no map
+ // entry means Allowed says no, so no ACME issuance either.
+ if t.Kind == "sidecar" {
+ continue
+ }
// Only serve ready routes: a 'creating'/'failed' entry has no working
// backend, so proxying it would just stall → 502 — AND, since Allowed
// is the autocert HostPolicy, serving it would burn an ACME issuance
diff --git a/internal/edge/edge_test.go b/internal/edge/edge_test.go
index 157fc7d71..503ed79b9 100644
--- a/internal/edge/edge_test.go
+++ b/internal/edge/edge_test.go
@@ -2175,3 +2175,68 @@ func TestServeHTTPMetersRequestsAndBytesExcludingStatus(t *testing.T) {
t.Fatalf("status route must not be metered, got %+v", rows["status.example.com"])
}
}
+
+// TestSidecarRouteIsNeverServable is the structural guarantee behind the
+// sidecar member kind: a sidecar is a process that serves nothing and that
+// nothing may dial, and the edge must refuse it on KIND rather than on the
+// accident of it carrying no backend columns.
+//
+// The row below is adversarial on purpose. It carries a Socket AND an Addr
+// AND a Tunnel AND a fleet placement (Region + FleetBox), on an edge with a
+// tunnel hub armed and the fleet registered — every column and every
+// precondition that any arm of Reload's dispatch switch looks at, all at
+// once. Eleven's sidecars are fleet-placed, so the fleet arm (which checks no
+// kind at all) is reached by such a row on shape alone. If this test ever
+// fails, a sidecar is reachable from the internet.
+func TestSidecarRouteIsNeverServable(t *testing.T) {
+ const host = "bot.eleven.bdm.sidecar.internal"
+
+ reg := openReg(t)
+ if err := reg.UpsertFleet(registry.Fleet{Name: "eleven", Account: "amadan", Bucket: "eleven-data"}); err != nil {
+ t.Fatalf("UpsertFleet: %v", err)
+ }
+ if err := reg.AddFleetBox(registry.FleetBox{Fleet: "eleven", Label: "box1", TokenHash: sha256Hex("tok"), Status: "active"}); err != nil {
+ t.Fatalf("AddFleetBox: %v", err)
+ }
+ if err := reg.Upsert(registry.Route{
+ Host: host,
+ Kind: "sidecar",
+ Socket: filepath.Join(t.TempDir(), "whatever.sock"),
+ Addr: "127.0.0.1:9",
+ Tunnel: "eleven/box1",
+ Region: "fleet/eleven",
+ FleetBox: "box1",
+ Status: "ready",
+ App: "eleven",
+ Account: "bdm",
+ Platform: true,
+ }); err != nil {
+ t.Fatalf("upsert sidecar route: %v", err)
+ }
+
+ e, err := New(reg, nil)
+ if err != nil {
+ t.Fatalf("New: %v", err)
+ }
+ // Arm the hub, so the Tunnel and fleet arms of the dispatch switch are
+ // live rather than skipped by their e.tunnels == nil guard.
+ e.EnableTunnels(NewTunnelHub("tunnel.example.com", nil))
+ if err := e.Reload(); err != nil {
+ t.Fatalf("Reload: %v", err)
+ }
+
+ if _, ok := e.current()[host]; ok {
+ t.Fatalf("a sidecar row entered the routing map — it is reachable from the internet")
+ }
+ // Allowed is the autocert HostPolicy: a sidecar host must never cost an
+ // ACME issuance either.
+ if e.Allowed(host) {
+ t.Fatalf("Allowed(%s) = true — a certificate must never be requested for a sidecar host", host)
+ }
+
+ srv := httptest.NewServer(e)
+ defer srv.Close()
+ if code, _ := getHost(t, srv.URL, host); code != http.StatusNotFound {
+ t.Fatalf("GET %s = %d, want 404 — a sidecar must be unreachable", host, code)
+ }
+}
diff --git a/internal/host/fleetbind.go b/internal/host/fleetbind.go
index adbed21e0..e75fdb298 100644
--- a/internal/host/fleetbind.go
+++ b/internal/host/fleetbind.go
@@ -289,8 +289,8 @@ func fleetBindingHook(store release.Store, fleet, box string) func(host string)
// - an agent restart discards activator.Service's in-memory entries, but
// an already-running exec child (or a unit-backed tenant) keeps serving
// — its host would otherwise have no fresh binding until its NEXT idle
-// wake, which may be a long time coming, or never for an always-awake
-// sidecar host;
+// wake, which may be a long time coming, or never for a host that is
+// always awake;
// - a route that isn't activator-managed at all (Hibernate unset, or a
// unit-backed Backing) never goes through activator.wake in the first
// place, so it would never get a binding published any other way.
@@ -307,10 +307,32 @@ func publishResidentFleetBindings(store release.Store, fleet, box string, reg *r
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
now := time.Now().UnixMilli()
- for _, rt := range routes {
- b := FleetBinding{Host: rt.Host, Fleet: fleet, Box: box, BoundAt: now}
+ for _, host := range residentBindingHosts(routes) {
+ b := FleetBinding{Host: host, Fleet: fleet, Box: box, BoundAt: now}
if err := WriteFleetBinding(ctx, store, b, false); err != nil {
- log.Printf("agent: fleet binding for %s: %v", rt.Host, err)
+ log.Printf("agent: fleet binding for %s: %v", host, err)
+ }
+ }
+}
+
+// residentBindingHosts is the set of hosts this box advertises as served —
+// the pure host-selection half of publishResidentFleetBindings, split out so
+// the refusal below is directly testable.
+//
+// Sidecar routes (Kind "sidecar") are excluded. Nothing routes to a sidecar,
+// so a binding claiming a box serves it is a lie the platform side then acts
+// on: ReconcileFleetBindings stamps SetFleetBox from it, and a fleet-placed
+// route is exactly what the edge's fleet dispatch arm turns into a live
+// internet-facing proxy. Filtered at the publisher rather than at the
+// reconciler because this is the only place that decides what a box CLAIMS —
+// the claim is what must never exist.
+func residentBindingHosts(routes []registry.Route) []string {
+ var out []string
+ for _, rt := range routes {
+ if rt.Host == "" || rt.Kind == "sidecar" {
+ continue
}
+ out = append(out, rt.Host)
}
+ return out
}
diff --git a/internal/host/fleetbind_test.go b/internal/host/fleetbind_test.go
index ea8c24e57..7f24c1031 100644
--- a/internal/host/fleetbind_test.go
+++ b/internal/host/fleetbind_test.go
@@ -591,3 +591,43 @@ func TestFleetBindingHookTakesOverFromAnotherBox(t *testing.T) {
t.Fatalf("Box = %q, want new-box (OnWake's hook must take over)", got.Box)
}
}
+
+// TestSidecarBindingIsNeverPublished: a fleet binding says "this box serves
+// this host". Nothing routes to a sidecar, so publishing one would be a lie
+// the platform-side reconciler then acts on — ReconcileFleetBindings stamps
+// SetFleetBox from it, and a fleet-placed route is exactly what the edge's
+// fleet dispatch arm binds. The filter therefore lives in the publisher's own
+// host selection, not in the reconciler that reads it back.
+func TestSidecarBindingIsNeverPublished(t *testing.T) {
+ got := residentBindingHosts([]registry.Route{
+ {Host: "app.example.com", Kind: "instance", Socket: "/run/carlos/a.sock"},
+ {Host: "bot.eleven.bdm.sidecar.internal", Kind: "sidecar", Region: "fleet/eleven"},
+ })
+ if len(got) != 1 || got[0] != "app.example.com" {
+ t.Fatalf("published %v — a sidecar must never be advertised as served by a box", got)
+ }
+}
+
+// The end-to-end shape of the same refusal: the startup pass over a registry
+// holding both kinds must write a binding object for the instance and none
+// at all for the sidecar.
+func TestPublishResidentFleetBindingsSkipsSidecars(t *testing.T) {
+ ctx := context.Background()
+ s := release.NewDirStore(t.TempDir())
+ reg := testRegistry(t)
+ if err := reg.Upsert(registry.Route{Host: "app.example.com", Kind: "instance"}); err != nil {
+ t.Fatalf("upsert instance: %v", err)
+ }
+ if err := reg.Upsert(registry.Route{Host: "bot.eleven.bdm.sidecar.internal", Kind: "sidecar", Region: "fleet/eleven"}); err != nil {
+ t.Fatalf("upsert sidecar: %v", err)
+ }
+
+ publishResidentFleetBindings(s, "eleven", "box1", reg)
+
+ if _, _, err := s.Get(ctx, fleetBindingKey("eleven", "app.example.com")); err != nil {
+ t.Fatalf("instance binding missing: %v", err)
+ }
+ if _, _, err := s.Get(ctx, fleetBindingKey("eleven", "bot.eleven.bdm.sidecar.internal")); err == nil {
+ t.Fatalf("a sidecar binding was published — a box must never advertise a sidecar as served")
+ }
+}