amadan

carlos / platform Public

Merge branch 'tenant-iso-routes' into main

Routes: only operators add instance upstreams; the console trusts loopback X-Forwarded-For only with the edge's host-bound proof (#214 items 4-5, #50)
claude 1c43c2ce82f621c6ac84a623db54f0ef63075603 parent 9bed97bparent e87f9ae
24 files changed, +951 −58
  • docs/site/cli.md +6 −0
  • instances_e2e_test.go +5 −0
  • internal/console/apiinstancemove_test.go +1 −0
  • internal/console/apiinstances.go +52 −0
  • internal/console/apiinstances_test.go +148 −3
  • internal/console/apiinstancesenvironment_test.go +1 −0
  • internal/console/appsettings_gate_test.go +2 −0
  • internal/console/authcli.go +17 −5
  • internal/console/authcli_test.go +9 −1
  • internal/console/controlplane_test.go +7 −3
  • internal/console/edgeproof.go +172 −0
  • internal/console/edgeproof_test.go +122 −0
  • internal/console/placementdoors_test.go +1 −0
  • internal/console/regionfill_test.go +3 −1
  • internal/console/server.go +3 −0
  • internal/console/store.go +7 −0
  • internal/edge/edge.go +11 −0
  • internal/edge/forwardproof.go +52 −0
  • internal/edge/forwardproof_test.go +150 −0
  • internal/edge/homemap.go +3 −0
  • internal/edge/homemap_test.go +3 −0
  • internal/edge/mesh.go +11 −41
  • internal/edgeproof/edgeproof.go +160 −0
  • internal/provision/provision.go +5 −4

CI cancelled — superseded by a newer push — run details

diff --git a/docs/site/cli.md b/docs/site/cli.md
index a75e46179..1ea8c232e 100644
--- a/docs/site/cli.md
+++ b/docs/site/cli.md
@@ -598,6 +598,12 @@ but not at another app's platform apex. That is refused while the other app
is live and while it is in the Trash, since restoring it would bring the
clash back.
+`--addr` on `create`, and `set-upstreams`, point an instance at TCP backends
+of its own. Only a platform operator can set them, because the edge reaches
+those addresses from inside the platform. If an instance already has
+backends, you can still keep them, remove one, or change its other
+settings; you cannot add a new one.
+
```sh
carlos instances enable --app hello
carlos instances list --app hello
diff --git a/instances_e2e_test.go b/instances_e2e_test.go
index 59fef05b6..01ba884c7 100644
--- a/instances_e2e_test.go
+++ b/instances_e2e_test.go
@@ -50,6 +50,11 @@ func TestE2EInstancesThroughConsole(t *testing.T) {
t.Fatal(err)
}
t.Cleanup(func() { cst.Close() })
+ // The -addr / set-upstreams leg below states upstreams, which only a
+ // platform operator may do.
+ if err := cst.SetOperator("paul@keymail.dev", true); err != nil {
+ t.Fatal(err)
+ }
var srv *console.Server
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
diff --git a/internal/console/apiinstancemove_test.go b/internal/console/apiinstancemove_test.go
index a92ecb25e..c493e634d 100644
--- a/internal/console/apiinstancemove_test.go
+++ b/internal/console/apiinstancemove_test.go
@@ -140,6 +140,7 @@ func TestAPIInstanceMoveVerbs(t *testing.T) {
// The class rails: only an exec-backed instance route moves.
func TestAPIInstanceMoveClassRails(t *testing.T) {
fx, token := instanceFixture(t, "code-inst-move-rails")
+ letStateUpstreams(t, fx, "alice@keymail.dev")
m := titoManifest()
m.Regions = nil
m.Pool = ""
diff --git a/internal/console/apiinstances.go b/internal/console/apiinstances.go
index f118f8b4c..91844c54c 100644
--- a/internal/console/apiinstances.go
+++ b/internal/console/apiinstances.go
@@ -1093,6 +1093,9 @@ func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request)
jsonError(w, http.StatusBadRequest, err.Error())
return
}
+ if s.refuseMemberUpstreams(w, p, rec.Addrs, nil) {
+ return
+ }
// THE STORE IS PROVISIONED BEFORE THE RECORD IS WRITTEN, and a failure
// here means no record at all (issue #412, spec §7). This ordering is
// the load-bearing decision of the feature, so it is argued here rather
@@ -1344,6 +1347,49 @@ func (s *Server) staticHeadRefusal(ctx context.Context, sqid, app, channel, kind
" --kind static --host " + host + " <site-dir>"}
}
+// refuseMemberUpstreams answers whether this caller may point a route at
+// addrs, and writes the refusal when not. Stating an upstream is an
+// operator's act: the edge dials these addresses from inside the platform,
+// so a member's could reach the box's own services (the console on
+// 127.0.0.1:8090) or another tenant's private upstream (discussion 214,
+// item 4). Paul's ruling on discussion 50 (option a): members
+// state none until the "own fleet" allowance exists, public addresses
+// included, and operators keep today's rail.
+//
+// kept lists the addresses the record already carries. Those pass whoever
+// sends them, so a member whose route an operator pointed somewhere can
+// keep it, change its probe, or drop a dead upstream; only adding one is
+// refused.
+//
+// "Operator" is a person holding the operator bit. A service credential
+// never is one, whichever account it belongs to. A failed operator lookup
+// refuses, so a database blip is not the way past it.
+func (s *Server) refuseMemberUpstreams(w http.ResponseWriter, p Principal, addrs, kept []string) bool {
+ added := false
+ for _, a := range addrs {
+ if !slices.Contains(kept, a) {
+ added = true
+ break
+ }
+ }
+ if !added {
+ return false
+ }
+ if p.Kind == PrincipalUser {
+ op, err := s.store.IsOperator(p.Address)
+ if err != nil {
+ log.Printf("console: upstream rail: is-operator %s: %v", p.Address, err)
+ jsonError(w, http.StatusServiceUnavailable, "could not check who may set these upstreams — nothing was changed. Try again.")
+ return true
+ }
+ if op {
+ return false
+ }
+ }
+ jsonError(w, http.StatusForbidden, "only a platform operator can point an instance at its own backends")
+ return true
+}
+
// handleAPIInstanceUpdate repoints a live instance's UPSTREAM SET, probe
// config and (Task 3, multi-region-steering plan) DNS STEERING opt-in — the
// fields of a record that are meant to change while the instance keeps
@@ -1534,6 +1580,12 @@ func (s *Server) handleAPIInstanceUpdate(w http.ResponseWriter, r *http.Request)
jsonError(w, http.StatusBadRequest, err.Error())
return
}
+ // An address the record already carries is not a new statement, so a
+ // member may keep it: dropping one dead upstream from an operator-made
+ // set is theirs to do.
+ if req.Addrs != nil && s.refuseMemberUpstreams(w, p, *req.Addrs, stored.AddrsOrDefault()) {
+ return
+ }
// v1's pool guardrail (multi-region-steering plan, plan-2 ledger):
// latency steering needs the SHARED pool, because a pool's own edges are
// the only ones that can serve a pool-scoped route — a steered
diff --git a/internal/console/apiinstances_test.go b/internal/console/apiinstances_test.go
index c8e15630e..c90473faa 100644
--- a/internal/console/apiinstances_test.go
+++ b/internal/console/apiinstances_test.go
@@ -21,6 +21,7 @@ import (
"net/http"
"net/http/httptest"
"net/url"
+ "os"
"slices"
"sort"
"strings"
@@ -923,6 +924,7 @@ func TestAPIInstanceListHealthAndFilters(t *testing.T) {
// suffix rule refuses).
func TestAPIInstanceCreateCarriesRouteShape(t *testing.T) {
fx, token := instanceFixture(t, "code-inst-shape")
+ letStateUpstreams(t, fx, "alice@keymail.dev")
enableInstances(t, fx, "titogo", titoManifest())
u := instancesURL(fx, "bab", "titogo")
@@ -1027,6 +1029,7 @@ func updateURL(fx *consoleFixture, acct, app, host string) string {
// a bucket write and a fleet-wide wake.
func TestAPIInstanceUpdateUpstreams(t *testing.T) {
fx, token := instanceFixture(t, "code-inst-patch")
+ letStateUpstreams(t, fx, "alice@keymail.dev")
enableInstances(t, fx, "titogo", titoManifest())
u := instancesURL(fx, "bab", "titogo")
if status, _, out := rawReq(t, http.MethodPost, u, token,
@@ -1165,6 +1168,7 @@ func TestAPIInstanceUpdateUpstreams(t *testing.T) {
// of the three answers.
func TestAPIInstanceUpdateRefusesShapeTransitions(t *testing.T) {
fx, token := instanceFixture(t, "code-inst-patch-shape")
+ letStateUpstreams(t, fx, "alice@keymail.dev")
enableInstances(t, fx, "titogo", titoManifest())
u := instancesURL(fx, "bab", "titogo")
// A socket-backed instance (the ordinary provisioned tenant) and an
@@ -1235,6 +1239,7 @@ func TestAPIInstanceUpdateRefusesShapeTransitions(t *testing.T) {
// paid for; a new route is exactly where it gets forgotten.
func TestAPIInstanceUpdateRefusalMatrix(t *testing.T) {
fx, member := instanceFixture(t, "code-inst-patch-refuse")
+ letStateUpstreams(t, fx, "alice@keymail.dev")
enableInstances(t, fx, "titogo", titoManifest())
if status, _, out := rawReq(t, http.MethodPost, instancesURL(fx, "bab", "titogo"), member,
`{"host":"ha.go.tito.io","region":"ie","kind":"service","addrs":["127.0.0.1:7001"]}`); status != http.StatusCreated {
@@ -1278,10 +1283,10 @@ func TestAPIInstanceUpdateRefusalMatrix(t *testing.T) {
if !slices.Equal(rec.Addrs, []string{"127.0.0.1:7001"}) {
t.Fatalf("a refused patch reached the record: %+v", rec)
}
- // An app token MAY patch its own app's instance: this is the door a
- // signup-time app uses to repoint its own upstreams.
+ // An app token MAY patch its own app's instance. Stating a NEW upstream
+ // is an operator's alone (refuseMemberUpstreams), so it patches the probe.
if status, _, out := rawReq(t, http.MethodPatch, updateURL(fx, "bab", "titogo", "ha.go.tito.io"),
- appTok, body); status != http.StatusOK {
+ appTok, `{"probe":{"path":"/up"}}`); status != http.StatusOK {
t.Fatalf("app token patching its own app = %d %s, want 200", status, out)
}
// And a console with no deployment bucket answers 503, not a panic.
@@ -1310,6 +1315,7 @@ func (s staleETagStore) Get(ctx context.Context, key string) ([]byte, string, er
// announced: no write landed, so there is no news.
func TestAPIInstanceUpdateLostRace(t *testing.T) {
fx, token := instanceFixture(t, "code-inst-patch-race")
+ letStateUpstreams(t, fx, "alice@keymail.dev")
enableInstances(t, fx, "titogo", titoManifest())
if status, _, out := rawReq(t, http.MethodPost, instancesURL(fx, "bab", "titogo"), token,
`{"host":"ha.go.tito.io","region":"ie","kind":"service","addrs":["127.0.0.1:7001"]}`); status != http.StatusCreated {
@@ -1351,6 +1357,7 @@ func sharedPoolManifest() provision.Manifest {
// disturbs the addrs/probe fields this same door already owns.
func TestAPIInstanceUpdateSteeringRoundTrip(t *testing.T) {
fx, token := instanceFixture(t, "code-inst-steer-round")
+ letStateUpstreams(t, fx, "alice@keymail.dev")
enableInstances(t, fx, "titogo", sharedPoolManifest())
u := instancesURL(fx, "bab", "titogo")
if status, _, out := rawReq(t, http.MethodPost, u, token,
@@ -1449,6 +1456,7 @@ func lastEventDetail(t *testing.T, fx *consoleFixture, app string) string {
// claim a change the request didn't make (or omit one it did).
func TestAPIInstanceUpdateAuditNamesChangedFields(t *testing.T) {
fx, token := instanceFixture(t, "code-inst-audit-fields")
+ letStateUpstreams(t, fx, "alice@keymail.dev")
enableInstances(t, fx, "titogo", sharedPoolManifest())
u := instancesURL(fx, "bab", "titogo")
if status, _, out := rawReq(t, http.MethodPost, u, token,
@@ -2468,6 +2476,7 @@ func TestAPIInstanceCreateStaticInAPoolDoesNotSteer(t *testing.T) {
// the same flag back.
func TestAPIInstanceUpdateGateRoundTrip(t *testing.T) {
fx, token := instanceFixture(t, "code-inst-gate-round")
+ letStateUpstreams(t, fx, "alice@keymail.dev")
enableInstances(t, fx, "titogo", sharedPoolManifest())
u := instancesURL(fx, "bab", "titogo")
if status, _, out := rawReq(t, http.MethodPost, u, token,
@@ -2535,6 +2544,7 @@ func TestAPIInstanceUpdateGateRoundTrip(t *testing.T) {
// would otherwise land in a log or a terminal's scrollback.
func TestAPIInstanceUpdateGateRefusals(t *testing.T) {
fx, token := instanceFixture(t, "code-inst-gate-refuse")
+ letStateUpstreams(t, fx, "alice@keymail.dev")
enableInstances(t, fx, "titogo", sharedPoolManifest())
u := instancesURL(fx, "bab", "titogo")
if status, _, out := rawReq(t, http.MethodPost, u, token,
@@ -2683,3 +2693,138 @@ func TestAPIInstanceCreateReplicaFields(t *testing.T) {
}
}
}
+
+// letStateUpstreams gives addr the operator bit. Only an operator may point
+// an instance at its own backends (refuseMemberUpstreams), so tests about
+// something else that need an addr-backed record use this to make one.
+func letStateUpstreams(t *testing.T, fx *consoleFixture, addr string) {
+ t.Helper()
+ if err := fx.st.SetOperator(addr, true); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestAPIInstanceUpstreamsAreOperatorOnly: a member may not point an
+// instance at its own backends, at any address, public included — the edge
+// dials these from inside the platform, and a member's could reach the box's
+// own services or another tenant's private upstream (discussion 214, item 4;
+// Paul's ruling on discussion 50, option a). Operators keep the old rail,
+// which is where the platform's own addr-backed routes point.
+//
+// Nothing a member already has is taken away: a record an operator pointed
+// somewhere keeps its upstreams through a member's unrelated PATCH, and the
+// member may restate or drop the addresses already on it.
+func TestAPIInstanceUpstreamsAreOperatorOnly(t *testing.T) {
+ fx, member := instanceFixture(t, "code-inst-internal")
+ enableInstances(t, fx, "titogo", titoManifest())
+ if err := fx.st.SetOperator("paul@keymail.dev", true); err != nil {
+ t.Fatal(err)
+ }
+ operator := cliToken(t, fx, "paul@keymail.dev", "code-inst-internal-op")
+ u := instancesURL(fx, "bab", "titogo")
+ const refusal = "only a platform operator can point an instance at its own backends"
+
+ for _, addr := range []string{"203.0.113.7:443", "127.0.0.1:8090", "10.0.0.5:3000", "[fd00::5]:3000"} {
+ body := `{"host":"in.go.tito.io","region":"ie","kind":"service","addrs":["` + addr + `"]}`
+ status, _, out := rawReq(t, http.MethodPost, u, member, body)
+ if status != http.StatusForbidden || !strings.Contains(out, refusal) {
+ t.Errorf("member create with %s = %d %s, want 403 %q", addr, status, out, refusal)
+ }
+ }
+ if _, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "in.go.tito.io"); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("a refused create left a record behind: %v", err)
+ }
+ // A member's create that states no upstream is untouched by the rail.
+ if status, _, out := rawReq(t, http.MethodPost, u, member, `{"host":"plain.go.tito.io","region":"ie"}`); status != http.StatusCreated {
+ t.Fatalf("member create without addrs = %d %s, want 201", status, out)
+ }
+
+ // The operator's create is the platform's own path, unchanged.
+ if status, _, out := rawReq(t, http.MethodPost, u, operator,
+ `{"host":"in.go.tito.io","region":"ie","kind":"service","addrs":["127.0.0.1:7001","127.0.0.1:7002"]}`); status != http.StatusCreated {
+ t.Fatalf("operator create with loopback = %d %s, want 201", status, out)
+ }
+ pu := updateURL(fx, "bab", "titogo", "in.go.tito.io")
+
+ // A member's PATCH of other fields leaves the upstreams standing.
+ for _, body := range []string{`{"probe":{"path":"/up"}}`, `{"environment":"acme"}`, `{"steering":""}`} {
+ if status, _, out := rawReq(t, http.MethodPatch, pu, member, body); status != http.StatusOK {
+ t.Fatalf("member patch %s on an operator-made route = %d %s, want 200", body, status, out)
+ }
+ }
+ stored, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "in.go.tito.io")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !slices.Equal(stored.Addrs, []string{"127.0.0.1:7001", "127.0.0.1:7002"}) {
+ t.Fatalf("stored addrs after member patches = %v, want the operator's pair untouched", stored.Addrs)
+ }
+ // It may not add one, public or not...
+ for _, body := range []string{`{"addrs":["127.0.0.1:7001","192.168.1.1:7003"]}`, `{"addrs":["127.0.0.1:7001","203.0.113.7:443"]}`} {
+ if status, _, out := rawReq(t, http.MethodPatch, pu, member, body); status != http.StatusForbidden || !strings.Contains(out, refusal) {
+ t.Fatalf("member patch %s = %d %s, want 403 %q", body, status, out, refusal)
+ }
+ }
+ // ...but may restate what is there, or drop a dead one.
+ for _, body := range []string{`{"addrs":["127.0.0.1:7002","127.0.0.1:7001"]}`, `{"addrs":["127.0.0.1:7001"]}`} {
+ if status, _, out := rawReq(t, http.MethodPatch, pu, member, body); status != http.StatusOK {
+ t.Fatalf("member patch %s = %d %s, want 200", body, status, out)
+ }
+ }
+ // The operator's PATCH keeps the old rail.
+ if status, _, out := rawReq(t, http.MethodPatch, pu, operator, `{"addrs":["10.0.0.5:7001"]}`); status != http.StatusOK {
+ t.Fatalf("operator patch to a private addr = %d %s, want 200", status, out)
+ }
+ stored, err = provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "in.go.tito.io")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !slices.Equal(stored.Addrs, []string{"10.0.0.5:7001"}) {
+ t.Fatalf("stored addrs = %v, want the operator's", stored.Addrs)
+ }
+
+ // A service credential is not a person, operator or not.
+ appTok, err := fx.st.NewAppToken(OpsAccountID, "titogo", "paul@keymail.dev", "ci", time.Now())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if status, _, out := rawReq(t, http.MethodPatch, pu, appTok, `{"addrs":["10.0.0.6:7001"]}`); status != http.StatusForbidden {
+ t.Fatalf("app token patch to a new addr = %d %s, want 403", status, out)
+ }
+}
+
+// TestExistingUpstreamRecordsKeepServing: records already in the bucket with
+// upstreams — including the legacy single Addr, written before this rail by
+// whoever could then — still pass the shape rail the box reconciles with
+// (which this branch does not touch; reconcile_test.go's addr-backed
+// records cover the minting) and survive a member's PATCH of unrelated
+// fields. The rail governs only what a member newly states.
+func TestExistingUpstreamRecordsKeepServing(t *testing.T) {
+ fx, member := instanceFixture(t, "code-inst-legacy")
+ enableInstances(t, fx, "titogo", titoManifest())
+ rec := provision.Record{Schema: 1, Host: "legacy.go.tito.io", Slug: "legacy", App: "titogo",
+ Account: "bab", Pool: "tito", Region: "ie", Channel: "stable", Kind: "service",
+ Addr: "10.0.0.1:8090", CreatedAt: time.Now().UTC(), CreatedBy: "alice@keymail.dev"}
+ if err := provision.ValidateShape(rec); err != nil {
+ t.Fatalf("an existing private-addr record no longer validates: %v", err)
+ }
+ if err := provision.CreateRecord(context.Background(), fx.rel, rec); err != nil {
+ t.Fatal(err)
+ }
+ pu := updateURL(fx, "bab", "titogo", "legacy.go.tito.io")
+ if status, _, out := rawReq(t, http.MethodPatch, pu, member, `{"probe":{"path":"/up"}}`); status != http.StatusOK {
+ t.Fatalf("member probe patch on a legacy addr record = %d %s, want 200", status, out)
+ }
+ got, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "legacy.go.tito.io")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !slices.Equal(got.AddrsOrDefault(), []string{"10.0.0.1:8090"}) {
+ t.Fatalf("upstreams after a member probe patch = %v, want the record's own", got.AddrsOrDefault())
+ }
+ // Restating the legacy addr through the addrs door moves it into the set,
+ // as an operator's PATCH would; that is not a new upstream.
+ if status, _, out := rawReq(t, http.MethodPatch, pu, member, `{"addrs":["10.0.0.1:8090"]}`); status != http.StatusOK {
+ t.Fatalf("member restating the legacy addr = %d %s, want 200", status, out)
+ }
+}
diff --git a/internal/console/apiinstancesenvironment_test.go b/internal/console/apiinstancesenvironment_test.go
index 208ebd0a0..8dbad03c1 100644
--- a/internal/console/apiinstancesenvironment_test.go
+++ b/internal/console/apiinstancesenvironment_test.go
@@ -89,6 +89,7 @@ func TestAPIInstanceCreateRefusesBadEnvironment(t *testing.T) {
// convergeOwned opened by reverting `carlos route -environment`.
func TestAPIInstanceUpdateEnvironmentRoundTrip(t *testing.T) {
fx, token := instanceFixture(t, "code-inst-env-round")
+ letStateUpstreams(t, fx, "alice@keymail.dev")
enableInstances(t, fx, "titogo", titoManifest())
u := instancesURL(fx, "bab", "titogo")
if status, _, out := rawReq(t, http.MethodPost, u, token,
diff --git a/internal/console/appsettings_gate_test.go b/internal/console/appsettings_gate_test.go
index 340d742a2..e62d138cb 100644
--- a/internal/console/appsettings_gate_test.go
+++ b/internal/console/appsettings_gate_test.go
@@ -12,6 +12,7 @@ import (
func TestAppSettingsGateSetAndClear(t *testing.T) {
fx, token := instanceFixture(t, "code-settings-gate")
+ letStateUpstreams(t, fx, "alice@keymail.dev")
enableInstances(t, fx, "titogo", sharedPoolManifest())
if status, _, out := rawReq(t, http.MethodPost, instancesURL(fx, "bab", "titogo"), token,
`{"host":"pw.go.tito.io","region":"ie","addrs":["127.0.0.1:7001"]}`); status != http.StatusCreated {
@@ -88,6 +89,7 @@ func TestAppSettingsGateSetAndClear(t *testing.T) {
// account out of its own site.
func TestAppSettingsGateIsNotAnOracle(t *testing.T) {
fx, token := instanceFixture(t, "code-settings-gate-stranger")
+ letStateUpstreams(t, fx, "alice@keymail.dev")
enableInstances(t, fx, "titogo", sharedPoolManifest())
if status, _, out := rawReq(t, http.MethodPost, instancesURL(fx, "bab", "titogo"), token,
`{"host":"pw.go.tito.io","region":"ie","addrs":["127.0.0.1:7001"]}`); status != http.StatusCreated {
diff --git a/internal/console/authcli.go b/internal/console/authcli.go
index 9bd9e0a22..e1653a5fa 100644
--- a/internal/console/authcli.go
+++ b/internal/console/authcli.go
@@ -891,6 +891,11 @@ func (s *Server) registerCLIRoutes() {
// client typed, and honouring it would let one caller present a fresh "IP"
// per request and walk straight past every per-IP limit below.
//
+// Loopback is not the edge alone, though: any process on this box can
+// connect to it. A loopback peer's chain is therefore believed only with the
+// edge's signature beside it (forwardedByEdge, edgeproof.go); without one the
+// request counts against the peer address like any other.
+//
// When the peer is trusted, the LAST element is the one to take: the edge
// appends its own view of the peer with SetXForwarded, so everything before
// it may have been supplied by the client.
@@ -902,11 +907,18 @@ func (s *Server) clientIP(r *http.Request) string {
if !s.trustsProxy(host) {
return host
}
- if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
- parts := strings.Split(xff, ",")
- if ip := strings.TrimSpace(parts[len(parts)-1]); ip != "" {
- return ip
- }
+ // Every line, joined: that is the chain the edge signed, and a second
+ // line is how a client would try to slip its own element in.
+ xff := strings.Join(r.Header.Values("X-Forwarded-For"), ", ")
+ if xff == "" {
+ return host
+ }
+ if isLoopbackPeer(host) && !s.forwardedByEdge(r, xff) {
+ return host
+ }
+ parts := strings.Split(xff, ",")
+ if ip := strings.TrimSpace(parts[len(parts)-1]); ip != "" {
+ return ip
}
return host
}
diff --git a/internal/console/authcli_test.go b/internal/console/authcli_test.go
index 12c0546e9..94ae6e206 100644
--- a/internal/console/authcli_test.go
+++ b/internal/console/authcli_test.go
@@ -7,6 +7,7 @@ import (
"io"
"net/http"
"net/url"
+ "path/filepath"
"strconv"
"strings"
"testing"
@@ -280,7 +281,14 @@ func TestCLIIssuanceNotifiesTheUser(t *testing.T) {
// X-Forwarded-For is only believed from a peer we put there. Otherwise one
// caller presents a fresh "IP" per request and walks past every per-IP limit.
func TestClientIPIgnoresUntrustedForwardedFor(t *testing.T) {
- s := &Server{cfg: Config{EdgeIPs: []string{"198.51.100.9"}}}
+ st, err := Open(filepath.Join(t.TempDir(), "console.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { st.Close() })
+ // No bucket, so no mesh key and no proof has ever been seen: loopback
+ // keeps the trust it had before proofs existed (edgeproof.go).
+ s := &Server{cfg: Config{EdgeIPs: []string{"198.51.100.9"}}, store: st}
r := &http.Request{Header: http.Header{}, RemoteAddr: "203.0.113.5:41234"}
r.Header.Set("X-Forwarded-For", "1.2.3.4")
if got := s.clientIP(r); got != "203.0.113.5" {
diff --git a/internal/console/controlplane_test.go b/internal/console/controlplane_test.go
index 9e13d9fb4..024167233 100644
--- a/internal/console/controlplane_test.go
+++ b/internal/console/controlplane_test.go
@@ -965,6 +965,7 @@ func TestControlPlaneAppRefusesInstanceDelete(t *testing.T) {
// one. A create gate here would close the door the flag exists to keep open.
func TestControlPlaneAppAllowsAPersonToCreateAndPatchItsInstance(t *testing.T) {
fx, token := cpInstanceFixture(t, "code-cp-inst-person")
+ letStateUpstreams(t, fx, "paul@keymail.dev")
u := instancesURL(fx, "bab", "console")
if status, _, out := rawReq(t, http.MethodPost, u, token,
`{"host":"box1.cp.tito.io","region":"ie","kind":"service","addrs":["127.0.0.1:9001"]}`); status != http.StatusCreated {
@@ -1019,6 +1020,7 @@ func TestControlPlaneAppRefusesAServiceInstanceCreate(t *testing.T) {
// every pass, so a PATCH repoints the control plane. Same split as create.
func TestControlPlaneAppRefusesAServiceInstancePatch(t *testing.T) {
fx, token := cpInstanceFixture(t, "code-cp-inst-svc-patch")
+ letStateUpstreams(t, fx, "paul@keymail.dev")
u := instancesURL(fx, "bab", "console")
if status, _, out := rawReq(t, http.MethodPost, u, token,
`{"host":"box3.cp.tito.io","region":"ie","kind":"service","addrs":["127.0.0.1:9001"]}`); status != http.StatusCreated {
@@ -1041,14 +1043,16 @@ func TestControlPlaneAppRefusesAServiceInstancePatch(t *testing.T) {
t.Fatalf("addrs after a refused service patch = %v, want the created pair untouched", rec.Addrs)
}
- // An ordinary app's record is still a service's to patch.
+ // An ordinary app's record is still a service's to patch. An operator
+ // states its upstream (only one may: refuseMemberUpstreams); the service
+ // then restates it, which is not a new one.
ws := serviceToken(t, fx.st, OpsAccountID, "ci-widget", "widget", svcrole.Instance)
- if status, _, out := rawReq(t, http.MethodPost, instancesURL(fx, "bab", "widget"), ws,
+ if status, _, out := rawReq(t, http.MethodPost, instancesURL(fx, "bab", "widget"), token,
`{"host":"box3.widget.tito.io","region":"ie","kind":"service","addrs":["127.0.0.1:9001"]}`); status != http.StatusCreated {
t.Fatalf("ordinary create = %d %s", status, out)
}
if status, _, out := rawReq(t, http.MethodPatch, updateURL(fx, "bab", "widget", "box3.widget.tito.io"), ws,
- `{"addrs":["127.0.0.1:9002"]}`); status != http.StatusOK {
+ `{"addrs":["127.0.0.1:9001"]}`); status != http.StatusOK {
t.Fatalf("a SERVICE patching an ORDINARY app's instance = %d %s, want 200", status, out)
}
}
diff --git a/internal/console/edgeproof.go b/internal/console/edgeproof.go
new file mode 100644
index 000000000..5c929111b
--- /dev/null
+++ b/internal/console/edgeproof.go
@@ -0,0 +1,172 @@
+package console
+
+// Who sent this X-Forwarded-For (discussion 214, item 5).
+//
+// The console listens on 127.0.0.1, behind the edge on its own box, so every
+// proxied request arrives from loopback and the client's address is the
+// edge's X-Forwarded-For. But loopback is not the edge alone: any tenant
+// process on the console's box can connect to 127.0.0.1:8090 and send any
+// X-Forwarded-For it likes, one fresh address per request, past every per-IP
+// limit on the device-code login. So a loopback peer's chain is believed only
+// with the edge's signature over it (internal/edgeproof), checked against the
+// deployment's mesh key, which tenants cannot read.
+//
+// Rollout needs one more rule. An edge that predates the proof sends none,
+// and refusing its chains would put every client in the deployment into the
+// single 127.0.0.1 rate-limit bucket — ten login starts a minute for
+// everybody — until the edge caught up. The console therefore keeps the old
+// trust until it has seen ONE valid proof, and from then on requires one for
+// good. Only the real edge can produce that proof, and the moment is stored
+// (edge_proof_seen) so a restart does not reopen the window. Rolling the edge
+// back past the proof after that is the one way to land in the shared
+// bucket; the console logs it once when it happens.
+//
+// The latch must only close on a proof from THIS box's edge. Far edges send
+// none across a backhaul hop (internal/edge's backhaulProxy) for exactly
+// that reason: an older home edge would pass a far edge's proof straight
+// through, and the console would latch while its own edge still signed
+// nothing. The proof also binds the request's Host, so a tenant app on this
+// box cannot replay the proofs it receives on its own traffic here.
+
+import (
+ "context"
+ "database/sql"
+ "log"
+ "net"
+ "net/http"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/carlosframework/platform/internal/edgeproof"
+)
+
+// edgeProofKeyTTL is how long one read of the mesh keys stands. The edge
+// polls the same objects every 30s, so a rotation reaches both sides on the
+// same scale.
+const edgeProofKeyTTL = 30 * time.Second
+
+// edgeProofKeyTimeout bounds one key read: it runs on the login path, and a
+// hung store must cost a bounded wait, not a wedged door.
+const edgeProofKeyTimeout = 5 * time.Second
+
+// edgeProofState is the cached key pair and the seen latch. Zero value is
+// ready to use.
+type edgeProofState struct {
+ mu sync.Mutex
+ at time.Time
+ current, next []byte
+
+ // loaded is set once the stored mark has been read; seen is the latch.
+ loaded atomic.Bool
+ seen atomic.Bool
+ warned atomic.Bool
+}
+
+// forwardedByEdge reports whether a loopback peer's X-Forwarded-For chain
+// (xff, its lines comma-joined) came from the edge.
+func (s *Server) forwardedByEdge(r *http.Request, xff string) bool {
+ st := &s.edgeProof
+ if proof := r.Header.Get(edgeproof.Header); proof != "" {
+ current, next := s.edgeProofKeys(r.Context())
+ if edgeproof.Verify(current, next, r.Host, xff, proof, time.Now()) {
+ s.markEdgeProofSeen()
+ return true
+ }
+ }
+ if !s.edgeProofSeen() {
+ return true // no proof yet from this deployment's edge: the old trust
+ }
+ if st.warned.CompareAndSwap(false, true) {
+ log.Printf("console: a loopback request carried X-Forwarded-For with no valid edge proof; counting it against the peer address. Expected now and then from a process on this box; on every request it means the edge no longer signs (rolled back?) and every client shares one rate limit")
+ }
+ return false
+}
+
+// edgeProofSeen answers the latch, reading the stored row once per process.
+// A read failure counts as seen for this request and is retried on the
+// next: a database blip must not be the way back to the old trust.
+func (s *Server) edgeProofSeen() bool {
+ st := &s.edgeProof
+ if st.seen.Load() {
+ return true
+ }
+ if st.loaded.Load() {
+ return false
+ }
+ seen, err := s.store.EdgeProofSeen()
+ if err != nil {
+ log.Printf("console: edge proof: read the seen mark: %v", err)
+ return true
+ }
+ if seen {
+ st.seen.Store(true)
+ }
+ st.loaded.Store(true)
+ return seen
+}
+
+func (s *Server) markEdgeProofSeen() {
+ st := &s.edgeProof
+ if st.seen.Load() {
+ return
+ }
+ if err := s.store.MarkEdgeProofSeen(time.Now()); err != nil {
+ log.Printf("console: edge proof: store the seen mark: %v", err)
+ } else {
+ log.Printf("console: edge proof: first signed request from the edge — X-Forwarded-For from loopback now needs a proof")
+ }
+ st.seen.Store(true)
+}
+
+// edgeProofKeys is the cached mesh key pair. With no deployment bucket there
+// are no keys, nothing verifies, and the latch never closes — the old trust,
+// on a console that cannot have an edge signing for it anyway. A failed read
+// keeps the last good pair.
+func (s *Server) edgeProofKeys(ctx context.Context) (current, next []byte) {
+ st := &s.edgeProof
+ if s.cfg.Releases == nil {
+ return nil, nil
+ }
+ st.mu.Lock()
+ defer st.mu.Unlock()
+ if !st.at.IsZero() && time.Since(st.at) < edgeProofKeyTTL {
+ return st.current, st.next
+ }
+ ctx, cancel := context.WithTimeout(ctx, edgeProofKeyTimeout)
+ defer cancel()
+ c, n, err := edgeproof.LoadKeys(ctx, s.cfg.Releases)
+ // Stamped either way, so a store that is down is asked once per TTL
+ // rather than on every login request.
+ st.at = time.Now()
+ if err != nil {
+ log.Printf("console: edge proof: %v", err)
+ return st.current, st.next
+ }
+ st.current, st.next = c, n
+ return c, n
+}
+
+// isLoopbackPeer reports whether host (a RemoteAddr's host part) is this box.
+func isLoopbackPeer(host string) bool {
+ ip := net.ParseIP(host)
+ return ip != nil && ip.IsLoopback()
+}
+
+// EdgeProofSeen reports whether this console has ever received a request
+// carrying a valid edge forwarded-for proof.
+func (s *Store) EdgeProofSeen() (bool, error) {
+ var at int64
+ err := s.db.QueryRow(`SELECT seen_at FROM edge_proof_seen WHERE id = 1`).Scan(&at)
+ if err == sql.ErrNoRows {
+ return false, nil
+ }
+ return err == nil, err
+}
+
+// MarkEdgeProofSeen records the first valid proof. Later calls keep the
+// first moment.
+func (s *Store) MarkEdgeProofSeen(now time.Time) error {
+ _, err := s.db.Exec(`INSERT OR IGNORE INTO edge_proof_seen (id, seen_at) VALUES (1, ?)`, now.Unix())
+ return err
+}
diff --git a/internal/console/edgeproof_test.go b/internal/console/edgeproof_test.go
new file mode 100644
index 000000000..f6607032a
--- /dev/null
+++ b/internal/console/edgeproof_test.go
@@ -0,0 +1,122 @@
+package console
+
+import (
+ "context"
+ "encoding/hex"
+ "net/http"
+ "net/http/httptest"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/edgeproof"
+)
+
+// TestClientIPLoopbackNeedsTheEdgeProof: a process on the console's box can
+// reach 127.0.0.1:8090 as easily as the edge can, so once the edge signs its
+// X-Forwarded-For, a loopback chain without that signature is a string the
+// caller typed and counts against the peer (discussion 214, item 5). Before
+// the first signed request the old trust stands, so a console rolled ahead
+// of its edge does not put every client in one rate-limit bucket; after it,
+// the latch holds across a restart.
+func TestClientIPLoopbackNeedsTheEdgeProof(t *testing.T) {
+ fx := newConsoleFixture(t)
+ key := []byte("0123456789abcdef0123456789abcdef")
+ if _, err := fx.rel.Put(context.Background(), edgeproof.MeshKeyObject, []byte(hex.EncodeToString(key)), ""); err != nil {
+ t.Fatal(err)
+ }
+ req := func(xff, proof string) *http.Request {
+ r := httptest.NewRequest(http.MethodPost, "/api/cli/auth/start", nil)
+ r.RemoteAddr = "127.0.0.1:50000"
+ r.Header.Set("X-Forwarded-For", xff)
+ if proof != "" {
+ r.Header.Set(edgeproof.Header, proof)
+ }
+ return r
+ }
+
+ // Before any proof: an unsigned chain is still believed.
+ if got := fx.srv.clientIP(req("9.9.9.9", "")); got != "9.9.9.9" {
+ t.Fatalf("unsigned chain before any proof = %q, want the old trust (9.9.9.9)", got)
+ }
+ // A forged proof is not a proof, and does not close the latch.
+ if got := fx.srv.clientIP(req("9.9.9.9", edgeproof.Sign([]byte("not the mesh key"), "example.com", "9.9.9.9", time.Now()))); got != "9.9.9.9" {
+ t.Fatalf("forged proof before any real one = %q, want the old trust", got)
+ }
+ if seen, _ := fx.st.EdgeProofSeen(); seen {
+ t.Fatal("a forged proof closed the latch")
+ }
+
+ // The edge's signed chain is believed, and closes the latch.
+ if got := fx.srv.clientIP(req("1.2.3.4", edgeproof.Sign(key, "example.com", "1.2.3.4", time.Now()))); got != "1.2.3.4" {
+ t.Fatalf("signed chain = %q, want 1.2.3.4", got)
+ }
+ // From now on: no proof, a forged proof, a proof for another chain, or a
+ // stale one all count against the peer.
+ for what, r := range map[string]*http.Request{
+ "no proof": req("5.6.7.8", ""),
+ "forged": req("5.6.7.8", edgeproof.Sign([]byte("not the mesh key"), "example.com", "5.6.7.8", time.Now())),
+ "other chain": req("5.6.7.8", edgeproof.Sign(key, "example.com", "1.2.3.4", time.Now())),
+ "stale": req("5.6.7.8", edgeproof.Sign(key, "example.com", "5.6.7.8", time.Now().Add(-2*edgeproof.Skew))),
+ "extra xff line": func() *http.Request {
+ r := req("1.2.3.4", edgeproof.Sign(key, "example.com", "1.2.3.4", time.Now()))
+ r.Header.Add("X-Forwarded-For", "5.6.7.8")
+ return r
+ }(),
+ } {
+ if got := fx.srv.clientIP(r); got != "127.0.0.1" {
+ t.Errorf("%s after the latch = %q, want the peer 127.0.0.1", what, got)
+ }
+ }
+ // A proof is bound to the host it was minted for: a tenant app on this
+ // box sees valid proofs on its own traffic, and must not be able to
+ // replay one here as the visitor it names.
+ if got := fx.srv.clientIP(req("5.6.7.8", edgeproof.Sign(key, "tenant.example.net", "5.6.7.8", time.Now()))); got != "127.0.0.1" {
+ t.Errorf("proof minted for another host = %q, want the peer 127.0.0.1", got)
+ }
+ // The host's port and case are not part of what is bound.
+ spelled := req("5.6.7.8", edgeproof.Sign(key, "Example.COM", "5.6.7.8", time.Now()))
+ spelled.Host = "example.com:443"
+ if got := fx.srv.clientIP(spelled); got != "5.6.7.8" {
+ t.Errorf("proof with the host in another spelling = %q, want 5.6.7.8", got)
+ }
+ // A signed multi-element chain still yields its last element.
+ if got := fx.srv.clientIP(req("9.9.9.9, 1.2.3.4", edgeproof.Sign(key, "example.com", "9.9.9.9, 1.2.3.4", time.Now()))); got != "1.2.3.4" {
+ t.Fatalf("signed chain of two = %q, want 1.2.3.4", got)
+ }
+
+ // The latch is stored: a fresh process on the same database starts closed.
+ again := &Server{cfg: fx.srv.cfg, store: fx.st}
+ if got := again.clientIP(req("5.6.7.8", "")); got != "127.0.0.1" {
+ t.Fatalf("unsigned chain after a restart = %q, want the peer", got)
+ }
+ // And a configured edge address is still believed without a proof: it is
+ // not a peer a tenant on this box can be.
+ r := req("5.6.7.8", "")
+ r.RemoteAddr = "198.51.100.9:9"
+ edged := &Server{cfg: Config{EdgeIPs: []string{"198.51.100.9"}, Releases: fx.rel}, store: fx.st}
+ if got := edged.clientIP(r); got != "5.6.7.8" {
+ t.Fatalf("configured edge peer = %q, want 5.6.7.8", got)
+ }
+}
+
+// TestEdgeProofSeenMark: the stored latch is one row and keeps its first
+// moment.
+func TestEdgeProofSeenMark(t *testing.T) {
+ st, err := Open(filepath.Join(t.TempDir(), "console.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer st.Close()
+ if seen, err := st.EdgeProofSeen(); err != nil || seen {
+ t.Fatalf("fresh store seen = %v, %v; want false, nil", seen, err)
+ }
+ for i := 0; i < 2; i++ {
+ if err := st.MarkEdgeProofSeen(time.Now()); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if seen, err := st.EdgeProofSeen(); err != nil || !seen {
+ t.Fatalf("marked store seen = %v, %v; want true, nil", seen, err)
+ }
+}
diff --git a/internal/console/placementdoors_test.go b/internal/console/placementdoors_test.go
index da30abc65..47bc00185 100644
--- a/internal/console/placementdoors_test.go
+++ b/internal/console/placementdoors_test.go
@@ -59,6 +59,7 @@ func TestCreateDoorLetsStaticIntoANonHostingRegion(t *testing.T) {
// it needs the region to exist, not to host.
func TestCreateDoorLetsAddrBackedServiceIntoANonHostingRegion(t *testing.T) {
fx, token := instanceFixture(t, "code-inst-svctonohost")
+ letStateUpstreams(t, fx, "alice@keymail.dev")
m := titoManifest()
m.Regions = nil
m.Pool = ""
diff --git a/internal/console/regionfill_test.go b/internal/console/regionfill_test.go
index 7f5e095aa..100ae7778 100644
--- a/internal/console/regionfill_test.go
+++ b/internal/console/regionfill_test.go
@@ -56,7 +56,8 @@ func mustCreate(t *testing.T, u, token, body string) fillRec {
}
func TestFillFromTheAppRegion(t *testing.T) {
- _, token, u := fillFixture(t, "uk")
+ fx, token, u := fillFixture(t, "uk")
+ letStateUpstreams(t, fx, "alice@keymail.dev") // one case below is addr-backed
if rec := mustCreate(t, u, token, `{"host":"a.go.tito.io"}`); rec.Region != "uk" || rec.RegionFrom != "app" || rec.RegionName != ukPlace {
t.Fatalf("fill = %+v", rec)
}
@@ -231,6 +232,7 @@ func TestAppDerivedRefusals(t *testing.T) {
// region keyless, and only for what needs config on the box.
func TestExplicitCreateIntoAKeylessRegion(t *testing.T) {
fx, token, u := fillFixture(t, "")
+ letStateUpstreams(t, fx, "alice@keymail.dev") // one case below is addr-backed
setEdges(t, fx, map[string]string{"uk-box": ukNoKey})
if status, _, out := create(t, u, token, `{"host":"a.go.tito.io","region":"uk"}`); status != http.StatusBadRequest || !strings.Contains(out, msgNoSecretsKey("uk")) {
t.Fatalf("keyless instance = %d %s, want C31", status, out)
diff --git a/internal/console/server.go b/internal/console/server.go
index e9bded56f..bcfead4c3 100644
--- a/internal/console/server.go
+++ b/internal/console/server.go
@@ -935,6 +935,9 @@ type Server struct {
// plus one Get per edges file behind every row of every listing. Zero
// value is ready to use.
edgeFold edgeFoldCache
+ // edgeProof is the mesh-key cache and the seen latch behind clientIP's
+ // loopback check (edgeproof.go). Zero value is ready to use.
+ edgeProof edgeProofState
// foldEdgeIPs is edgeFoldCache's test seam: nil means the real fold
// over cfg.Releases (s.realFoldEdgeIPs). Tests inject a counter so the
// TTL can be proven without a bucket, the same way storeSizer and
diff --git a/internal/console/store.go b/internal/console/store.go
index a442ce3fe..35597f3bb 100644
--- a/internal/console/store.go
+++ b/internal/console/store.go
@@ -311,6 +311,13 @@ func (s *Store) migrate() error {
// backlog.
`CREATE INDEX IF NOT EXISTS mail_event_deliveries_app_due ON mail_event_deliveries (account, app, delivered_at, abandoned_at, next_at);`,
`CREATE INDEX IF NOT EXISTS mail_event_deliveries_by_age ON mail_event_deliveries (created_at);`,
+ // One row, written the first time a request arrives carrying a valid
+ // edge forwarded-for proof (edgeproof.go). From then on a loopback
+ // peer's X-Forwarded-For is believed only with a proof beside it.
+ `CREATE TABLE IF NOT EXISTS edge_proof_seen (
+ id INTEGER PRIMARY KEY CHECK (id = 1),
+ seen_at INTEGER NOT NULL
+);`,
} {
if _, err := s.db.Exec(stmt); err != nil {
return err
diff --git a/internal/edge/edge.go b/internal/edge/edge.go
index d57020f91..a4222f36b 100644
--- a/internal/edge/edge.go
+++ b/internal/edge/edge.go
@@ -51,6 +51,7 @@ import (
"golang.org/x/crypto/acme/autocert"
"github.com/carlosframework/platform/internal/apperr"
+ "github.com/carlosframework/platform/internal/edgeproof"
"github.com/carlosframework/platform/internal/edgesfile"
"github.com/carlosframework/platform/internal/gate"
"github.com/carlosframework/platform/internal/provision"
@@ -1400,6 +1401,9 @@ func (e *Edge) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// On a request carrying no X-Carlos-Hop this is one Header.Get and
// nothing else.
r, hopped := e.acceptHop(r, host)
+ // The forwarded-for proof's key (forwardproof.go), after the hop has
+ // settled which X-Forwarded-For this edge will send on.
+ r = e.withForwardKey(r)
up, ok := e.lookupUpstreamPath(host, r.URL.Path)
// A local match through a one-label "*.<parent>" key (a route
@@ -2861,6 +2865,13 @@ func backhaulProxy(host, eip, box, auth string, tr http.RoundTripper, logThrottl
return &httputil.ReverseProxy{
Rewrite: func(pr *httputil.ProxyRequest) {
setForwarded(pr)
+ // No forwarded-for proof across a hop. The proof is for the
+ // upstream the home edge dials, and the home edge signs that
+ // itself. An older home edge would pass this one through
+ // untouched, and the console would take it as the sign that its
+ // own edge signs, while its local traffic carried none
+ // (edgeproof.go's rollout rule, in internal/console).
+ pr.Out.Header.Del(edgeproof.Header)
pr.Out.URL.Scheme = "https"
pr.Out.URL.Host = host // SNI + verification + pool key; the dial is the transport's
pr.Out.Host = pr.In.Host // the home edge routes on the real hostname
diff --git a/internal/edge/forwardproof.go b/internal/edge/forwardproof.go
new file mode 100644
index 000000000..757df2e0b
--- /dev/null
+++ b/internal/edge/forwardproof.go
@@ -0,0 +1,52 @@
+package edge
+
+// The forwarded-for proof (internal/edgeproof): the edge signs the
+// X-Forwarded-For chain it sends upstream, so an app on this box can tell a
+// client address the edge saw from one a local process typed. The console's
+// per-IP login limits are the reason (discussion 214, item 5): every request
+// the edge proxies to it arrives from loopback, and so does every request a
+// tenant process on the same box makes to 127.0.0.1:8090 directly.
+
+import (
+ "context"
+ "net/http"
+ "net/http/httputil"
+ "strings"
+ "time"
+
+ "github.com/carlosframework/platform/internal/edgeproof"
+)
+
+type forwardKeyCtx struct{}
+
+// withForwardKey puts the current signing key on the request, read once per
+// request from the home map's cached pair, so the proxies' Rewrite funcs —
+// which have no Edge to ask — can sign what they send. No key (no store,
+// nothing minted yet) leaves the request as it is, and nothing is signed.
+func (e *Edge) withForwardKey(r *http.Request) *http.Request {
+ key := signingKey(e.homes.Load().meshKeys())
+ if len(key) == 0 {
+ return r
+ }
+ return r.WithContext(context.WithValue(r.Context(), forwardKeyCtx{}, key))
+}
+
+// signForwarded replaces whatever proof header the inbound request carried
+// with the edge's own signature over the outbound chain. The delete is
+// unconditional: a client-sent proof is at best a replay, and with no key to
+// sign, the honest outbound request carries none.
+func signForwarded(pr *httputil.ProxyRequest) {
+ pr.Out.Header.Del(edgeproof.Header)
+ key, _ := pr.In.Context().Value(forwardKeyCtx{}).([]byte)
+ if len(key) == 0 {
+ return
+ }
+ xff := strings.Join(pr.Out.Header.Values("X-Forwarded-For"), ", ")
+ if xff == "" {
+ return
+ }
+ // pr.In.Host, because every Rewrite here sends it on as pr.Out.Host
+ // (and runs setForwarded before it does): it is the Host the upstream
+ // will see and verify against.
+ pr.Out.Header.Set(edgeproof.Header, edgeproof.Sign(key, pr.In.Host, xff, time.Now()))
+}
diff --git a/internal/edge/forwardproof_test.go b/internal/edge/forwardproof_test.go
new file mode 100644
index 000000000..ef9070eca
--- /dev/null
+++ b/internal/edge/forwardproof_test.go
@@ -0,0 +1,150 @@
+package edge
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/edgeproof"
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// proofBackend records the forwarded chain and proof of the last request.
+type proofBackend struct {
+ mu sync.Mutex
+ xff, proof string
+}
+
+func (b *proofBackend) get() (string, string) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ return b.xff, b.proof
+}
+
+// proofEdge serves app.test from a recording backend. With armed, the edge
+// has a home map carrying a minted mesh key; without, it has no store at all.
+func proofEdge(t *testing.T, armed bool) (*Edge, *proofBackend, []byte) {
+ t.Helper()
+ b := &proofBackend{}
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ b.mu.Lock()
+ b.xff = strings.Join(r.Header.Values("X-Forwarded-For"), ", ")
+ b.proof = r.Header.Get(edgeproof.Header)
+ b.mu.Unlock()
+ }))
+ t.Cleanup(ts.Close)
+ reg := openReg(t)
+ if err := reg.Upsert(registry.Route{Host: "app.test", Kind: "service", Addr: ts.Listener.Addr().String(), Status: "ready"}); err != nil {
+ t.Fatal(err)
+ }
+ e, err := New(reg, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(e.probers.Stop)
+ if !armed {
+ return e, b, nil
+ }
+ s := meshTestStore(t)
+ if err := mintMeshKey(context.Background(), s); err != nil {
+ t.Fatal(err)
+ }
+ m := newHomeMap(s, "annie")
+ refreshHomeMap(t, m)
+ e.homes.Store(m)
+ key, _ := m.meshKeys()
+ return e, b, key
+}
+
+// TestEdgeSignsTheForwardedChain: what the edge proxies carries its signature
+// over the X-Forwarded-For it sends, so the console on this box can tell the
+// edge from a local process writing the same header (discussion 214, item
+// 5). A proof the client sent is replaced, never passed on.
+func TestEdgeSignsTheForwardedChain(t *testing.T) {
+ e, b, key := proofEdge(t, true)
+ forged := edgeproof.Sign(key, "app.test", "6.6.6.6", time.Now())
+ resp := hopRequest(t, e, "app.test", map[string]string{
+ "X-Forwarded-For": "6.6.6.6",
+ edgeproof.Header: forged,
+ })
+ if resp.StatusCode != http.StatusOK {
+ t.Fatalf("status = %d, want 200", resp.StatusCode)
+ }
+ xff, proof := b.get()
+ if strings.Contains(xff, "6.6.6.6") {
+ t.Fatalf("X-Forwarded-For = %q: an ordinary client's chain reached the app", xff)
+ }
+ if proof == "" || proof == forged {
+ t.Fatalf("proof = %q, want the edge's own signature (client sent %q)", proof, forged)
+ }
+ if !edgeproof.Verify(key, nil, "app.test", xff, proof, time.Now()) {
+ t.Fatalf("proof %q does not verify over the chain sent (%q)", proof, xff)
+ }
+
+ // A verified hop's chain is the one this edge sends on, and it is signed
+ // as sent.
+ resp = hopRequest(t, e, "app.test", map[string]string{
+ hopHeader: "1",
+ hopAuthHeader: signHop(key, "app.test", time.Now()),
+ "X-Forwarded-For": "203.0.113.9",
+ })
+ if resp.StatusCode != http.StatusOK {
+ t.Fatalf("hop status = %d, want 200", resp.StatusCode)
+ }
+ xff, proof = b.get()
+ if xff != "203.0.113.9" || !edgeproof.Verify(key, nil, "app.test", xff, proof, time.Now()) {
+ t.Fatalf("hop forwarded xff=%q proof=%q, want the hop's chain, signed", xff, proof)
+ }
+}
+
+// TestEdgeWithoutAKeyStripsTheProof: no mesh key, no signature — and a proof
+// header the client sent still never reaches the app.
+func TestEdgeWithoutAKeyStripsTheProof(t *testing.T) {
+ e, b, _ := proofEdge(t, false)
+ resp := hopRequest(t, e, "app.test", map[string]string{
+ edgeproof.Header: edgeproof.Sign([]byte("k"), "app.test", "1.2.3.4", time.Now()),
+ })
+ if resp.StatusCode != http.StatusOK {
+ t.Fatalf("status = %d, want 200", resp.StatusCode)
+ }
+ if _, proof := b.get(); proof != "" {
+ t.Fatalf("proof = %q, want none from an edge with no key", proof)
+ }
+}
+
+// The proof MAC must never double as a hop MAC: both are keyed by the mesh
+// key, and a tenant app sees the proof on its own traffic.
+func TestForwardProofIsNotAHopAuth(t *testing.T) {
+ key := []byte("0123456789abcdef0123456789abcdef")
+ now := time.Now()
+ if verifyHop(key, "app.test", edgeproof.Sign(key, "app.test", "", now), now) {
+ t.Fatal("a forwarded-for proof verified as a hop signature")
+ }
+ if edgeproof.Verify(key, nil, "app.test", "", signHop(key, "app.test", now), now) {
+ t.Fatal("a hop signature verified as a forwarded-for proof")
+ }
+}
+
+// TestBackhaulCarriesNoForwardProof: the proof is for the upstream an edge
+// dials, so a backhauled request leaves without one, whatever the client
+// sent. An older home edge would otherwise pass the far edge's proof to the
+// console untouched, and the console would start requiring proofs its own
+// edge does not yet send.
+func TestBackhaulCarriesNoForwardProof(t *testing.T) {
+ e, m, key := hopEdge(t, "annie", "local.test")
+ rec, eip := fakeHomeEdge(t, nil)
+ homedOn(t, m, "betty", eip, "example.com")
+ resp := hopRequest(t, e, "example.com", map[string]string{
+ edgeproof.Header: edgeproof.Sign(key, "example.com", "1.2.3.4", time.Now()),
+ })
+ if resp.StatusCode != http.StatusOK {
+ t.Fatalf("status = %d, want 200", resp.StatusCode)
+ }
+ if got := rec.get(); got.n != 1 || got.proof != "" {
+ t.Fatalf("home edge saw %d request(s) with proof %q, want one with none", got.n, got.proof)
+ }
+}
diff --git a/internal/edge/homemap.go b/internal/edge/homemap.go
index 483e81313..dfc1c85ec 100644
--- a/internal/edge/homemap.go
+++ b/internal/edge/homemap.go
@@ -1048,6 +1048,9 @@ func (m *homeMap) logStaleHop(host string) {
// before it.
func setForwarded(pr *httputil.ProxyRequest) {
pr.SetXForwarded()
+ // Last, whichever branch below returns: the proof signs the chain as it
+ // finally leaves (forwardproof.go).
+ defer signForwarded(pr)
// A door request is a request from this box (door.go): its RemoteAddr is
// already the box's public address, so SetXForwarded wrote the right
// chain. Only the scheme differs from the public route to the same URL,
diff --git a/internal/edge/homemap_test.go b/internal/edge/homemap_test.go
index 2b9c37d38..5c9d5e0e9 100644
--- a/internal/edge/homemap_test.go
+++ b/internal/edge/homemap_test.go
@@ -17,6 +17,7 @@ import (
"testing"
"time"
+ "github.com/carlosframework/platform/internal/edgeproof"
"github.com/carlosframework/platform/internal/registry"
"github.com/carlosframework/platform/internal/release"
)
@@ -675,6 +676,7 @@ func TestOrdinaryClientForgedXFFIsStillDiscarded(t *testing.T) {
type homeEdgeSeen struct {
n int
sni, host, hop, auth string
+ proof string
xff []string
}
@@ -716,6 +718,7 @@ func fakeHomeEdge(t *testing.T, h http.Handler) (*homeEdgeRecord, string) {
rec.seen.hop = r.Header.Get(hopHeader)
rec.seen.auth = r.Header.Get(hopAuthHeader)
rec.seen.xff = r.Header.Values("X-Forwarded-For")
+ rec.seen.proof = r.Header.Get(edgeproof.Header)
rec.mu.Unlock()
h.ServeHTTP(w, r)
}))
diff --git a/internal/edge/mesh.go b/internal/edge/mesh.go
index 25d14fe24..14d13b48b 100644
--- a/internal/edge/mesh.go
+++ b/internal/edge/mesh.go
@@ -80,11 +80,11 @@ import (
"fmt"
"log"
"net"
- "os"
"strconv"
"strings"
"time"
+ "github.com/carlosframework/platform/internal/edgeproof"
"github.com/carlosframework/platform/internal/release"
)
@@ -92,13 +92,13 @@ import (
// deployment. Named beside the storedoor ACL tripwire
// (TestDoorNeverServesCertMaterial), which refuses to serve it — a fleet
// box authenticating through the store door must never read this key.
-const meshKeyKey = "control/edge-mesh-key"
+const meshKeyKey = edgeproof.MeshKeyObject
// meshKeyNextKey is the rotation successor, mirrored alongside meshKeyKey
// only while a rotation is in flight. Promoting it (copy over meshKeyKey,
// delete this object) is a runbook step, not code this task ships — see the
// file header.
-const meshKeyNextKey = meshKeyKey + ".next"
+const meshKeyNextKey = edgeproof.NextMeshKeyObject
// meshKeyBytes is the random key length before hex encoding: 256 bits,
// matched to the HMAC-SHA256 hop signature below.
@@ -147,51 +147,21 @@ func MintMeshKey(ctx context.Context, s release.Store) error {
return mintMeshKey(ctx, s)
}
-// getMeshKeyHex reads and hex-decodes one mirrored mesh-key object, folding
-// "does not exist" (os.ErrNotExist) into (nil, nil) — the ordinary case for
-// meshKeyNextKey outside a rotation window, and for meshKeyKey itself before
-// the housekeeper's first tick on a brand new deployment.
-func getMeshKeyHex(ctx context.Context, s release.Store, key string) ([]byte, error) {
- body, _, err := s.Get(ctx, key)
- if errors.Is(err, os.ErrNotExist) {
- return nil, nil
- }
- if err != nil {
- return nil, fmt.Errorf("mesh key: get %s: %w", key, err)
- }
- dec, err := hex.DecodeString(string(body))
- if err != nil {
- return nil, fmt.Errorf("mesh key: %s: not valid hex: %w", key, err)
- }
- return dec, nil
-}
-
// loadMeshKeys is the dual-accept read side's key material access: it reads
// BOTH control/edge-mesh-key and control/edge-mesh-key.next. current is nil
// only before the housekeeper has minted anything; next is nil the whole
// rest of the time — nearly always, since a rotation is a deliberate,
// short-lived operator action.
//
-// A decode failure on meshKeyNextKey does NOT propagate: .next is written
-// by hand during a rotation (there is no code in this task that writes it),
-// so one bad upload is a real possibility, and the safe response to "I
-// can't read the rotation successor" is to act as though no rotation is in
-// flight — not to take every hop fleet-wide down by refusing the perfectly
-// good current key alongside it. The failure is loud (an operator mistake
-// worth paging on), just not fatal to this call. A decode failure on
-// meshKeyKey itself DOES propagate: that is the live, day-to-day signing
-// and verifying key, and there is no safe fallback for it being unreadable.
+// The read itself lives in internal/edgeproof, because the console reads the
+// same pair to check the edge's forwarded-for proof and must not import this
+// package. Its failure rules are the ones this read always had: a bad .next
+// is logged and treated as no rotation in flight (it is written by hand, and
+// one bad upload must not take every hop down with the good current key); a
+// bad current key fails the call, since there is no safe fallback for the
+// live key being unreadable.
func loadMeshKeys(ctx context.Context, s release.Store) (current, next []byte, err error) {
- current, err = getMeshKeyHex(ctx, s, meshKeyKey)
- if err != nil {
- return nil, nil, err
- }
- next, nextErr := getMeshKeyHex(ctx, s, meshKeyNextKey)
- if nextErr != nil {
- log.Printf("edge: mesh key: %s is unreadable, treating as no rotation in flight until fixed: %v", meshKeyNextKey, nextErr)
- next = nil
- }
- return current, next, nil
+ return edgeproof.LoadKeys(ctx, s)
}
// signingKey is which of the two mirrored keys a signer should use: .next
diff --git a/internal/edgeproof/edgeproof.go b/internal/edgeproof/edgeproof.go
new file mode 100644
index 000000000..b122c478d
--- /dev/null
+++ b/internal/edgeproof/edgeproof.go
@@ -0,0 +1,160 @@
+// Package edgeproof lets an app on the edge's own box tell a request the edge
+// forwarded from one that only looks like it.
+//
+// Every request the edge proxies arrives from a loopback or box-local peer,
+// so the peer address says nothing about who the client was; the edge's
+// X-Forwarded-For does. But any process on the box can open the same
+// loopback connection and write any X-Forwarded-For it likes. The console
+// counts its per-IP login limits on that header, so a tenant process on the
+// console's box could present a fresh "client" per request and walk past
+// every one of them (discussion 214, item 5).
+//
+// The edge therefore signs the X-Forwarded-For chain it sends, with the
+// deployment's mesh key, under Header. A verifier that holds the same key
+// believes the chain only when the signature matches it. The signature binds
+// the request's Host, the exact chain and a timestamp. Tenant apps see these
+// headers on their own traffic, so the host is what stops one replaying a
+// visitor's proof to the console on the same box: a proof minted for the
+// tenant's hostname does not verify at the console's. Within one host, a
+// lifted header vouches for that request's client and nothing else, for
+// ±Skew.
+//
+// The proof is between an edge and the upstream it dials, never between
+// edges: a backhauled request leaves without one, and the home edge signs
+// what it sends on (internal/edge's backhaulProxy says why).
+//
+// The MAC is domain-separated from the hop MAC (internal/edge's hopMAC):
+// both use the mesh key, and without the label a forwarded-proof value could
+// be replayed as a hop's X-Carlos-Hop-Auth or the other way round.
+//
+// The key objects are named here, not in internal/edge, because the console
+// reads them too and must not import the edge.
+package edgeproof
+
+import (
+ "context"
+ "crypto/hmac"
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "log"
+ "net"
+ "os"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// Header carries the edge's signature over the request's X-Forwarded-For.
+const Header = "X-Carlos-Forwarded-Proof"
+
+// MeshKeyObject is the control-plane object the housekeeper mints once per
+// deployment (internal/edge's mintMeshKey). NextMeshKeyObject is its
+// rotation successor, present only while a rotation is in flight.
+const (
+ MeshKeyObject = "control/edge-mesh-key"
+ NextMeshKeyObject = MeshKeyObject + ".next"
+)
+
+// Skew is how far a signature's timestamp may drift from the verifier's
+// clock. The signer and the verifier this exists for share one box and one
+// clock, so this is mostly request latency.
+const Skew = 30 * time.Second
+
+// label separates this MAC from every other use of the mesh key.
+const label = "carlos-forwarded-for\n"
+
+// Sign returns the Header value for a request to host carrying xff, its
+// full X-Forwarded-For chain as one comma-joined string.
+func Sign(key []byte, host, xff string, now time.Time) string {
+ ts := strconv.FormatInt(now.Unix(), 10)
+ return ts + ":" + hex.EncodeToString(mac(key, host, xff, ts))
+}
+
+// Verify reports whether header is a signature over host and xff under
+// current or next, made within Skew of now. Either key is accepted so a
+// rotation needs no cutover. The comparison is constant-time.
+func Verify(current, next []byte, host, xff, header string, now time.Time) bool {
+ ts, sig, ok := strings.Cut(header, ":")
+ if !ok || ts == "" || sig == "" {
+ return false
+ }
+ sec, err := strconv.ParseInt(ts, 10, 64)
+ if err != nil {
+ return false
+ }
+ if d := now.Sub(time.Unix(sec, 0)); d > Skew || d < -Skew {
+ return false
+ }
+ got, err := hex.DecodeString(sig)
+ if err != nil {
+ return false
+ }
+ for _, k := range [][]byte{next, current} {
+ if len(k) > 0 && hmac.Equal(got, mac(k, host, xff, ts)) {
+ return true
+ }
+ }
+ return false
+}
+
+func mac(key []byte, host, xff, ts string) []byte {
+ m := hmac.New(sha256.New, key)
+ m.Write([]byte(label))
+ m.Write([]byte(canonicalHost(host)))
+ m.Write([]byte("\n"))
+ m.Write([]byte(xff))
+ m.Write([]byte("\n"))
+ m.Write([]byte(ts))
+ return m.Sum(nil)
+}
+
+// canonicalHost is the host as both ends must agree on it: port stripped,
+// lowercased, one trailing root dot dropped — the same rules as the hop
+// MAC's canonicalizeHopHost.
+func canonicalHost(host string) string {
+ if h, _, err := net.SplitHostPort(host); err == nil {
+ host = h
+ }
+ return strings.TrimSuffix(strings.ToLower(host), ".")
+}
+
+// LoadKeys reads both mesh-key objects. current is nil only before the
+// housekeeper has minted one; next is nil outside a rotation.
+//
+// A bad .next does not fail the call: it is written by hand during a
+// rotation, and refusing the good current key alongside a bad upload would
+// take every hop down. It is logged loudly instead. A bad current key DOES
+// fail it: there is no safe fallback for the live key being unreadable.
+func LoadKeys(ctx context.Context, s release.Store) (current, next []byte, err error) {
+ current, err = getKeyHex(ctx, s, MeshKeyObject)
+ if err != nil {
+ return nil, nil, err
+ }
+ next, nextErr := getKeyHex(ctx, s, NextMeshKeyObject)
+ if nextErr != nil {
+ log.Printf("mesh key: %s is unreadable, treating as no rotation in flight until fixed: %v", NextMeshKeyObject, nextErr)
+ next = nil
+ }
+ return current, next, nil
+}
+
+// getKeyHex reads and hex-decodes one mesh-key object, folding "does not
+// exist" into (nil, nil).
+func getKeyHex(ctx context.Context, s release.Store, key string) ([]byte, error) {
+ body, _, err := s.Get(ctx, key)
+ if errors.Is(err, os.ErrNotExist) {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, fmt.Errorf("mesh key: get %s: %w", key, err)
+ }
+ dec, err := hex.DecodeString(string(body))
+ if err != nil {
+ return nil, fmt.Errorf("mesh key: %s: not valid hex: %w", key, err)
+ }
+ return dec, nil
+}
diff --git a/internal/provision/provision.go b/internal/provision/provision.go
index 64fd039a4..2616cdfda 100644
--- a/internal/provision/provision.go
+++ b/internal/provision/provision.go
@@ -616,9 +616,9 @@ func validAddr(a string) error {
// This rail is DELIBERATELY minimal (ruling 2026-08-12): loopback and
// RFC1918 stay allowed, because that is where every legitimate addr-backed
// route on the fleet points today (127.0.0.1:8090 consoles, private-subnet
- // tenants). The wider question — whether a MEMBER should be able to state
- // an upstream at all without a manifest opt-in, and what tenant isolation
- // means for one — is escalated separately; do not widen this check into an
+ // tenants). The wider question — what a MEMBER may state — was ruled on
+ // in discussion 50 (GitHub #170) and is answered at the console's doors
+ // (refuseMemberUpstreams), not here; do not widen this check into an
// answer for it.
if ip := net.ParseIP(h); ip != nil && ip.To4() != nil && ip.IsLinkLocalUnicast() {
return fmt.Errorf("addr %q: 169.254.0.0/16 is link-local (the cloud metadata service lives there) — an upstream must be a real backend address", a)
@@ -648,7 +648,8 @@ func validAddr(a string) error {
// Everything else validAddr allows stays allowed here — loopback and RFC1918
// are where every legitimate addr-backed route on the fleet points today
// (127.0.0.1:8090 consoles, private-subnet tenants). This is the metadata
-// rail closing its own holes, not a widening of it.
+// rail closing its own holes, not a widening of it. Whether a MEMBER may
+// state an upstream at all is the console's call: refuseMemberUpstreams.
func validAddrsElement(a string) error {
if err := validAddr(a); err != nil {
return err