| 1 | package edge |
| 2 | |
| 3 | // The home map, and the RECEIVING half of edge-to-edge backhaul (spec |
| 4 | // docs/superpowers/specs/2026-08-20-multi-region-ingress-design.md, Part 3, |
| 5 | // Task 3). |
| 6 | // |
| 7 | // Two things live here because they are read on the same poll and used in the |
| 8 | // same breath: |
| 9 | // |
| 10 | // - homeMap mirrors control/edges/ — the per-box files internal/host's |
| 11 | // edgesFilePass publishes (its edgesFile shape, parsed here as |
| 12 | // edgesFileView) — and folds them into one deployment-wide host→home |
| 13 | // answer, Home(host). It is the same 30s list-then-get, carry-forward, |
| 14 | // last-good mirror certsync.go's certMirror is; that shape survived heavy |
| 15 | // review and is copied rather than reinvented. It NEVER writes: the |
| 16 | // writer map gives control/edges/<box>.json to that box's own loop, and |
| 17 | // nothing in this package bumps control/serial. |
| 18 | // |
| 19 | // - the mesh keys ride the SAME refresh (loadMeshKeys, mesh.go), so a |
| 20 | // request path never reads the store: who homes a host and what key |
| 21 | // proves a hop are answered from one poll's view of the bucket rather |
| 22 | // than two. |
| 23 | // |
| 24 | // acceptHop is where a hop lands on the request path. The trust reasoning it |
| 25 | // rests on — why the MAC binds the concrete host, why ±hopSkew is the replay |
| 26 | // bound, and why dual-accept rotation means the request path must call |
| 27 | // verifyHopEitherKey and never the single-key verifyHop — is written out in |
| 28 | // mesh.go's file header; read that, this code does not re-derive it. Two |
| 29 | // consequences of it are worth naming HERE because they are this file's |
| 30 | // decisions rather than mesh.go's: |
| 31 | // |
| 32 | // - The replay envelope is deliberately loose and that is affordable. A |
| 33 | // captured X-Carlos-Hop-Auth is good for at most 2×hopSkew against ONE |
| 34 | // host, and what it buys a replayer is a request to a route the edge |
| 35 | // already serves to the whole internet — no new reachability, no |
| 36 | // privilege — PLUS, for that one host and that one window, XFF belief: |
| 37 | // verification gates exactly one thing, whether the inbound |
| 38 | // X-Forwarded-For is believed, and a valid MAC is what grants that |
| 39 | // belief. What keeps the loose envelope affordable is not that XFF |
| 40 | // belief is withheld — it is that capturing a valid Hop-Auth in the |
| 41 | // first place requires breaking the hop's TLS or holding |
| 42 | // control/edge-mesh-key outright, either of which already hands an |
| 43 | // attacker strictly more than "who does this one host think the client |
| 44 | // was for 2×hopSkew seconds." |
| 45 | // |
| 46 | // - XFF is trusted as an inbound chain to CARRY, never one to extend. On a |
| 47 | // verified hop the chain the sending edge built is passed through |
| 48 | // verbatim (setForwarded, below), so the app sees exactly what a |
| 49 | // single-region app sees and the LAST element is still the client. The |
| 50 | // backhaul leg is deliberately invisible in that header: this |
| 51 | // deployment's own consumers take the last element as the caller |
| 52 | // (internal/console/authcli.go's clientIP, and every tenant framework's |
| 53 | // rightmost-public heuristic), so appending the sibling edge would make |
| 54 | // every backhauled request look like it came from infrastructure. On |
| 55 | // anything but a verified hop the chain is discarded outright: an |
| 56 | // unverified sender is an ordinary client, and an ordinary client has |
| 57 | // never been allowed to state its own address here. |
| 58 | // |
| 59 | // One hop, ever, is STRUCTURAL and not a policy check: a verified hop that |
| 60 | // names a host this box does not serve leaves through staleHopRefusal (503 + |
| 61 | // Retry-After: 2) from inside the lookupUpstream-miss branch, BEFORE the point |
| 62 | // where Task 4 adds the backhaul dial. There is no path from an accepted hop |
| 63 | // to Home() at all, so a re-forward cannot be introduced by adding a branch — |
| 64 | // only by deleting one. |
| 65 | // |
| 66 | // Single-box inertness: with one platform box the only edges file is this |
| 67 | // box's own, own-box exclusion drops it, and Home() therefore never offers a |
| 68 | // target. Nothing signs hops either (Task 4 is the only signer, and it needs a |
| 69 | // target), so acceptHop's verified branch is unreachable on a single-box |
| 70 | // deployment — this task ships the receiving half, dormant. |
| 71 | |
| 72 | import ( |
| 73 | "context" |
| 74 | "encoding/json" |
| 75 | "errors" |
| 76 | "fmt" |
| 77 | "log" |
| 78 | "net/http" |
| 79 | "net/http/httputil" |
| 80 | "os" |
| 81 | "slices" |
| 82 | "sort" |
| 83 | "strconv" |
| 84 | "strings" |
| 85 | "sync" |
| 86 | "time" |
| 87 | |
| 88 | "github.com/carlosframework/platform/internal/edgesfile" |
| 89 | "github.com/carlosframework/platform/internal/lbhealth" |
| 90 | "github.com/carlosframework/platform/internal/release" |
| 91 | ) |
| 92 | |
| 93 | // edgesPrefix is where every box publishes what it homes — one flat object |
| 94 | // per box, control/edges/<box>.json (internal/host's edgesFileKey). |
| 95 | const edgesPrefix = "control/edges/" |
| 96 | |
| 97 | // The hop's two headers, verbatim from the spec's Part 3. Task 2 left them |
| 98 | // prose-only; this file is their first consumer, and Task 4's signer imports |
| 99 | // these constants rather than spelling the strings a second time. |
| 100 | const ( |
| 101 | hopHeader = "X-Carlos-Hop" |
| 102 | hopAuthHeader = "X-Carlos-Hop-Auth" |
| 103 | ) |
| 104 | |
| 105 | // edgeRoleHome is the only role v1 writes, and the only one that routes. |
| 106 | // Unknown roles (the spec's read-local anticipation) are IGNORED rather than |
| 107 | // refused: a newer box publishing a role this build has never heard of must |
| 108 | // not take its whole file — or the poll — down with it. |
| 109 | const edgeRoleHome = "home" |
| 110 | |
| 111 | // edgeRoleWildcard is the cert-only half of a delegated wildcard parent |
| 112 | // (#305): the parent's PEM should be loaded everywhere, but this box does |
| 113 | // not serve the catch-all, so the entry is no dial target. |
| 114 | const edgeRoleWildcard = "wildcard" |
| 115 | |
| 116 | // homeMapInterval / homeMapTimeout / homeMapBootTimeout mirror the cert |
| 117 | // mirror's cadence and budgets exactly (certsync.go), for the same reasons: a |
| 118 | // home move is a placement event that converge heals, the poll runs on its own |
| 119 | // goroutine and blocks nothing, and the ONE boot refresh must delay the front |
| 120 | // door by seconds rather than minutes. Vars so tests can shorten them. |
| 121 | var ( |
| 122 | homeMapInterval = 30 * time.Second |
| 123 | homeMapTimeout = 2 * time.Minute |
| 124 | homeMapBootTimeout = 20 * time.Second |
| 125 | ) |
| 126 | |
| 127 | // edgesFileView is the READER's half of internal/host's edgesFile wire shape |
| 128 | // (edgesfile.go). Deliberately a separate declaration rather than a shared |
| 129 | // type: internal/host imports this package and never the reverse, and the |
| 130 | // fields this side needs are the stable three (box, eip, hosts) plus the |
| 131 | // version. Additive growth on the writer's side lands here as fields this |
| 132 | // decoder ignores, which is the point. |
| 133 | type edgesFileView struct { |
| 134 | V int `json:"v"` |
| 135 | Box string `json:"box"` |
| 136 | EIP string `json:"eip"` |
| 137 | Hosts map[string]edgesHostView `json:"hosts"` |
| 138 | // PrivateIP, PrivateCIDR, Region and Placement choose the dial address |
| 139 | // (region HA spec 2 §6, dialAddress). All are absent from a writer that |
| 140 | // predates them, and the dial is then EIP, as before. |
| 141 | PrivateIP string `json:"private_ip"` |
| 142 | PrivateCIDR string `json:"private_cidr"` |
| 143 | Region string `json:"region"` |
| 144 | Placement string `json:"placement"` |
| 145 | } |
| 146 | |
| 147 | // dialAddress is where this edge dials f's box (region HA spec 2 §6). With |
| 148 | // privateDial (this edge's own region HA switch) on, a box that publishes |
| 149 | // the same AWS region and placement as this one is dialled at its private |
| 150 | // address, so the dial stays inside the region's network, but only when |
| 151 | // that address passes edgesfile.PrivateAddress against the network this |
| 152 | // box's own file publishes (census ruling region-ha-2a). The home's |
| 153 | // placement code is "", so two home boxes share a placement as two boxes |
| 154 | // of one named region do; a home box and a named one never do (Astra 2a |
| 155 | // final review, P2). Every other box, and every box with privateDial off, |
| 156 | // is dialled at its public address, as before. self is this box's own |
| 157 | // file, which selfKnown says was found. "" is nothing to dial. |
| 158 | func dialAddress(f, self edgesFileView, selfKnown, privateDial bool) string { |
| 159 | if privateDial && selfKnown && f.Region != "" && |
| 160 | f.Region == self.Region && f.Placement == self.Placement { |
| 161 | if addr, _ := edgesfile.PrivateAddress(f.PrivateIP, []string{self.PrivateCIDR}); addr != "" { |
| 162 | return addr |
| 163 | } |
| 164 | } |
| 165 | return f.EIP |
| 166 | } |
| 167 | |
| 168 | // regionHAFromEnv is CARLOS_REGION_HA as strconv.ParseBool reads it, the |
| 169 | // reading every region HA switch makes. |
| 170 | func regionHAFromEnv(getenv func(string) string) bool { |
| 171 | on, err := strconv.ParseBool(strings.TrimSpace(getenv("CARLOS_REGION_HA"))) |
| 172 | return err == nil && on |
| 173 | } |
| 174 | |
| 175 | // edgesHostView is the per-host object. An object rather than a bare string is |
| 176 | // what makes the role field addable without a wire break — see edgeRoleHome. |
| 177 | type edgesHostView struct { |
| 178 | Role string `json:"role"` |
| 179 | // Platform is the publisher's class mark (internal/host edgesfile.go): |
| 180 | // a platform-minted row. Absent on files written before the mark |
| 181 | // existed, which reads as "not platform" — fail closed for the one |
| 182 | // consumer that cares (Hosts, the wildcard parent set). |
| 183 | Platform bool `json:"platform,omitempty"` |
| 184 | // Alias marks an attached custom domain (#287). On a name collision |
| 185 | // across boxes a route entry beats an alias entry (foldHomes). |
| 186 | Alias bool `json:"alias,omitempty"` |
| 187 | // Wildcard marks a delegated customer wildcard parent's catch-all entry |
| 188 | // (#305): WildcardParents feeds it to the wildcard manager. |
| 189 | Wildcard bool `json:"wildcard,omitempty"` |
| 190 | // Account and App are the one app that owns every route the publishing |
| 191 | // box serves under this name (#175 P7). The publisher leaves both out |
| 192 | // when the name is split between apps; so does a publisher that predates |
| 193 | // them. The edge-local door refuses a remote host with no owner. |
| 194 | Account string `json:"account,omitempty"` |
| 195 | App string `json:"app,omitempty"` |
| 196 | // Stamp orders two boxes' claims on one host: see foldHomes. |
| 197 | Stamp int64 `json:"stamp,omitempty"` |
| 198 | } |
| 199 | |
| 200 | // homeEntry is one folded answer: where to dial, and which box that is. |
| 201 | type homeEntry struct { |
| 202 | // eip is the address to dial: dialAddress's answer, the home box's |
| 203 | // private address within its region and its EIP from anywhere else. |
| 204 | eip string |
| 205 | box string |
| 206 | // platform carries the publisher's class mark through the fold. |
| 207 | platform bool |
| 208 | // alias marks an attached custom domain's entry (never platform). |
| 209 | alias bool |
| 210 | // wildcard marks a delegated wildcard parent's catch-all entry. |
| 211 | wildcard bool |
| 212 | // dial reports whether the entry is a backhaul target (role home). A |
| 213 | // cert-only wildcard entry folds with dial false and Home skips it. |
| 214 | dial bool |
| 215 | // account and app are the owner the dial target's box published. They |
| 216 | // travel with eip and box and are never merged across entries. |
| 217 | account, app string |
| 218 | // stamp is the dial winner's installation stamp, 0 when unstamped. |
| 219 | stamp int64 |
| 220 | } |
| 221 | |
| 222 | // homeSnapshot is one poll's result, swapped in whole and immutable once |
| 223 | // published, so a reader holds no lock past the pointer read. |
| 224 | type homeSnapshot struct { |
| 225 | // files is the parsed per-box file, keyed by its BUCKET key. This is the |
| 226 | // carry-forward unit: an object that could not be read (or could not be |
| 227 | // parsed) this poll keeps its previous parse, per box, so one bad file |
| 228 | // never costs the deployment its whole map. |
| 229 | files map[string]edgesFileView |
| 230 | // hosts is the fold: host → home, own box already excluded. |
| 231 | hosts map[string]homeEntry |
| 232 | } |
| 233 | |
| 234 | var emptyHomeSnapshot = &homeSnapshot{files: map[string]edgesFileView{}, hosts: map[string]homeEntry{}} |
| 235 | |
| 236 | type homeMap struct { |
| 237 | s release.Store |
| 238 | // self is THIS box's label — the same derivation the box's own edges file |
| 239 | // is published under (internal/host's metricsBoxLabel), handed down at |
| 240 | // edge construction through WithBoxLabel. "" means this process does not |
| 241 | // know which box it is, and Home then offers nothing at all: see its doc. |
| 242 | self string |
| 243 | |
| 244 | // refreshMu serializes Refresh's whole body, for the reason certMirror's |
| 245 | // does — two overlapping polls could otherwise publish out of order, the |
| 246 | // slower one clobbering the fresher. |
| 247 | refreshMu sync.Mutex |
| 248 | |
| 249 | mu sync.RWMutex |
| 250 | snap *homeSnapshot |
| 251 | // hostsWake fires after a Refresh that changed the NAME set (setHostsWake). |
| 252 | hostsWake func() |
| 253 | // partialPolls counts polls that could not read (or parse) every object, |
| 254 | // for the first-and-every-tenth log cadence. |
| 255 | partialPolls int |
| 256 | // staleHopN counts staleHopRefusal occurrences per "box host" key (box is |
| 257 | // always m.self — despite the name this is a PER-HOST counter in |
| 258 | // practice, not cross-sender isolation; see staleHopRefusal's doc for |
| 259 | // why box is still named in the key and the log line), for the SAME |
| 260 | // first-and-every-tenth cadence partialPolls drives. Guarded by mu, |
| 261 | // following logPartialPoll's own choice to reuse the one lock rather |
| 262 | // than mint a second. |
| 263 | // |
| 264 | // Capped at staleHopKeyCap distinct keys — host here is CLIENT- |
| 265 | // CONTROLLED (Home()'s one-label wildcard match means a single |
| 266 | // catch-all edges-file entry lets a sending box mint a signed hop for |
| 267 | // any invented subdomain, edge.go's backhaul doc), so an unbounded map |
| 268 | // would be unbounded memory for the life of the process. Review catch |
| 269 | // on this same commit: the identical reasoning already gated the |
| 270 | // backhaul-error logThrottled key (edge.go) and is logThrottled's own |
| 271 | // documented rule (never key from remote input) — this map just has no |
| 272 | // logThrottled to inherit the cap from, so it gets its own. |
| 273 | staleHopN map[string]int |
| 274 | |
| 275 | // The mirrored mesh keys, read on the same refresh. Replaced only on a |
| 276 | // SUCCESSFUL load: a bucket blip leaves the last good key verifying hops, |
| 277 | // exactly as a blip leaves the last good host map routing them. |
| 278 | keyMu sync.RWMutex |
| 279 | keyCurrent []byte |
| 280 | keyNext []byte |
| 281 | |
| 282 | // privateDial is this edge's region HA switch: only with it on does |
| 283 | // the fold dial a same-region box's private address (dialAddress). |
| 284 | privateDial bool |
| 285 | |
| 286 | // health is the load balancer health state (region HA spec 2 §6), |
| 287 | // nil without region HA: Refresh latches the first loaded map and key |
| 288 | // and stamps every attempt it completes. |
| 289 | health *lbhealth.State |
| 290 | |
| 291 | // watch and now are certMirror's (#336): the process's change-index |
| 292 | // watch, resolved on every poll (nil, or a nil watch, is today's map), |
| 293 | // and the poll's clock (nil is time.Now). Only the hosts half is gated |
| 294 | // on them; the mesh keys keep their own read on every poll. |
| 295 | watch func() *release.ChangeWatch |
| 296 | now func() time.Time |
| 297 | // watchTimeout bounds the poll of the watch (0 is |
| 298 | // release.ChangeIndexPollTimeout; tests shorten it). |
| 299 | watchTimeout time.Duration |
| 300 | |
| 301 | // Guarded by refreshMu: the last hosts rebuild that read every edges |
| 302 | // file (lastHosts), the jittered backstop after which the next poll |
| 303 | // reads them all again (hostsAge), and the control/edges generation, |
| 304 | // captured BEFORE that read, which the fold is known to reflect |
| 305 | // (actedOK false = none: the watch could not vouch, or a file went |
| 306 | // unread or unparsed and must be retried). |
| 307 | lastHosts time.Time |
| 308 | hostsAge time.Duration |
| 309 | actedGen uint64 |
| 310 | actedOK bool |
| 311 | } |
| 312 | |
| 313 | func newHomeMap(s release.Store, self string) *homeMap { |
| 314 | return &homeMap{s: s, self: self, snap: emptyHomeSnapshot} |
| 315 | } |
| 316 | |
| 317 | // newHomeMap is the Edge's map, or nil when there is nothing to mirror FROM — |
| 318 | // the same no-deployment-store case that leaves the cert mirror nil. A box |
| 319 | // with no deployment bucket is not part of any mesh: it offers no backhaul |
| 320 | // target AND verifies no inbound hop (no key to verify against), which is the |
| 321 | // fail-closed answer either way. |
| 322 | func (e *Edge) newHomeMap() *homeMap { |
| 323 | if e.staticStore == nil { |
| 324 | return nil |
| 325 | } |
| 326 | m := newHomeMap(e.staticStore, e.box) |
| 327 | m.privateDial = regionHAFromEnv(os.Getenv) |
| 328 | m.health = e.lbHealth |
| 329 | m.watch = e.changeWatch |
| 330 | return m |
| 331 | } |
| 332 | |
| 333 | // Refresh rebuilds the host fold and re-reads the mesh keys. The two halves |
| 334 | // fail INDEPENDENTLY — a bucket that answered the edges list but not the key |
| 335 | // object should still land the fold — so both run and their errors join. |
| 336 | // Whatever fails leaves its own last-good in place; the caller logs. |
| 337 | // |
| 338 | // For the load balancer's health answer (region HA spec 2 §6) each half |
| 339 | // latches its first success, and every call that returns stamps an |
| 340 | // attempt, failed or not: a bucket outage after arming leaves the box |
| 341 | // healthy, and only a refresh stuck inside one attempt goes stale. |
| 342 | func (m *homeMap) Refresh(ctx context.Context) error { |
| 343 | m.refreshMu.Lock() |
| 344 | defer m.refreshMu.Unlock() |
| 345 | return m.refresh(ctx, true) |
| 346 | } |
| 347 | |
| 348 | // poll is the 30s loop's refresh (#336). Disarmed, it is Refresh. Armed, it |
| 349 | // polls the change index and skips the hosts half when control/edges has not |
| 350 | // moved since the generation the fold reflects and the last full hosts read |
| 351 | // is younger than the jittered backstop; otherwise (moved, the watch unable |
| 352 | // to vouch, no read yet, or at the backstop) the hosts half runs as today. |
| 353 | // The mesh keys are read on every poll whatever the index says: their |
| 354 | // objects (control/edge-mesh-key and .next) bump no scope, so a rotation |
| 355 | // must not wait on control/edges (codex 8). Poll's moved list is advisory |
| 356 | // (the watch is shared), so the decision compares generations instead. |
| 357 | func (m *homeMap) poll(ctx context.Context) error { |
| 358 | w := m.changeWatch() |
| 359 | if w == nil { |
| 360 | return m.Refresh(ctx) |
| 361 | } |
| 362 | // Bounded on its own (pollWatch): a hung index GET elsewhere in the |
| 363 | // process must not stop this poll from stamping RefreshAttempted. Any |
| 364 | // error, giving up on the watch's lock included, reads the hosts. |
| 365 | _, perr := pollWatch(ctx, w, m.watchTimeout) |
| 366 | m.refreshMu.Lock() |
| 367 | defer m.refreshMu.Unlock() |
| 368 | gen, ok := w.Gen(edgesScope) |
| 369 | current := perr == nil && ok && m.actedOK && gen == m.actedGen && |
| 370 | !m.lastHosts.IsZero() && m.clock().Before(m.lastHosts.Add(m.hostsAge)) |
| 371 | return m.refresh(ctx, !current) |
| 372 | } |
| 373 | |
| 374 | // refresh is Refresh's body, under refreshMu. readHosts false skips the |
| 375 | // hosts half, which then counts as a success: the fold it keeps is one a |
| 376 | // clean read landed (poll skips only after one, and that read latched |
| 377 | // MapLoaded), so the latch below is a no-op and the health answer is what a |
| 378 | // full read would have given. The attempt is stamped either way, so the |
| 379 | // five-minute refresh heartbeat holds across skipped polls. |
| 380 | func (m *homeMap) refresh(ctx context.Context, readHosts bool) error { |
| 381 | var hostsErr error |
| 382 | if readHosts { |
| 383 | hostsErr = m.refreshHosts(ctx) |
| 384 | } |
| 385 | keysErr := m.refreshKeys(ctx) |
| 386 | if hostsErr == nil { |
| 387 | m.health.MapLoaded() |
| 388 | } |
| 389 | if current, _ := m.meshKeys(); keysErr == nil && len(current) > 0 { |
| 390 | m.health.KeyLoaded() |
| 391 | } |
| 392 | m.health.RefreshAttempted(m.clock()) |
| 393 | return errors.Join(hostsErr, keysErr) |
| 394 | } |
| 395 | |
| 396 | // edgesScope is the change-index scope every write under edgesPrefix bumps. |
| 397 | const edgesScope = "control/edges" |
| 398 | |
| 399 | func (m *homeMap) changeWatch() *release.ChangeWatch { |
| 400 | if m.watch == nil { |
| 401 | return nil |
| 402 | } |
| 403 | return m.watch() |
| 404 | } |
| 405 | |
| 406 | func (m *homeMap) clock() time.Time { |
| 407 | if m.now != nil { |
| 408 | return m.now() |
| 409 | } |
| 410 | return time.Now() |
| 411 | } |
| 412 | |
| 413 | // refreshHosts rebuilds the whole map from one list plus a get per key — no |
| 414 | // delta cleverness, for certMirror.Refresh's reason: a full rebuild is what |
| 415 | // makes a RETIRED box's hosts actually leave the map. |
| 416 | // |
| 417 | // An error leaves the last-good snapshot in place. That matters as much here |
| 418 | // as it does for certs: an empty map on a bucket blip would stop every |
| 419 | // backhaul in the deployment at once, turning a blip into a regional outage |
| 420 | // for every host homed elsewhere. |
| 421 | func (m *homeMap) refreshHosts(ctx context.Context) error { |
| 422 | // The generation this read will reflect, captured BEFORE the list, so a |
| 423 | // write landing during the read leaves control/edges moved past it (#336). |
| 424 | var gen uint64 |
| 425 | var genOK bool |
| 426 | if w := m.changeWatch(); w != nil { |
| 427 | gen, genOK = w.Gen(edgesScope) |
| 428 | } |
| 429 | keys, err := m.s.List(ctx, edgesPrefix) |
| 430 | if err != nil { |
| 431 | return fmt.Errorf("list %s: %w", edgesPrefix, err) |
| 432 | } |
| 433 | prev := m.snapshot() |
| 434 | next := make(map[string]edgesFileView, len(keys)) |
| 435 | // failed counts both fault classes; unread counts only the objects the |
| 436 | // BUCKET would not hand over, which is the one that says "this is not an |
| 437 | // object fault at all" when it covers everything (see below). |
| 438 | var tried, failed, unread int |
| 439 | var lastErr error |
| 440 | for _, key := range keys { |
| 441 | // The prefix is flat and every member is <box>.json. Anything else |
| 442 | // under it was not written by this deployment's edges loop. |
| 443 | name, ok := strings.CutPrefix(key, edgesPrefix) |
| 444 | if !ok || !strings.HasSuffix(name, ".json") || strings.Contains(name, "/") { |
| 445 | continue |
| 446 | } |
| 447 | tried++ |
| 448 | body, _, err := m.s.Get(ctx, key) |
| 449 | if errors.Is(err, os.ErrNotExist) { |
| 450 | // Raced a delete between the list and the get — the box was |
| 451 | // retired mid-poll. Not a fault; it is simply gone. |
| 452 | continue |
| 453 | } |
| 454 | // ONE bad object is not a bad poll, whichever way it went bad. |
| 455 | // Aborting would discard every other box's file that landed this tick, |
| 456 | // and a permanently unreadable (or permanently corrupt) key would |
| 457 | // freeze the map forever. Carry this box's previous parse forward and |
| 458 | // keep going. |
| 459 | if err != nil { |
| 460 | unread++ |
| 461 | } else if f, perr := parseEdgesFile(key, body); perr != nil { |
| 462 | err = perr |
| 463 | } else { |
| 464 | next[key] = f |
| 465 | continue |
| 466 | } |
| 467 | failed++ |
| 468 | lastErr = err |
| 469 | if old, ok := prev.files[key]; ok { |
| 470 | next[key] = old |
| 471 | } |
| 472 | } |
| 473 | if tried > 0 && unread == tried { |
| 474 | // Nothing at all could be READ: a bucket fault, not an object fault, |
| 475 | // reported like a failed List — last-good stands. Note the bound is |
| 476 | // unread and not failed: a deployment whose only edges file is a |
| 477 | // corrupt one has a bucket that answered perfectly well, and calling |
| 478 | // that a bucket fault would report a blip that is not happening. |
| 479 | return fmt.Errorf("get %s*: every one of %d object(s) was unreadable, last: %w", edgesPrefix, tried, lastErr) |
| 480 | } |
| 481 | if failed > 0 { |
| 482 | m.logPartialPoll(failed, tried, lastErr) |
| 483 | } |
| 484 | hosts, tied := foldHomes(next, m.self, m.privateDial) |
| 485 | if len(tied) > 0 { |
| 486 | // One claim revision names one executor, so two boxes publishing |
| 487 | // one host at the same stamp is a fault upstream. The fold stays |
| 488 | // deterministic; say so once a refresh. |
| 489 | log.Printf("edge: home map: %d host(s) published by two boxes at the same installation stamp, e.g. %s", len(tied), tied[0]) |
| 490 | } |
| 491 | snap := &homeSnapshot{files: next, hosts: hosts} |
| 492 | m.mu.Lock() |
| 493 | m.snap = snap |
| 494 | wake := m.hostsWake |
| 495 | m.mu.Unlock() |
| 496 | // A partial read reflects no generation: the file it could not read or |
| 497 | // parse is retried on the next poll, as it always was. |
| 498 | m.lastHosts, m.hostsAge = m.clock(), jitteredBackstop() |
| 499 | m.actedGen, m.actedOK = gen, genOK && failed == 0 |
| 500 | if wake != nil && !samePlatformSet(prev.hosts, snap.hosts) { |
| 501 | // The deployment's platform name set moved: a wildcard parent may |
| 502 | // have appeared for an account homed elsewhere (#300). Off this |
| 503 | // goroutine — the wake re-derives through Hosts() (its own locks) |
| 504 | // and may touch the registry, and refreshKeys should not wait on |
| 505 | // that. |
| 506 | go wake() |
| 507 | } |
| 508 | return nil |
| 509 | } |
| 510 | |
| 511 | // samePlatformSet reports whether two folds name the same PLATFORM hosts — |
| 512 | // the only names that can move a wildcard parent (homes may differ; an |
| 513 | // unclassed name may come and go). |
| 514 | func samePlatformSet(a, b map[string]homeEntry) bool { |
| 515 | na, nb := 0, 0 |
| 516 | for h, e := range a { |
| 517 | if !parentInput(h, e) && !wildcardInput(h, e) { |
| 518 | continue |
| 519 | } |
| 520 | na++ |
| 521 | if eb, ok := b[h]; !ok || (parentInput(h, e) != parentInput(h, eb)) || (wildcardInput(h, e) != wildcardInput(h, eb)) { |
| 522 | return false |
| 523 | } |
| 524 | } |
| 525 | for h, e := range b { |
| 526 | if parentInput(h, e) || wildcardInput(h, e) { |
| 527 | nb++ |
| 528 | } |
| 529 | } |
| 530 | return na == nb |
| 531 | } |
| 532 | |
| 533 | // parentInput is the filter Hosts and samePlatformSet share for ACCOUNT |
| 534 | // parents: a marked, exact (non catch-all) name. wildcardInput is its twin |
| 535 | // for delegated parents (WildcardParents). Anything else coming or going |
| 536 | // cannot move a parent, so it must not wake the manager either. |
| 537 | func parentInput(host string, e homeEntry) bool { |
| 538 | return e.platform && !strings.HasPrefix(host, "*.") |
| 539 | } |
| 540 | |
| 541 | func wildcardInput(host string, e homeEntry) bool { |
| 542 | if !e.wildcard || !strings.HasPrefix(host, "*.") { |
| 543 | return false |
| 544 | } |
| 545 | parent := host[2:] |
| 546 | // The shape a local wildcard_domains row passed (validHostShape at |
| 547 | // reconcile), applied to a remote one: a real dotted name, no further |
| 548 | // wildcard label, nothing a challenge FQDN could not be built from. |
| 549 | return parent != "" && strings.Contains(parent, ".") && !strings.ContainsAny(parent, "*/ \t") |
| 550 | } |
| 551 | |
| 552 | // WildcardParents is every delegated customer wildcard parent another box |
| 553 | // publishes ("*.<parent>" entries marked wildcard, #305), as bare parents. |
| 554 | // The wildcard manager unions these with its own wildcard_domains rows so |
| 555 | // a region edge loads the parent's mirrored PEM and terminates the |
| 556 | // one-label hosts Home already dials for it. Sorted, nil-safe, nothing for |
| 557 | // an unlabelled box (as Hosts). |
| 558 | func (m *homeMap) WildcardParents() []string { |
| 559 | if m == nil || m.self == "" { |
| 560 | return nil |
| 561 | } |
| 562 | snap := m.snapshot() |
| 563 | out := make([]string, 0) |
| 564 | for h, e := range snap.hosts { |
| 565 | if wildcardInput(h, e) { |
| 566 | out = append(out, strings.TrimPrefix(h, "*.")) |
| 567 | } |
| 568 | } |
| 569 | sort.Strings(out) |
| 570 | return out |
| 571 | } |
| 572 | |
| 573 | // Hosts is every exact PLATFORM-class name the deployment's OTHER boxes |
| 574 | // home. Sorted, nil-safe. The wildcard manager unions these with this box's |
| 575 | // own routes so its parent set is the deployment's, not the box's (#300): |
| 576 | // a region edge that adopts only everywhere-class routes would otherwise |
| 577 | // never derive the account parent of an exec app homed elsewhere, and so |
| 578 | // never load the wildcard PEM the mirror had already written to its disk. |
| 579 | // |
| 580 | // Two bounds, both fail-closed: the publisher's class mark (a row that is |
| 581 | // neither Platform nor Provisioned — an operator's stray `carlos add` |
| 582 | // under the apps domain — derives nothing here, exactly as currentHosts |
| 583 | // refuses it locally; a file from before the mark existed contributes |
| 584 | // nothing until its box republishes), and parentsFor's two-labels-under- |
| 585 | // the-apps-domain rule on top. Catch-all "*." keys are excluded: a |
| 586 | // remote catch-all is a claim's row, never an account parent. An |
| 587 | // unlabelled box (self == "") reports nothing, as Home does — its own |
| 588 | // file would otherwise fold in as "another box". |
| 589 | func (m *homeMap) Hosts() []string { |
| 590 | if m == nil || m.self == "" { |
| 591 | return nil |
| 592 | } |
| 593 | snap := m.snapshot() |
| 594 | out := make([]string, 0, len(snap.hosts)) |
| 595 | for h, e := range snap.hosts { |
| 596 | if !parentInput(h, e) { |
| 597 | continue |
| 598 | } |
| 599 | out = append(out, h) |
| 600 | } |
| 601 | sort.Strings(out) |
| 602 | return out |
| 603 | } |
| 604 | |
| 605 | // ServedHosts lists every EXACT hostname any box in this deployment homes — |
| 606 | // routes and attached custom domains alike, with no class filter. |
| 607 | // |
| 608 | // Deliberately NOT Hosts(): that one applies parentInput, which keeps only |
| 609 | // platform-class rows because its consumer derives wildcard PARENTS from |
| 610 | // them. The cert warm's question is the opposite one — "what exact names does |
| 611 | // this deployment answer for?" — and a custom domain is precisely the |
| 612 | // non-platform row Hosts() drops. Filtering by class here would warm every |
| 613 | // canary and no customer domain, which is backwards. |
| 614 | // |
| 615 | // Catch-all "*." keys are excluded: they are covered by a wildcard PEM, and |
| 616 | // autocert refuses the wildcard label anyway (Allowed). Only dial-able |
| 617 | // entries count — a cert-only wildcard row homes nothing. |
| 618 | // |
| 619 | // An unlabelled box (self == "") reports nothing, as Home does: an edge that |
| 620 | // cannot tell its own rows from another box's must not answer at all. |
| 621 | func (m *homeMap) ServedHosts() []string { |
| 622 | if m == nil || m.self == "" { |
| 623 | return nil |
| 624 | } |
| 625 | snap := m.snapshot() |
| 626 | out := make([]string, 0, len(snap.hosts)) |
| 627 | for h, e := range snap.hosts { |
| 628 | if !e.dial || strings.HasPrefix(h, "*.") { |
| 629 | continue |
| 630 | } |
| 631 | out = append(out, h) |
| 632 | } |
| 633 | sort.Strings(out) |
| 634 | return out |
| 635 | } |
| 636 | |
| 637 | // setHostsWake installs the callback a changed host set fires (Refresh). |
| 638 | // Under the map's own lock: run() is already polling when the wildcard |
| 639 | // manager is wired in after it. |
| 640 | func (m *homeMap) setHostsWake(fn func()) { |
| 641 | if m == nil { |
| 642 | return |
| 643 | } |
| 644 | m.mu.Lock() |
| 645 | m.hostsWake = fn |
| 646 | m.mu.Unlock() |
| 647 | } |
| 648 | |
| 649 | // parseEdgesFile decodes one box's file. A parse failure gets the same |
| 650 | // treatment an unreadable object gets — carry the last good parse forward — |
| 651 | // because a half-written body is no more usable than a body that never |
| 652 | // arrived, and the fix for both is the publisher's next pass. |
| 653 | func parseEdgesFile(key string, body []byte) (edgesFileView, error) { |
| 654 | var f edgesFileView |
| 655 | if err := json.Unmarshal(body, &f); err != nil { |
| 656 | return edgesFileView{}, fmt.Errorf("%s: %w", key, err) |
| 657 | } |
| 658 | if f.V < 1 { |
| 659 | // No version at all: not a file this reader wrote a decoder for. A |
| 660 | // HIGHER version is accepted on purpose — the schema grows additively |
| 661 | // (the plan's global constraint), so a newer box's file still carries |
| 662 | // the box/eip/hosts this reader needs, and refusing it would black out |
| 663 | // backhaul to every box that upgraded first. |
| 664 | return edgesFileView{}, fmt.Errorf("%s: unversioned edges file", key) |
| 665 | } |
| 666 | return f, nil |
| 667 | } |
| 668 | |
| 669 | // logPartialPoll reports unreadable or unparseable files on the loop's own |
| 670 | // first-and-every-tenth cadence: a permanently bad object would otherwise say |
| 671 | // the same thing every 30 seconds forever. |
| 672 | func (m *homeMap) logPartialPoll(failed, tried int, err error) { |
| 673 | m.mu.Lock() |
| 674 | m.partialPolls++ |
| 675 | n := m.partialPolls |
| 676 | m.mu.Unlock() |
| 677 | if n == 1 || n%10 == 0 { |
| 678 | log.Printf("edge: %d of %d object(s) under %s unreadable this poll (#%d, last: %v) — the previous copy of each is carried forward", failed, tried, edgesPrefix, n, err) |
| 679 | } |
| 680 | } |
| 681 | |
| 682 | // foldHomes turns the per-box files into one host→home map. |
| 683 | // |
| 684 | // Three exclusions, each of which is a request this box must never make: |
| 685 | // |
| 686 | // - OWN BOX. A host homed here is SERVED here; offering it as a backhaul |
| 687 | // target would have the edge dial itself. Excluded by the file's own box |
| 688 | // field rather than by its object name, because the field is what the |
| 689 | // publisher stamps and what self is derived to match. |
| 690 | // - NO DIAL ADDRESS. Task 1 publishes a box with no discoverable public IP |
| 691 | // as eip:"" so the DR runbook still gets its hosts list. There is |
| 692 | // nothing to dial, so it is not a target. The address is dialAddress's: |
| 693 | // a box in this box's own region is dialled at its private address when |
| 694 | // it publishes one (region HA spec 2 §6), every other at its EIP. |
| 695 | // - A ROLE THIS BUILD DOES NOT KNOW. Only "home" routes. The role field |
| 696 | // exists so read-local roles can be added later; a reader that treated |
| 697 | // every role as a home would send traffic to a box that had declared it |
| 698 | // serves only reads. |
| 699 | // |
| 700 | // Iteration is over SORTED file keys so a duplicate claim — two boxes both |
| 701 | // saying they home one host, which is a placement bug mid-move — resolves the |
| 702 | // same way on every edge in the deployment. Convergent-but-wrong beats |
| 703 | // split-and-wrong: the whole fleet aiming at one box is a condition that heals |
| 704 | // on the next publish, where half the fleet aiming at each is a condition that |
| 705 | // looks fine from any single box. |
| 706 | func foldHomes(files map[string]edgesFileView, self string, privateDial bool) (map[string]homeEntry, []string) { |
| 707 | keys := make([]string, 0, len(files)) |
| 708 | for k := range files { |
| 709 | keys = append(keys, k) |
| 710 | } |
| 711 | slices.Sort(keys) |
| 712 | var own edgesFileView |
| 713 | ownKnown := false |
| 714 | for _, k := range keys { |
| 715 | if f := files[k]; self != "" && f.Box == self { |
| 716 | own, ownKnown = f, true |
| 717 | } |
| 718 | } |
| 719 | hosts := make(map[string]homeEntry) |
| 720 | for _, k := range keys { |
| 721 | f := files[k] |
| 722 | if f.Box == "" || f.Box == self { |
| 723 | continue |
| 724 | } |
| 725 | addr := dialAddress(f, own, ownKnown, privateDial) |
| 726 | for host, h := range f.Hosts { |
| 727 | if host == "" { |
| 728 | continue |
| 729 | } |
| 730 | switch h.Role { |
| 731 | case edgeRoleHome: |
| 732 | // A dial target needs an address. A file with none (a box |
| 733 | // with no discoverable public IP) still carries its cert |
| 734 | // halves below; its dial halves are simply not dialable. |
| 735 | if addr == "" { |
| 736 | if !h.Wildcard || !strings.HasPrefix(host, "*.") { |
| 737 | continue |
| 738 | } |
| 739 | h.Role = edgeRoleWildcard |
| 740 | } |
| 741 | case edgeRoleWildcard: |
| 742 | // The cert-only role is defined for marked catch-all keys |
| 743 | // alone; on an exact key it would be a KnownHost admission |
| 744 | // with no dial behind it — refused. |
| 745 | if !h.Wildcard || !strings.HasPrefix(host, "*.") { |
| 746 | continue |
| 747 | } |
| 748 | default: |
| 749 | continue |
| 750 | } |
| 751 | in := homeEntry{eip: addr, box: f.Box, platform: h.Platform && !h.Alias && !h.Wildcard, alias: h.Alias, wildcard: h.Wildcard, dial: h.Role == edgeRoleHome, |
| 752 | account: h.Account, app: h.App, stamp: h.Stamp} |
| 753 | prev, dup := hosts[host] |
| 754 | if !dup { |
| 755 | hosts[host] = in |
| 756 | continue |
| 757 | } |
| 758 | // A collision merges CLASSES and picks ONE dial target: the |
| 759 | // classes are facts about the name (a delegated parent's PEM |
| 760 | // must load everywhere even if some other box also routes the |
| 761 | // key), the dial target is a choice. First box in sorted order |
| 762 | // wins the dial — except that a ROUTE beats an ALIAS whatever |
| 763 | // the order (#287), and any dial target beats none. The owner |
| 764 | // is part of the dial choice, not a class: it comes whole from |
| 765 | // the winner, so a route that replaces an alias never inherits |
| 766 | // the alias's owner, even when it publishes none of its own. |
| 767 | merged := prev |
| 768 | merged.platform = prev.platform || in.platform |
| 769 | merged.wildcard = prev.wildcard || in.wildcard |
| 770 | if dialBeats(in, prev) { |
| 771 | merged.eip, merged.box, merged.alias, merged.dial = in.eip, in.box, in.alias, true |
| 772 | merged.account, merged.app, merged.stamp = in.account, in.app, in.stamp |
| 773 | } |
| 774 | hosts[host] = merged |
| 775 | } |
| 776 | } |
| 777 | return hosts, tiedStamps(files) |
| 778 | } |
| 779 | |
| 780 | // tiedStamps lists, sorted, the hosts that two boxes publish as home at |
| 781 | // the same positive stamp. It looks at every file, this box's own |
| 782 | // included, and at every pair, not only at whichever entry won the dial: |
| 783 | // in a two-box region each box must be able to see that it and its |
| 784 | // neighbour disagree, and a third box's higher stamp must not hide a tie |
| 785 | // between two others. |
| 786 | // |
| 787 | // Unstamped duplicates are not reported. They are normal: a static site |
| 788 | // or a public address route runs on every box, and every box publishes it. |
| 789 | func tiedStamps(files map[string]edgesFileView) []string { |
| 790 | type key struct { |
| 791 | host string |
| 792 | stamp int64 |
| 793 | } |
| 794 | boxes := map[key]map[string]bool{} |
| 795 | for _, f := range files { |
| 796 | if f.Box == "" { |
| 797 | continue |
| 798 | } |
| 799 | for host, h := range f.Hosts { |
| 800 | if host == "" || h.Role != edgeRoleHome || h.Stamp <= 0 { |
| 801 | continue |
| 802 | } |
| 803 | k := key{host, h.Stamp} |
| 804 | if boxes[k] == nil { |
| 805 | boxes[k] = map[string]bool{} |
| 806 | } |
| 807 | boxes[k][f.Box] = true |
| 808 | } |
| 809 | } |
| 810 | seen := map[string]bool{} |
| 811 | var tied []string |
| 812 | for k, set := range boxes { |
| 813 | if len(set) > 1 && !seen[k.host] { |
| 814 | seen[k.host] = true |
| 815 | tied = append(tied, k.host) |
| 816 | } |
| 817 | } |
| 818 | slices.Sort(tied) |
| 819 | return tied |
| 820 | } |
| 821 | |
| 822 | // dialBeats reports whether a later file's entry takes the dial from the |
| 823 | // one already folded. In order: |
| 824 | // |
| 825 | // - only a dial target can take it, and any dial target beats none; |
| 826 | // - a ROUTE beats an ALIAS whatever else is true (#287); |
| 827 | // - between two of the same kind, the newer installation wins: a stamped |
| 828 | // entry beats an unstamped one, and a higher stamp beats a lower one |
| 829 | // (region HA spec 1 HA4). A dead box's file lingers; its successor's |
| 830 | // stamp is always higher, so the stale one loses without any liveness |
| 831 | // rule; |
| 832 | // - otherwise the earlier file keeps it, which is the first box in |
| 833 | // sorted order, as before. |
| 834 | func dialBeats(in, prev homeEntry) bool { |
| 835 | if !in.dial { |
| 836 | return false |
| 837 | } |
| 838 | if !prev.dial { |
| 839 | return true |
| 840 | } |
| 841 | if prev.alias != in.alias { |
| 842 | return prev.alias && !in.alias |
| 843 | } |
| 844 | return in.stamp > prev.stamp |
| 845 | } |
| 846 | |
| 847 | func (m *homeMap) snapshot() *homeSnapshot { |
| 848 | m.mu.RLock() |
| 849 | defer m.mu.RUnlock() |
| 850 | return m.snap |
| 851 | } |
| 852 | |
| 853 | // Home answers "which OTHER box homes this host, and where do I dial it?". |
| 854 | // |
| 855 | // The lookup is lookupUpstream's, rule for rule, and that is load-bearing |
| 856 | // rather than tidy: Task 1 publishes registry row hosts VERBATIM, catch-all |
| 857 | // "*.<parent>" rows included, so an exact match wins and a miss retries with |
| 858 | // the leading dot of the FIRST label swapped for "*" — one label, never two, |
| 859 | // and never the parent matching its own wildcard. Deliberately NOT |
| 860 | // case-folded, for the same reason: a host lookupUpstream would miss on case |
| 861 | // must miss here too, or this edge would backhaul a request the home edge |
| 862 | // would then refuse with a 503. |
| 863 | // |
| 864 | // A nil map (no deployment store) and a map with no self label both answer |
| 865 | // "nothing". The self case is fail-closed on purpose: without knowing which |
| 866 | // box it is, this edge cannot exclude its own hosts, and an edge that cannot |
| 867 | // exclude itself must not be handing out dial targets at all. |
| 868 | func (m *homeMap) Home(host string) (eip, box string, ok bool) { |
| 869 | h, ok := m.lookupHome(host) |
| 870 | if !ok { |
| 871 | return "", "", false |
| 872 | } |
| 873 | return h.eip, h.box, true |
| 874 | } |
| 875 | |
| 876 | // ExactHome is Home without the one-label wildcard fallback: the dial target |
| 877 | // another box published for exactly host. The edge asks it when its own |
| 878 | // answer for host came only through a local "*.<parent>" key, so that an |
| 879 | // exact name served elsewhere still wins (ServeHTTP). Fail-closed without a |
| 880 | // self label, exactly as Home is. |
| 881 | func (m *homeMap) ExactHome(host string) (eip, box string, ok bool) { |
| 882 | if m == nil || m.self == "" { |
| 883 | return "", "", false |
| 884 | } |
| 885 | if h, found := m.snapshot().hosts[host]; found && h.dial { |
| 886 | return h.eip, h.box, true |
| 887 | } |
| 888 | return "", "", false |
| 889 | } |
| 890 | |
| 891 | // lookupHome is Home's lookup, answering the whole winning entry from one |
| 892 | // snapshot read. Home and Owner both go through it, so they can never pick |
| 893 | // different entries for the same host. |
| 894 | func (m *homeMap) lookupHome(host string) (homeEntry, bool) { |
| 895 | if m == nil || m.self == "" { |
| 896 | return homeEntry{}, false |
| 897 | } |
| 898 | hosts := m.snapshot().hosts |
| 899 | if h, ok := hosts[host]; ok && h.dial { |
| 900 | return h, true |
| 901 | } |
| 902 | if i := strings.IndexByte(host, '.'); i > 0 { |
| 903 | if h, ok := hosts["*"+host[i:]]; ok && h.dial { |
| 904 | return h, true |
| 905 | } |
| 906 | } |
| 907 | return homeEntry{}, false |
| 908 | } |
| 909 | |
| 910 | // Owner answers, from one read of one snapshot, both whom the edge-local |
| 911 | // door must check and where it would dial (#175 P7), so the check and the |
| 912 | // dial cannot disagree. The lookup is Home's, one-label wildcard included, |
| 913 | // and eip and box are exactly what Home returns for the same host. |
| 914 | // |
| 915 | // ok is false, and everything else zero, when the winning entry is not a |
| 916 | // dial target or published no owner: a hostname split between apps, or a |
| 917 | // publisher that predates the field. An ownerless exact entry does not fall |
| 918 | // through to a wildcard that names one, because the exact entry is the one |
| 919 | // Home dials. A nil map, or one with no self label, answers nothing, as Home |
| 920 | // does. |
| 921 | func (m *homeMap) Owner(host string) (account, app, eip, box string, ok bool) { |
| 922 | h, found := m.lookupHome(host) |
| 923 | if !found || h.account == "" || h.app == "" { |
| 924 | return "", "", "", "", false |
| 925 | } |
| 926 | return h.account, h.app, h.eip, h.box, true |
| 927 | } |
| 928 | |
| 929 | // KnownHost reports whether host is an EXACT key in the mirrored home map — a |
| 930 | // name some OTHER box in this deployment homes, spelled precisely. |
| 931 | // |
| 932 | // Deliberately NOT Home()'s one-label wildcard fallback, and refusing "*." |
| 933 | // outright: this predicate widens cert issuance and challenge admission |
| 934 | // (certsync.go's three gates, through Edge.deploymentHost), where PR #180's |
| 935 | // rule stands — attacker-chosen labels under a catch-all must never drive an |
| 936 | // ACME order or a bucket read, and their TLS coverage is the wildcard cert's |
| 937 | // job (Allowed's own doc says the same thing for this box's own rows). The |
| 938 | // widening loses nothing by staying exact: an order only ever exists for a |
| 939 | // name that passed the exact gate, so a challenge only ever needs answering |
| 940 | // for an exact name. |
| 941 | // |
| 942 | // Unlike Home it needs no EIP and no dial target — membership is the whole |
| 943 | // question — but it reads the same fold, so it inherits both of the fold's |
| 944 | // exclusions. Own-box hosts are excluded and that is harmless: they pass |
| 945 | // Allowed on the other half of deploymentHost. Boxes with EIP=="" are |
| 946 | // excluded and that is accepted: a box with no public IP offers no backhaul |
| 947 | // target and appears in no DNS answer, so a validation for its hosts has |
| 948 | // nowhere legitimate to land anyway. |
| 949 | // |
| 950 | // There is deliberately no m.self gate, unlike Home: Home fail-closes without |
| 951 | // a self label because it could otherwise offer this box itself as a dial |
| 952 | // target, and membership carries no such hazard — an unlabelled box's fold |
| 953 | // keeps its own file, whose hosts are its own registry rows and so pass |
| 954 | // Allowed anyway, modulo edges-file staleness (a route deleted from the |
| 955 | // registry lingers in the published file until the next publish, minutes; |
| 956 | // inherent to the design, and the widened set is registry rows either way). |
| 957 | // |
| 958 | // Nil-receiver-safe, like Home: the map may not be armed yet (or at all, on a |
| 959 | // box with no deployment store). |
| 960 | func (m *homeMap) KnownHost(host string) bool { |
| 961 | if m == nil || strings.HasPrefix(host, "*.") { |
| 962 | return false |
| 963 | } |
| 964 | _, ok := m.snapshot().hosts[host] |
| 965 | return ok |
| 966 | } |
| 967 | |
| 968 | // refreshKeys re-reads the mesh key pair. loadMeshKeys is already tolerant of |
| 969 | // a malformed .next and fail-closed on the current key (mesh.go); this only |
| 970 | // adds the mirror's own posture on top — the pair is replaced ONLY on a clean |
| 971 | // read, so a bucket blip leaves the last good key verifying hops. |
| 972 | // |
| 973 | // A clean read that finds NOTHING does replace the pair with nil, and that is |
| 974 | // not a blip: an absent control/edge-mesh-key means no hop can be proven, and |
| 975 | // refusing every hop (they degrade to ordinary clients) is the only safe |
| 976 | // reading of "the deployment has no mesh key". |
| 977 | func (m *homeMap) refreshKeys(ctx context.Context) error { |
| 978 | current, next, err := loadMeshKeys(ctx, m.s) |
| 979 | if err != nil { |
| 980 | return err |
| 981 | } |
| 982 | m.keyMu.Lock() |
| 983 | m.keyCurrent, m.keyNext = current, next |
| 984 | m.keyMu.Unlock() |
| 985 | return nil |
| 986 | } |
| 987 | |
| 988 | // meshKeys is the request path's cached view of the key pair — never a store |
| 989 | // read per request. Both may be nil (nothing minted yet, or nothing mirrored |
| 990 | // yet), which verifyHopEitherKey answers as "no". |
| 991 | func (m *homeMap) meshKeys() (current, next []byte) { |
| 992 | if m == nil { |
| 993 | return nil, nil |
| 994 | } |
| 995 | m.keyMu.RLock() |
| 996 | defer m.keyMu.RUnlock() |
| 997 | return m.keyCurrent, m.keyNext |
| 998 | } |
| 999 | |
| 1000 | // run polls until stop closes — the edge's standing background-loop shape, |
| 1001 | // the same one certMirror.run uses: fixed ticker, errors counted and logged at |
| 1002 | // the first and then every tenth, recovery logged once. |
| 1003 | func (m *homeMap) run(stop <-chan struct{}) { |
| 1004 | log.Printf("edge: home map armed (%s poll of %s*, this box is %q) — hosts homed on another box are reachable through it", homeMapInterval, edgesPrefix, m.self) |
| 1005 | tick := time.NewTicker(homeMapInterval) |
| 1006 | defer tick.Stop() |
| 1007 | errs := 0 |
| 1008 | for { |
| 1009 | select { |
| 1010 | case <-stop: |
| 1011 | return |
| 1012 | case <-tick.C: |
| 1013 | } |
| 1014 | ctx, cancel := context.WithTimeout(context.Background(), homeMapTimeout) |
| 1015 | err := m.poll(ctx) |
| 1016 | cancel() |
| 1017 | if err != nil { |
| 1018 | errs++ |
| 1019 | if errs == 1 || errs%10 == 0 { |
| 1020 | log.Printf("edge: home map refresh failed (#%d): %v — serving the last good map and key", errs, err) |
| 1021 | } |
| 1022 | continue |
| 1023 | } |
| 1024 | if errs > 0 { |
| 1025 | log.Printf("edge: home map refresh recovered after %d failure(s)", errs) |
| 1026 | errs = 0 |
| 1027 | } |
| 1028 | } |
| 1029 | } |
| 1030 | |
| 1031 | // ---- hop acceptance on the request path ------------------------------------- |
| 1032 | |
| 1033 | // hopVerifiedKey marks a request that arrived as a PROVEN hop from another |
| 1034 | // edge in this deployment's mesh. A context value rather than a header, |
| 1035 | // because a header is something a client can send: nothing outside this file |
| 1036 | // can put this mark on a request, which is exactly the property the XFF trust |
| 1037 | // below rests on. |
| 1038 | type hopVerifiedKey struct{} |
| 1039 | |
| 1040 | // hopVerified reports whether this request arrived as a verified hop. Task 4's |
| 1041 | // forwarding decision reads it too — a request that ARRIVED as a hop never |
| 1042 | // consults the home map. |
| 1043 | func hopVerified(ctx context.Context) bool { |
| 1044 | v, _ := ctx.Value(hopVerifiedKey{}).(bool) |
| 1045 | return v |
| 1046 | } |
| 1047 | |
| 1048 | // acceptHop is the whole of hop handling on the request path, and it runs |
| 1049 | // before the route lookup so that BOTH of its outcomes are decided ahead of |
| 1050 | // anything that serves: |
| 1051 | // |
| 1052 | // no hop header → nothing happens at all (the overwhelming case: one |
| 1053 | // Header.Get on a request that is not a hop). |
| 1054 | // verified hop → the mark goes on the context, the inbound |
| 1055 | // X-Forwarded-For is believed, and the caller serves |
| 1056 | // the route locally or refuses (staleHopRefusal). |
| 1057 | // unverified hop → every forwarded header is stripped and the sender is |
| 1058 | // an ordinary client from here on. |
| 1059 | // |
| 1060 | // The hop headers themselves are deleted in ALL hop cases, verified included: |
| 1061 | // they are the edge's vocabulary between boxes and have no business reaching a |
| 1062 | // tenant's app, and a stripped header cannot be laundered back out by an app |
| 1063 | // that reflects what it was sent (scrubInboundPurgeHeader, purge_header.go, |
| 1064 | // records what that class of bug already cost this package). |
| 1065 | // |
| 1066 | // host is the caller's already-port-stripped request host; canonicalization |
| 1067 | // into the exact bytes the MAC covers is hopMAC's job, in one place, so a |
| 1068 | // signer and a verifier can never disagree (mesh.go). |
| 1069 | func (e *Edge) acceptHop(r *http.Request, host string) (*http.Request, bool) { |
| 1070 | if r.Header.Get(hopHeader) == "" { |
| 1071 | // A signature with no marker beside it is not a hop — but it is still |
| 1072 | // this edge's vocabulary, and the invariant above is "deleted in ALL |
| 1073 | // hop cases" without an exception for the half a client made up. |
| 1074 | // scrubInboundPurgeHeader deletes both purge headers unconditionally |
| 1075 | // for exactly this reason (purge_header.go): an app that reflects what |
| 1076 | // it was sent turns a client-supplied edge header into an |
| 1077 | // app-DECLARED one, and no tenant's reflection bug should be able to |
| 1078 | // put hop vocabulary back on the wire. The Get stays the fast path's |
| 1079 | // single lookup — the Del only happens on a request that actually |
| 1080 | // carries the stray header, which is never, in the ordinary case. |
| 1081 | if r.Header.Get(hopAuthHeader) != "" { |
| 1082 | r.Header.Del(hopAuthHeader) |
| 1083 | } |
| 1084 | return r, false |
| 1085 | } |
| 1086 | auth := r.Header.Get(hopAuthHeader) |
| 1087 | r.Header.Del(hopHeader) |
| 1088 | r.Header.Del(hopAuthHeader) |
| 1089 | |
| 1090 | // verifyHopEitherKey, never verifyHop: the single-key form cannot accept a |
| 1091 | // signer that has already moved to .next, and a rotation is precisely when |
| 1092 | // refusing hops would hurt. Keys come from the mirror's cache — a hop must |
| 1093 | // never cost a bucket read. |
| 1094 | current, next := e.homes.Load().meshKeys() |
| 1095 | if auth == "" || !verifyHopEitherKey(current, next, host, auth, time.Now()) { |
| 1096 | stripForwardedHeaders(r) |
| 1097 | return r, false |
| 1098 | } |
| 1099 | return r.WithContext(context.WithValue(r.Context(), hopVerifiedKey{}, true)), true |
| 1100 | } |
| 1101 | |
| 1102 | // stripForwardedHeaders discards every claim a request makes about who sent |
| 1103 | // it. httputil.ReverseProxy's Rewrite path already deletes these from the |
| 1104 | // OUTBOUND request, so for a proxied route this is belt and braces — but the |
| 1105 | // edge serves plenty that is not a ReverseProxy (static trees, the status |
| 1106 | // host, cached replays, the activator's dispatch), and all of them read this |
| 1107 | // same inbound request. |
| 1108 | func stripForwardedHeaders(r *http.Request) { |
| 1109 | r.Header.Del("Forwarded") |
| 1110 | r.Header.Del("X-Forwarded-For") |
| 1111 | r.Header.Del("X-Forwarded-Host") |
| 1112 | r.Header.Del("X-Forwarded-Proto") |
| 1113 | } |
| 1114 | |
| 1115 | // staleHopRefusal answers a VERIFIED hop that named a host this box does not |
| 1116 | // serve: the sending edge's home map is stale, mid-move. 503 + Retry-After: 2 |
| 1117 | // is the same "come back shortly" answer a dark fleet gets (fleetProxy). |
| 1118 | // Retry-After: 2 is a keep-trying hint, not a heal estimate: there is no 2s |
| 1119 | // serial-converge path here (edges never bump control/serial), so what |
| 1120 | // actually heals a stale map is the edges-file cycle: the home boxes |
| 1121 | // republish within a couple of seconds of their registry changing |
| 1122 | // (internal/host's edgesFileLoop, since #174; before that only on a |
| 1123 | // 5min+jitter timer), and the sending box then picks that up on its 30s |
| 1124 | // mirror poll. So a sender can retry into this refusal for up to about half |
| 1125 | // a minute during a home move. |
| 1126 | // |
| 1127 | // The body says nothing. It is reachable only by a party holding the mesh key |
| 1128 | // — an unauthenticated stranger asking for the same host gets the ordinary, |
| 1129 | // unchanged unknownHost answer — so it cannot tell an outsider that a host |
| 1130 | // exists somewhere else in the deployment. Keeping it contentless holds that |
| 1131 | // line for the ONE case where it could still leak: a peer whose key is valid |
| 1132 | // but whose map is wrong learns only "not here", never "here is where it |
| 1133 | // went". |
| 1134 | // |
| 1135 | // Silent until now was leaving a home move that gets stuck mid-way with |
| 1136 | // nothing on this box's OWN journal to show it — a sender can retry into |
| 1137 | // this refusal for a while (see above) and the only visibility was on the |
| 1138 | // SENDING side. This logs first-and-every-tenth per (box, host), the same |
| 1139 | // cadence logPartialPoll (above) uses for its own standing condition; box is |
| 1140 | // always m.self (there is exactly one homeMap per process, and it never |
| 1141 | // learns the sending edge's identity — the hop MAC covers only host and |
| 1142 | // time, mesh.go), named in the key and the line anyway so a centrally |
| 1143 | // aggregated journal reads as "box X refused host Y" rather than an |
| 1144 | // anonymous host with no owner. |
| 1145 | // |
| 1146 | // A nil receiver (no deployment store — the no-mesh-at-all case) still |
| 1147 | // answers the refusal; it just cannot count or log one, the same "answer |
| 1148 | // first, degrade the bookkeeping" shape Home and meshKeys already have. |
| 1149 | func (m *homeMap) staleHopRefusal(w http.ResponseWriter, host string) { |
| 1150 | w.Header().Set("Retry-After", "2") |
| 1151 | http.Error(w, "not served here", http.StatusServiceUnavailable) |
| 1152 | if m == nil { |
| 1153 | return |
| 1154 | } |
| 1155 | m.logStaleHop(host) |
| 1156 | } |
| 1157 | |
| 1158 | // staleHopKeyCap bounds staleHopN's key set (review catch, 2026-08-23): |
| 1159 | // host is client-controlled (staleHopN's doc, above — a wildcard-homed |
| 1160 | // route lets a sending box mint a signed hop for any invented subdomain), |
| 1161 | // so an uncapped map would be unbounded memory, and an unbounded run of |
| 1162 | // distinct first-time log lines, for the life of the process. Once the cap |
| 1163 | // is reached, every NEW key folds into one "<box> (other)" overflow bucket |
| 1164 | // instead of growing the map further; keys already tracked keep their own |
| 1165 | // counts and cadence. |
| 1166 | const staleHopKeyCap = 1024 |
| 1167 | |
| 1168 | // logStaleHop is staleHopRefusal's counting half, split out so the HTTP |
| 1169 | // response is written before anything touches mu. |
| 1170 | func (m *homeMap) logStaleHop(host string) { |
| 1171 | key := m.self + " " + host |
| 1172 | m.mu.Lock() |
| 1173 | if m.staleHopN == nil { |
| 1174 | m.staleHopN = make(map[string]int) |
| 1175 | } |
| 1176 | if _, tracked := m.staleHopN[key]; !tracked && len(m.staleHopN) >= staleHopKeyCap { |
| 1177 | key = m.self + " (other)" |
| 1178 | } |
| 1179 | m.staleHopN[key]++ |
| 1180 | n := m.staleHopN[key] |
| 1181 | m.mu.Unlock() |
| 1182 | if n == 1 || n%10 == 0 { |
| 1183 | log.Printf("edge: stale hop refusal on box %s for %s (#%d) — the sending edge's home map has not caught up yet", m.self, host, n) |
| 1184 | } |
| 1185 | } |
| 1186 | |
| 1187 | // setForwarded is what every proxy Rewrite in this package calls in place of |
| 1188 | // pr.SetXForwarded(), and it exists for one narrow case. |
| 1189 | // |
| 1190 | // ReverseProxy's Rewrite path deletes Forwarded and X-Forwarded-* from the |
| 1191 | // outbound request before Rewrite runs, so SetXForwarded always writes a |
| 1192 | // single address: the peer that dialled us. That is right for a client and |
| 1193 | // wrong for a verified hop, where the peer that dialled us is a sibling edge |
| 1194 | // and the address the app wants is the one that edge already recorded. |
| 1195 | // |
| 1196 | // So on a verified hop — and only there — SetXForwarded's answer is REPLACED |
| 1197 | // by the inbound chain, verbatim. Not appended to: the last element has to |
| 1198 | // stay the client. This deployment's own consumer says so in as many words |
| 1199 | // (internal/console/authcli.go's clientIP: "the LAST element is the one to |
| 1200 | // take"), and tenant frameworks' rightmost-public heuristics agree — a chain |
| 1201 | // of "<client>, <sending edge EIP>" would rate-limit every backhauled console |
| 1202 | // request as one bucket per region and show tenant apps infrastructure in |
| 1203 | // place of their caller. Passing the chain through unchanged means a |
| 1204 | // backhauled request is indistinguishable from a local one at the app, which |
| 1205 | // is the whole promise of the hop. Multiple inbound X-Forwarded-For lines are |
| 1206 | // joined, which is what list-valued header semantics already say they mean |
| 1207 | // (RFC 9110 §5.3). |
| 1208 | // |
| 1209 | // SetXForwarded still runs first, and is still what sets X-Forwarded-Host and |
| 1210 | // X-Forwarded-Proto: the hop carries the original Host header and is itself |
| 1211 | // TLS, so both are correct as computed here and need no restore. It touches |
| 1212 | // only pr.Out, so reading the inbound chain after it is the same read as |
| 1213 | // before it. |
| 1214 | func setForwarded(pr *httputil.ProxyRequest) { |
| 1215 | pr.SetXForwarded() |
| 1216 | // Last, whichever branch below returns: the proof signs the chain as it |
| 1217 | // finally leaves (forwardproof.go). |
| 1218 | defer signForwarded(pr) |
| 1219 | // A door request is a request from this box (door.go): its RemoteAddr is |
| 1220 | // already the box's public address, so SetXForwarded wrote the right |
| 1221 | // chain. Only the scheme differs from the public route to the same URL, |
| 1222 | // which would have arrived over TLS. |
| 1223 | if _, ok := doorCallerFrom(pr.In.Context()); ok { |
| 1224 | pr.Out.Header.Set("X-Forwarded-Proto", "https") |
| 1225 | return |
| 1226 | } |
| 1227 | if !hopVerified(pr.In.Context()) { |
| 1228 | return |
| 1229 | } |
| 1230 | if chain := pr.In.Header.Values("X-Forwarded-For"); len(chain) > 0 { |
| 1231 | pr.Out.Header.Set("X-Forwarded-For", strings.Join(chain, ", ")) |
| 1232 | } |
| 1233 | } |
| 1234 | |
amadan