| 1 | // Package edge is the deployment host's front door: TLS termination |
| 2 | // (autocert), a host→backend reverse proxy, and a live-reloading route table. |
| 3 | // |
| 4 | // Ported from Eleven's router.go (elevenmessenger/messenger: router.go; |
| 5 | // design in its docs/instance-router.md) — the identity-free shape: the |
| 6 | // atomically-swapped host→upstream map, the retrying unix dialer that masks |
| 7 | // instance restarts, autocert with the route table as HostPolicy, the |
| 8 | // registry fingerprint watcher, SIGHUP reload, the unknown-host fallback, and |
| 9 | // the hibernation-activator hook — with titogo internal/carlos/router.go's |
| 10 | // additions carried in: -dev (plain-HTTP mode, so multi-instance routing runs |
| 11 | // locally with no DNS/TLS) and the first-observed-fingerprint reload. Dropped |
| 12 | // on port: titogo's Stripe relay, auth/identity, root-domain platformIndex; |
| 13 | // Eleven's mesh (its wildcard DNS-01 certs were later ported back — |
| 14 | // wildcard.go). New here: TCP upstreams (Route.Addr) so whole-app tenants sit |
| 15 | // behind the edge without unix-socket support, and the static serving class |
| 16 | // stub (static.go). |
| 17 | package edge |
| 18 | |
| 19 | import ( |
| 20 | "github.com/carlosframework/platform/internal/cliflag" |
| 21 | |
| 22 | "bufio" |
| 23 | "context" |
| 24 | "crypto/ed25519" |
| 25 | "crypto/tls" |
| 26 | "crypto/x509" |
| 27 | "encoding/json" |
| 28 | "errors" |
| 29 | "flag" |
| 30 | "fmt" |
| 31 | "io" |
| 32 | "log" |
| 33 | "maps" |
| 34 | "net" |
| 35 | "net/http" |
| 36 | "net/http/httputil" |
| 37 | "net/textproto" |
| 38 | "net/url" |
| 39 | "os" |
| 40 | "os/signal" |
| 41 | "path" |
| 42 | "path/filepath" |
| 43 | "slices" |
| 44 | "strconv" |
| 45 | "strings" |
| 46 | "sync" |
| 47 | "sync/atomic" |
| 48 | "syscall" |
| 49 | "time" |
| 50 | |
| 51 | "golang.org/x/crypto/acme/autocert" |
| 52 | |
| 53 | "github.com/carlosframework/platform/internal/apperr" |
| 54 | "github.com/carlosframework/platform/internal/edgeproof" |
| 55 | "github.com/carlosframework/platform/internal/edgesfile" |
| 56 | "github.com/carlosframework/platform/internal/gate" |
| 57 | "github.com/carlosframework/platform/internal/geo" |
| 58 | "github.com/carlosframework/platform/internal/geoip" |
| 59 | "github.com/carlosframework/platform/internal/lbhealth" |
| 60 | "github.com/carlosframework/platform/internal/provision" |
| 61 | "github.com/carlosframework/platform/internal/registry" |
| 62 | "github.com/carlosframework/platform/internal/release" |
| 63 | ) |
| 64 | |
| 65 | // Default cache sizing for EnableCache: 4096 entries is comfortably above a |
| 66 | // single box's route × Vary-variant count, and 1 MiB caps a single cached |
| 67 | // body well above a typical HTML page while still bounding worst-case memory |
| 68 | // (4096 * 1MiB = 4GiB ceiling, never actually reached in practice since most |
| 69 | // bodies are far smaller). |
| 70 | const ( |
| 71 | DefaultCacheMaxEntries = 4096 |
| 72 | DefaultCacheMaxBodyBytes = 1 << 20 // 1 MiB |
| 73 | ) |
| 74 | |
| 75 | // DefaultCacheMaxTTL is the ceiling every cached entry's TTL is clamped to, |
| 76 | // however long the app's own s-maxage or max-age asks for. It is what makes spec decision |
| 77 | // 4 true — "a lost purge self-heals within a bounded window": without a |
| 78 | // ceiling, an app setting max-age=31536000 turns one dropped purge call into a |
| 79 | // permanently wrong page. Five minutes is short enough that a missed purge is |
| 80 | // a blip and long enough that a crawler sweep still rides the cache. |
| 81 | const DefaultCacheMaxTTL = 5 * time.Minute |
| 82 | |
| 83 | // Activator is the hibernation seam (implemented by the activator package): |
| 84 | // given a hibernate-flagged instance route, it wakes (or joins an in-progress |
| 85 | // wake of) the backing process before letting the request through, and |
| 86 | // reports whether it wrote the response. nil = no hibernation, pure proxy. |
| 87 | type Activator interface { |
| 88 | ServeHTTP(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool |
| 89 | RetireExcept(keep map[string]bool) |
| 90 | } |
| 91 | |
| 92 | // Edge terminates TLS (autocert) and reverse-proxies each configured host to |
| 93 | // its backend. The host→backend map is swapped atomically, so reloads and |
| 94 | // zero-downtime deploys never drop a live connection. |
| 95 | type Edge struct { |
| 96 | reg *registry.Registry |
| 97 | upstreams atomic.Pointer[map[string]upstream] |
| 98 | // act, when non-nil, is the hibernation activator: hibernate-flagged |
| 99 | // instance routes stop being always-on backends and become on-demand |
| 100 | // processes woken per request. Nil keeps the proxy-only behavior; with it |
| 101 | // set, unflagged routes are still proxied straight (the flag is per host). |
| 102 | act Activator |
| 103 | // gate is the password curtain (internal/gate) for routes whose row |
| 104 | // carries a gate hash. Always constructed: the check is one string |
| 105 | // compare on up.route.Gate for every ungated host, and the limiter it |
| 106 | // carries is shared across the box's gated hosts. |
| 107 | gate *gate.Gate |
| 108 | // fallbackURL, when set, is where requests for unknown hosts are redirected |
| 109 | // (e.g. the apex marketing site) instead of a bare 404 — a wildcard DNS |
| 110 | // record then makes typo'd or retired subdomains land somewhere useful. Note |
| 111 | // the limit: over HTTPS this only helps hosts that still HAVE a cert (e.g. a |
| 112 | // removed instance); a never-served host fails the TLS handshake first, |
| 113 | // because HostPolicy refuses to mint certs for unknown hosts (anything else |
| 114 | // would let strangers burn the ACME rate limit). The :80 path always works. |
| 115 | // |
| 116 | // Hosts under a platform domain are the exception (pending.go, #174): they |
| 117 | // get the 503 pending page, which reaches this URL only through its |
| 118 | // script, after the retry window. curl, crawlers and no-JS clients see the |
| 119 | // 503 and never the redirect. |
| 120 | fallbackURL string |
| 121 | // platformDomains and pendingBody are the "not here YET" answer |
| 122 | // (pending.go, #174): an unknown host strictly under one of these gets a |
| 123 | // 503 retry page instead of unknownHost's 404 or fallback redirect. Both |
| 124 | // are set together by setPlatformDomains, at start; nil domains (every |
| 125 | // test that builds an Edge with New, and a deployment with neither env |
| 126 | // set) keep the old answer for every host. |
| 127 | platformDomains []string |
| 128 | pendingBody []byte |
| 129 | // steeringHost is this deployment's DNS-steering hostname |
| 130 | // (edge.<apps-domain>), or "" when the deployment does not steer — which |
| 131 | // is every deployment today. Set once at construction from the |
| 132 | // environment; see steeringprobe.go for why it is a serving concern at |
| 133 | // all. |
| 134 | steeringHost string |
| 135 | // lbHealth is the load balancer health state (region HA spec 2 §6): |
| 136 | // lbhealth.Default with CARLOS_REGION_HA on, nil and inert with it |
| 137 | // off. Set once in New. |
| 138 | lbHealth *lbhealth.State |
| 139 | // probing is set while a serving-path probe (servingPathLive) is |
| 140 | // waiting on a lock, so a wedged lock costs one goroutine, not one a |
| 141 | // tick. |
| 142 | probing atomic.Bool |
| 143 | // appsDomain is the platform apps domain (PLATFORM_APPS_DOMAIN, |
| 144 | // normalized), read once in New; cookieLogged is the set of route hosts |
| 145 | // whose cookie rewrite has been logged. Both serve cookiescope.go. "" |
| 146 | // disables the rewrite, which is every test Edge and every deployment |
| 147 | // without an apps domain. |
| 148 | appsDomain string |
| 149 | cookieLogged sync.Map // route host → true |
| 150 | // Static serving config (EnableStatic): the deployment store the site |
| 151 | // tars are fetched from, the release pubkey channel pointers must verify |
| 152 | // against, and the on-box cache the trees are extracted into. staticStore |
| 153 | // nil = static serving not configured; static routes answer 404 (the |
| 154 | // route can still exist and mint its cert before the config lands). |
| 155 | staticStore release.Store |
| 156 | staticPub ed25519.PublicKey |
| 157 | staticCacheDir string |
| 158 | staticRecheck time.Duration |
| 159 | // staticRatchet is #253's ratchet for static pointer reads: marks under |
| 160 | // <dataDir>/pointer-digests, root's under /opt read too. |
| 161 | staticRatchet *release.DigestRatchet |
| 162 | // usageMeter, when non-nil, is where ServeHTTP records the request/byte |
| 163 | // count of every route it serves (see EnableUsage). Atomic because it is |
| 164 | // armed by the agent role after Edge construction, mirroring how |
| 165 | // tunnelLoop attaches via the ready hook — serving may already be |
| 166 | // underway. |
| 167 | usageMeter atomic.Pointer[UsageMeter] |
| 168 | // errSink, when set, receives edge-observed errors (an upstream 5xx) |
| 169 | // — spec 2026-09-07 §3.2. Nil disables emission; nothing else changes. |
| 170 | errSink atomic.Pointer[errorSinkRef] |
| 171 | // analyticsMeter, when set, receives one observation per completed |
| 172 | // response for an app whose analytics are switched on (analytics.go, |
| 173 | // spec 2026-09-07 §3.3), and analyticsOn is that switch map keyed |
| 174 | // sqid+"/"+app. Both are atomic for the same reason usageMeter is: the |
| 175 | // agent role arms the meter after construction, and the telemetry pass |
| 176 | // replaces the switch map while serving is underway. Nil meter or nil |
| 177 | // map = the edge observes nothing, which is every `carlos edge` box and |
| 178 | // every app nobody has switched on. |
| 179 | analyticsMeter atomic.Pointer[analyticsMeterRef] |
| 180 | analyticsOn atomic.Pointer[map[string]bool] |
| 181 | // geo is the GeoIP reader every request's lookup goes through (geoip |
| 182 | // spec §5.4); never nil after New, empty until a database is installed. |
| 183 | // geoLookup is what decideGeo calls: geo.Lookup in production, a spy in |
| 184 | // tests (loopback test traffic is never Locatable). geoMirror fills geo |
| 185 | // when this edge has a deployment store and is not a tunnel client; |
| 186 | // geoMode is what the heartbeat reports (mirror | tunnel | none). |
| 187 | geo *geo.Swappable |
| 188 | geoLookup func(net.IP) geo.Snapshot |
| 189 | geoMirror *geoMirror |
| 190 | geoMode string |
| 191 | geoTunnelClient bool |
| 192 | // tunnels, when non-nil, is the reverse-channel hub (tunnel.go): remote |
| 193 | // hosts register on its reserved hostname and routes with Tunnel set are |
| 194 | // proxied over the channel they name. nil = tunnel serving not configured |
| 195 | // on this box: a route's Tunnel binding is then ignored and it falls back |
| 196 | // to its socket/addr, or goes unserved if it has neither — the same |
| 197 | // nil-means-disabled convention as act/cache/staticStore. |
| 198 | tunnels *TunnelHub |
| 199 | // door, when non-nil, is the fleet store door (storedoor.go): the five |
| 200 | // release.Store operations served over HTTPS under /store/ on the SAME |
| 201 | // reserved hostname the hub answers on, so a fleet box needs no AWS |
| 202 | // credentials on the deployment bucket. It exists only when BOTH halves |
| 203 | // it needs are configured — the hub (its only credential source) and the |
| 204 | // deployment store (EnableStatic's handle, reused, never a second |
| 205 | // client) — so armStoreDoor is called from both EnableXxx, whichever |
| 206 | // lands second. nil = /store/ falls through to the hub's own 404, the |
| 207 | // same nil-means-disabled convention as act/cache/staticStore. |
| 208 | door *storeDoor |
| 209 | // fleetAccounts is the reload-time fleet→account-sqid snapshot the door's |
| 210 | // ACL is derived from, refreshed on the SAME pass that feeds the hub's |
| 211 | // dynamic auth table (Reload, below) so a box's credentials and its |
| 212 | // tenancy can never come from two different reads of the registry. |
| 213 | // Atomic: written by Reload while requests are being served. |
| 214 | fleetAccounts atomic.Pointer[map[string]string] |
| 215 | // cache, when non-nil, answers cacheable GET/HEAD requests without ever |
| 216 | // invoking the hibernation activator or the backend — the mitigation |
| 217 | // this whole feature exists for. nil disables caching entirely. |
| 218 | cache *Cache |
| 219 | // logN counts occurrences per throttle key so logThrottled can rate-limit |
| 220 | // a repeating complaint. Two callers today: Carlos-Purge complaints, which |
| 221 | // are driven by app responses on the request hot path and would otherwise |
| 222 | // be one journal line per request, and Reload's unparseable-health line, |
| 223 | // which repeats on every reload for as long as the bad row exists. Lazily |
| 224 | // created under logMu. |
| 225 | logMu sync.Mutex |
| 226 | logN map[string]int |
| 227 | // reloadHook, when set (OnReload), runs after every successful route |
| 228 | // swap — the wildcard manager recomputes its parent set from it. Atomic |
| 229 | // because registration happens in RunWithActivator after Watch may |
| 230 | // already be reloading. |
| 231 | reloadHook atomic.Pointer[func()] |
| 232 | // probers owns the health-probe goroutines for multi-upstream addr routes |
| 233 | // (prober.go) and the live set multiProxy picks from. Created in New, |
| 234 | // BEFORE the startup Reload — Reload calls Sync on it unconditionally and |
| 235 | // a nil *proberSet panics by design, so a construction path that skipped |
| 236 | // this would be a wiring bug that fails loudly rather than a fleet that |
| 237 | // quietly 503s. Never stopped in production (the edge lives as long as the |
| 238 | // process); tests Stop it so probe goroutines don't outlive them. |
| 239 | probers *proberSet |
| 240 | // gzipOn is the CARLOS_EDGE_GZIP kill switch, read once in New. When |
| 241 | // false, ServeHTTP never constructs a gzipWriter and the writer stack is |
| 242 | // byte-for-byte what it was before compression existed (compress.go). |
| 243 | gzipOn bool |
| 244 | // issuer, when set, gates certificate ISSUANCE on this box holding the |
| 245 | // deployment's housekeeper lease (certsync.go). It arrives through |
| 246 | // WithIssuerGate because internal/host imports this package and never the |
| 247 | // reverse. nil = ungated, which is `carlos edge` without the agent and |
| 248 | // every single-box deployment's behaviour to date: mint as always. |
| 249 | // Serving is never gated by it. |
| 250 | issuer IssuerGate |
| 251 | // contends reports whether this box takes part in the issuer election |
| 252 | // at all (WithIssuerContender) — the cert backfill's eligibility, a |
| 253 | // coarser question than the gate above answers. nil = does not contend. |
| 254 | contends func() bool |
| 255 | // box is this box's own label — the SAME derivation its edges file is |
| 256 | // published under (internal/host's metricsBoxLabel), handed down through |
| 257 | // WithBoxLabel for the reason the issuer gate is: that package imports |
| 258 | // this one and never the reverse. It is what the home map excludes its own |
| 259 | // hosts by; "" (bare `carlos edge`) leaves the map offering nothing. |
| 260 | box string |
| 261 | // changeWatch resolves the process's change-index watch (#336), handed |
| 262 | // down through WithChangeWatch. The cert mirror and the home map call it |
| 263 | // on every poll. nil, or a nil watch, keeps both on today's full reads. |
| 264 | changeWatch func() *release.ChangeWatch |
| 265 | // homes, when non-nil, is the mirror of control/edges/ plus the mesh keys |
| 266 | // (homemap.go): who homes a host this box does not serve, and what proves |
| 267 | // an inbound hop. Atomic because it is armed in RunWithActivatorReady after |
| 268 | // construction, and read on the request path while that is happening. |
| 269 | homes atomic.Pointer[homeMap] |
| 270 | // backhaulTransports caches the outbound TLS transport of each backhaul |
| 271 | // target, keyed by the home box's EIP — see backhaulTransport. Bounded by |
| 272 | // the number of BOXES in the deployment, never by hostnames: a catch-all |
| 273 | // row in another box's edges file makes every subdomain under it a |
| 274 | // backhaul target, and keying per host would let a stranger's stream of |
| 275 | // invented hostnames grow this map without limit. |
| 276 | backhaulTransports sync.Map |
| 277 | // backhaulInflight counts the hops currently in flight toward each home |
| 278 | // box — the containment half of the egress ruling (backhaulAcquire). |
| 279 | // Keyed by BOX for the same reason backhaulTransports is keyed by EIP: a |
| 280 | // catch-all row in another box's edges file makes every invented |
| 281 | // subdomain a backhaul target, so a per-host key would let a stranger |
| 282 | // grow this map without limit. Bounded by the deployment's box count. |
| 283 | backhaulInflight sync.Map // box → *atomic.Int64 |
| 284 | // The edge-local door (door.go, #175 P7). doorPath and doorAddr come from |
| 285 | // WithEdgeDoor: the socket to listen on ("" = no door, the default) and |
| 286 | // the box's public address, which every door request presents as its |
| 287 | // own. doorKeys is the signed door-key mirror armDoor builds; nil admits |
| 288 | // nothing. closeDoor, set once the door is open, shuts it and removes the |
| 289 | // socket. All four are written before the door listens and never after. |
| 290 | doorPath string |
| 291 | doorAddr string |
| 292 | doorKeys *doorKeyMirror |
| 293 | closeDoor func() |
| 294 | // gens is the activator's generation view (zero-downtime spec §7.2), |
| 295 | // set in New only when the activator implements GenSource AND its |
| 296 | // switch is on; genPools is the edge's per-generation-socket idle |
| 297 | // pools, retired through gens.OnGenRetired. Both nil with the switch off |
| 298 | // (CARLOS_SWAP unset), and then nothing below reads them: managed |
| 299 | // routes get unixProxy(t.Socket) and the row's version, as before. |
| 300 | gens GenSource |
| 301 | genPools *GenPools |
| 302 | } |
| 303 | |
| 304 | // RunOption configures the Edge that Run/RunWithActivator* builds, before it |
| 305 | // serves anything. Variadic rather than another positional parameter: the |
| 306 | // composing caller (internal/host's agent role) hands host-owned state DOWN |
| 307 | // into this package, and every existing call site keeps compiling unchanged. |
| 308 | type RunOption func(*Edge) |
| 309 | |
| 310 | // WithIssuerGate hands the edge the deployment's certificate-issuer gate — |
| 311 | // the housekeeper's, in the agent role. Without it the edge issues exactly |
| 312 | // as it always has. See IssuerGate (certsync.go). |
| 313 | func WithIssuerGate(g IssuerGate) RunOption { return func(e *Edge) { e.issuer = g } } |
| 314 | |
| 315 | // WithIssuerContender tells the edge whether this box CONTENDS in the |
| 316 | // deployment's issuer election at all — a different question from holding |
| 317 | // it right now (the gate above answers that, moment to moment). The cert |
| 318 | // backfill (certsync.go, #318) is allowed exactly on contenders: an |
| 319 | // agent-run box whose housekeeper mode is "never" (a fleet box grandfathered |
| 320 | // with the bucket env) carries a gate func that can never be held, and its |
| 321 | // disk holds a customer's certs — not this deployment's to replicate. nil — |
| 322 | // no agent handed one down — reads as "does not contend". |
| 323 | func WithIssuerContender(c func() bool) RunOption { return func(e *Edge) { e.contends = c } } |
| 324 | |
| 325 | // WithBoxLabel hands the edge this box's own label, the one its edges file is |
| 326 | // published under (internal/host's metricsBoxLabel — the same call the edges |
| 327 | // publisher makes, so the two can never disagree about which file is ours). |
| 328 | // Without it the home map excludes nothing and therefore offers nothing: an |
| 329 | // edge that cannot recognise its own hosts must not hand out dial targets. |
| 330 | // See homemap.go. |
| 331 | func WithBoxLabel(label string) RunOption { return func(e *Edge) { e.box = label } } |
| 332 | |
| 333 | // WithChangeWatch hands the edge the agent's change-index watch (#336) as a |
| 334 | // function, called on every cert-mirror and home-map poll. A function and |
| 335 | // not the watch itself, because the agent may arm its watch after the edge |
| 336 | // has started (armChangeIndex runs on a goroutine on some boxes), and an |
| 337 | // edge holding a nil captured at start would stay disarmed for the life of |
| 338 | // the process. One watch per process, the agent's: the edge never arms one |
| 339 | // of its own, so CARLOS_CHANGE_INDEX and the fleet-box exclusion stay |
| 340 | // decided in one place. The function, or the watch it returns, nil keeps |
| 341 | // both loops on today's full reads. |
| 342 | func WithChangeWatch(w func() *release.ChangeWatch) RunOption { |
| 343 | return func(e *Edge) { e.changeWatch = w } |
| 344 | } |
| 345 | |
| 346 | // WithEdgeDoor asks for the edge-local door (door.go, #175 P7) on the unix |
| 347 | // socket at path, presenting publicAddr, the box's public address, as the |
| 348 | // address of every door request. The door opens once the home map and the |
| 349 | // door-key mirror are armed, and only with a deployment store, a release |
| 350 | // public key and a public address; otherwise the edge logs why it is off. |
| 351 | func WithEdgeDoor(path, publicAddr string) RunOption { |
| 352 | return func(e *Edge) { e.doorPath, e.doorAddr = path, publicAddr } |
| 353 | } |
| 354 | |
| 355 | // upstream is one served host: the handler for its backend, plus — when the |
| 356 | // activator manages it — the route itself, so a request can trigger the wake. |
| 357 | type upstream struct { |
| 358 | proxy http.Handler |
| 359 | route registry.Route |
| 360 | // hibernating marks an activator-managed instance: the edge itself knows |
| 361 | // whether its process is live (it's the edge's child), so "dead backend" |
| 362 | // detection is the activator's state table, not a failed dial. |
| 363 | hibernating bool |
| 364 | // viaTunnel marks a host THIS box proxies over a reverse channel (a |
| 365 | // Tunnel-backed or fleet-placed route with a hub to carry it). It is the |
| 366 | // build-time decision, not the row's columns: the fleet box that serves |
| 367 | // the app carries the same fleet placement on its row and serves it from |
| 368 | // the socket, and must file what it sees (emitEdgeError). |
| 369 | viaTunnel bool |
| 370 | // alias marks an entry mapped in from route_aliases: it shares the |
| 371 | // primary's proxy/route/hibernating verbatim (so hibernation wakes the |
| 372 | // primary's actEntry and #152's per-primary prober state applies), and |
| 373 | // the flag exists so wildcard parent derivation (currentHosts) never |
| 374 | // treats a customer hostname as a platform host. |
| 375 | alias bool |
| 376 | // paths, when non-nil, splits this hostname's traffic by leading URL |
| 377 | // segment: a bare first segment ("paulca") mapped to the HOSTNAME of the |
| 378 | // route that serves it, resolved against the same live map at request |
| 379 | // time. Nil for every host that does not use prefix routing, which is |
| 380 | // every host on the platform until a claim publishes a map — the nil |
| 381 | // check in lookupUpstreamPath is the entire cost this feature imposes on |
| 382 | // everyone else. |
| 383 | // |
| 384 | // Hostnames rather than resolved upstreams, deliberately: an upstream |
| 385 | // captured here would be a stale copy of the target's entry the moment |
| 386 | // the target's route changed within the same map build. |
| 387 | paths map[string]string |
| 388 | // genManaged marks an activator-managed instance proxied through |
| 389 | // appSocketProxy (generations on). Its cacheable responses are stored |
| 390 | // only when tagged with the generation that served them (cache.go). |
| 391 | genManaged bool |
| 392 | } |
| 393 | |
| 394 | // New builds an Edge over reg and performs the startup reload. act may be nil |
| 395 | // (no hibernation). |
| 396 | func New(reg *registry.Registry, act Activator) (*Edge, error) { |
| 397 | e := &Edge{reg: reg, act: act, gate: gate.New(), probers: newProberSet(), gzipOn: gzipEnabled(), steeringHost: steeringProbeHostFromEnv(), appsDomain: appsDomainFromEnv(), |
| 398 | lbHealth: lbHealthFromEnv(os.Getenv, act)} |
| 399 | e.geo = geo.NewSwappable() |
| 400 | e.geoLookup = e.geo.Lookup |
| 401 | // Generations (zero-downtime spec §7.2), before the startup Reload so |
| 402 | // its first table build already proxies through them. |
| 403 | if gs, ok := act.(GenSource); ok && gs.GenerationsOn() { |
| 404 | e.gens, e.genPools = gs, NewGenPools() |
| 405 | gs.OnGenRetired(e.genPools.Retire) |
| 406 | } |
| 407 | if err := e.Reload(); err != nil { |
| 408 | return nil, err |
| 409 | } |
| 410 | return e, nil |
| 411 | } |
| 412 | |
| 413 | // EnableStatic turns on the static serving class: routes with Kind=="static" |
| 414 | // resolve their app's channel pointer against store (verified with pub) and |
| 415 | // serve the extracted tree from cacheDir. Call Reload afterwards so existing |
| 416 | // static routes pick up real handlers; without EnableStatic they 404. |
| 417 | // |
| 418 | // Kind=="status" routes (statusHandlerFor, status.go) share this same store |
| 419 | // handle rather than getting their own EnableXxx entry point: it's the same |
| 420 | // deployment bucket, just a fixed unsigned three-key menu (status/index.html, |
| 421 | // status/fleet.json, status/status.css — see internal/status's PublishFleet) |
| 422 | // instead of a signed, versioned app tree. pub and cacheDir are meaningless |
| 423 | // to the status class (no signature to verify, nothing to extract to disk), |
| 424 | // so it only ever reads e.staticStore off this call. |
| 425 | func (e *Edge) EnableStatic(store release.Store, pub ed25519.PublicKey, cacheDir string) { |
| 426 | e.staticStore = store |
| 427 | e.staticPub = pub |
| 428 | e.staticCacheDir = cacheDir |
| 429 | e.staticRecheck = staticRecheck |
| 430 | e.armStoreDoor() |
| 431 | } |
| 432 | |
| 433 | // EnableCache wires a response cache into the edge. Called once at startup |
| 434 | // (RunWithActivator or Run); nil is a valid no-op, matching the package's |
| 435 | // existing nil-means-disabled convention for Activator. |
| 436 | func (e *Edge) EnableCache(cache *Cache) { |
| 437 | e.cache = cache |
| 438 | // The generation guard (spec §7.2), only with generations on: e.gens is |
| 439 | // nil otherwise and the cache stays untagged, exactly as before. |
| 440 | if cache != nil && e.gens != nil { |
| 441 | cache.SetGenerations(e.gens.CacheGen) |
| 442 | } |
| 443 | } |
| 444 | |
| 445 | // EnableTunnels wires the reverse-channel hub in: the edge answers hub.Host() |
| 446 | // itself (registration) and routes with Tunnel set proxy over their named |
| 447 | // channel. Call before Reload, same contract as EnableStatic — without a |
| 448 | // Reload afterwards, tunnel routes stay skipped from the last table build. |
| 449 | func (e *Edge) EnableTunnels(hub *TunnelHub) { |
| 450 | e.tunnels = hub |
| 451 | e.armStoreDoor() |
| 452 | } |
| 453 | |
| 454 | // EnableUsage arms per-route request/byte metering. Called by the agent |
| 455 | // role after construction (mirroring how tunnelLoop attaches via the ready |
| 456 | // hook); the bare `carlos edge` role never calls it, so pure-proxy edges |
| 457 | // meter nothing. Safe to call after serving has started (atomic). |
| 458 | func (e *Edge) EnableUsage(m *UsageMeter) { e.usageMeter.Store(m) } |
| 459 | |
| 460 | // ErrorSink is where the edge files an error it observed on the serving |
| 461 | // path. apperr.Sink satisfies it; the interface lives here so this package |
| 462 | // depends on the contract, not the buffer. |
| 463 | type ErrorSink interface { |
| 464 | Add(acct, app string, r apperr.Record) |
| 465 | } |
| 466 | |
| 467 | type errorSinkRef struct{ ErrorSink } |
| 468 | |
| 469 | // EnableErrors wires the error sink in. Called once at startup by the |
| 470 | // agent; nil is a valid no-op. |
| 471 | func (e *Edge) EnableErrors(s ErrorSink) { |
| 472 | if s == nil { |
| 473 | e.errSink.Store(nil) |
| 474 | return |
| 475 | } |
| 476 | e.errSink.Store(&errorSinkRef{s}) |
| 477 | } |
| 478 | |
| 479 | // emitEdgeError files one edge-observed 5xx for rt. The path is sanitised |
| 480 | // (query and token segments off) before it goes on the record. A 502, 503 |
| 481 | // or 504 is filed under Availability (apperr/availability.go): named and |
| 482 | // so fingerprinted by status and cause, never by path, with the path kept |
| 483 | // only as the occurrence's sample. cause is what a proxy ErrorHandler |
| 484 | // noted; none means the upstream wrote the status itself. A failure whose |
| 485 | // client had already gone is not filed at all. |
| 486 | // |
| 487 | // When this box proxied the request over a reverse channel (viaTunnel), |
| 488 | // the upstream is not the app: it is another box's edge (the agent's, the |
| 489 | // only tunnel client), which files what it observes serving the app — |
| 490 | // including deciding NOT to file a wake 503 its activator answered |
| 491 | // (MarkAnswered). So this box files only what it noted itself, the hop |
| 492 | // failing; a status that merely passed through would be the same failure |
| 493 | // filed twice, and a 502/503 labelled "the app answered it itself" when the |
| 494 | // app never saw the request. The box at the far end serves the same route |
| 495 | // from its socket (viaTunnel false) and files everything as usual. |
| 496 | func (e *Edge) emitEdgeError(rt registry.Route, viaTunnel bool, r *http.Request, status int, cause string) { |
| 497 | ref := e.errSink.Load() |
| 498 | if ref == nil || cause == causeClientGone || (viaTunnel && cause == "") { |
| 499 | return |
| 500 | } |
| 501 | path := apperr.SanitizePath(r.URL.Path) |
| 502 | rec := apperr.Record{ |
| 503 | Host: rt.Host, |
| 504 | Source: apperr.SourceEdge, |
| 505 | Kind: apperr.KindHTTP, |
| 506 | Name: fmt.Sprintf("%d %s %s", status, r.Method, path), |
| 507 | Path: path, |
| 508 | Status: status, |
| 509 | Release: rt.Version, |
| 510 | } |
| 511 | if apperr.UnavailableStatus(status) { |
| 512 | if cause == "" { |
| 513 | cause = apperr.CauseUpstream |
| 514 | } |
| 515 | rec.Kind, rec.Cause, rec.Name = apperr.KindUnavailable, cause, apperr.AvailabilityName(status, cause) |
| 516 | } |
| 517 | ref.Add(release.RouteAccount(rt), rt.App, rec) |
| 518 | } |
| 519 | |
| 520 | // proxyCause classifies why a proxy could not get an answer for r. The |
| 521 | // client's own context is checked before the error itself: a client that |
| 522 | // left surfaces as a canceled dial or read whatever the instance was doing. |
| 523 | // Leaving soon after asking is the client's business and is not filed; |
| 524 | // leaving after clientGoneGrace means the instance kept it waiting, and is. |
| 525 | func proxyCause(r *http.Request, err error) string { |
| 526 | if r.Context().Err() != nil { |
| 527 | if waited(r) >= clientGoneGrace { |
| 528 | return apperr.CauseAbandoned |
| 529 | } |
| 530 | return causeClientGone |
| 531 | } |
| 532 | var ne net.Error |
| 533 | switch { |
| 534 | case errors.Is(err, syscall.ECONNREFUSED), errors.Is(err, syscall.ENOENT): |
| 535 | return apperr.CauseRefused |
| 536 | case errors.Is(err, context.DeadlineExceeded), errors.As(err, &ne) && ne.Timeout(): |
| 537 | return apperr.CauseTimeout |
| 538 | case errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF), |
| 539 | errors.Is(err, syscall.ECONNRESET), errors.Is(err, syscall.EPIPE): |
| 540 | return apperr.CauseReset |
| 541 | } |
| 542 | return apperr.CauseProxy |
| 543 | } |
| 544 | |
| 545 | // tunnelCause is proxyCause for a proxy whose far side is a fleet box's |
| 546 | // tunnel: every failure but the client's own leaving is the hop's, not |
| 547 | // the instance's. |
| 548 | func tunnelCause(r *http.Request, err error) string { |
| 549 | if c := proxyCause(r, err); c == causeClientGone || c == apperr.CauseAbandoned { |
| 550 | return c |
| 551 | } |
| 552 | return apperr.CauseTunnel |
| 553 | } |
| 554 | |
| 555 | // defaultProxyError is httputil.ReverseProxy's own default ErrorHandler — |
| 556 | // the same log line, the same bare 502 — plus the cause note, for the |
| 557 | // proxies that had no handler of their own. |
| 558 | func defaultProxyError(classify func(*http.Request, error) string) func(http.ResponseWriter, *http.Request, error) { |
| 559 | return func(w http.ResponseWriter, r *http.Request, err error) { |
| 560 | noteCause(r, classify(r, err)) |
| 561 | log.Printf("http: proxy error: %v", err) |
| 562 | w.WriteHeader(http.StatusBadGateway) |
| 563 | } |
| 564 | } |
| 565 | |
| 566 | // armStoreDoor builds (or clears) the fleet store door from whatever is |
| 567 | // configured now. Called from both EnableStatic and EnableTunnels because the |
| 568 | // door needs one thing from each and the two are wired in either order at |
| 569 | // startup (enableStaticFromEnv runs first today, but a caller composing an |
| 570 | // Edge by hand — every test does — must not have to know that). A deployment |
| 571 | // with no release pubkey configured therefore has no store handle and so no |
| 572 | // door: /store/ then reaches the hub, which 404s it, byte-identically. |
| 573 | func (e *Edge) armStoreDoor() { |
| 574 | if e.tunnels == nil || e.staticStore == nil { |
| 575 | e.door = nil |
| 576 | return |
| 577 | } |
| 578 | e.door = &storeDoor{hub: e.tunnels, store: e.staticStore, fleetAccount: e.fleetAccount} |
| 579 | } |
| 580 | |
| 581 | // fleetAccount resolves an authenticated fleet name to its owning account's |
| 582 | // sqid from the last reload's snapshot. A fleet the snapshot does not carry |
| 583 | // (created since the last reload, or deleted) has no tenancy, so the door |
| 584 | // answers it nothing — the same convergence bound as its channel credentials, |
| 585 | // which come from the same pass. |
| 586 | func (e *Edge) fleetAccount(fleet string) (string, bool) { |
| 587 | m := e.fleetAccounts.Load() |
| 588 | if m == nil { |
| 589 | return "", false |
| 590 | } |
| 591 | sqid, ok := (*m)[fleet] |
| 592 | if !ok || sqid == "" { |
| 593 | return "", false |
| 594 | } |
| 595 | return sqid, true |
| 596 | } |
| 597 | |
| 598 | // SetStaticRecheck tunes how long a static route trusts a resolved channel |
| 599 | // pointer before re-reading it (default 60s; 0 = every request). Call after |
| 600 | // EnableStatic and before Reload; handlers built on the next reload use it. |
| 601 | func (e *Edge) SetStaticRecheck(d time.Duration) { e.staticRecheck = d } |
| 602 | |
| 603 | // staticHandlerFor builds the handler for one static route: the real serving |
| 604 | // class when EnableStatic has configured it, else the pre-config 404 (the |
| 605 | // route exists — and its cert can be minted — before the bytes can flow). |
| 606 | func (e *Edge) staticHandlerFor(rt registry.Route) http.Handler { |
| 607 | if e.staticStore == nil || len(e.staticPub) == 0 || rt.App == "" { |
| 608 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 609 | http.Error(w, "static serving not configured", http.StatusNotFound) |
| 610 | }) |
| 611 | } |
| 612 | h := newStaticHandler(rt, e.staticStore, e.staticPub, e.staticCacheDir) |
| 613 | if ss, ok := h.(*staticSite); ok { |
| 614 | ss.recheck = e.staticRecheck |
| 615 | ss.ratchet = e.staticRatchet |
| 616 | } |
| 617 | return h |
| 618 | } |
| 619 | |
| 620 | // statusHandlerFor builds the handler for one status route: the real |
| 621 | // bucket-backed handler (status.go) once EnableStatic has wired a store, else |
| 622 | // the pre-config 404 — same cert-first shape as staticHandlerFor, so a |
| 623 | // status route's cert can be minted before the store config lands. |
| 624 | func (e *Edge) statusHandlerFor(rt registry.Route) http.Handler { |
| 625 | if e.staticStore == nil { |
| 626 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 627 | http.Error(w, "status serving not configured", http.StatusNotFound) |
| 628 | }) |
| 629 | } |
| 630 | return newStatusHandler(e.staticStore) |
| 631 | } |
| 632 | |
| 633 | // Reload rebuilds the host→backend map from the registry and swaps it in |
| 634 | // atomically. Safe to call while serving. |
| 635 | func (e *Edge) Reload() error { |
| 636 | routes, err := e.reg.List() |
| 637 | if err != nil { |
| 638 | return err |
| 639 | } |
| 640 | m := make(map[string]upstream, len(routes)) |
| 641 | // The hosts the activator should manage after this reload — feeds both the |
| 642 | // per-upstream gate and the retire pass below. |
| 643 | managed := make(map[string]bool) |
| 644 | // The probing jobs this reload's route set implies — one entry per addr |
| 645 | // route that carries a health config, fed to proberSet.Sync below. Hosts |
| 646 | // absent from it have their probers stopped, exactly as RetireExcept |
| 647 | // retires activator children for hosts absent from `managed`. |
| 648 | probeSpecs := make(map[string]probeSpec) |
| 649 | for _, t := range routes { |
| 650 | // A sidecar is a member that serves nothing: it binds no socket, and |
| 651 | // nothing — inside the box or outside it — may ever dial it. Refused |
| 652 | // here on KIND, as the first act of the loop and ahead of every other |
| 653 | // check, rather than left to fall through the dispatch switch for want |
| 654 | // of a Socket/Addr/Tunnel. |
| 655 | // |
| 656 | // That omission is not a guarantee. The fleet arm below binds on |
| 657 | // placement alone (`isFleetPlaced && e.tunnels != nil`, no kind, no |
| 658 | // backend column), and a provisioned Eleven sidecar IS fleet-placed |
| 659 | // (region "fleet/<name>") — so the row this feature creates reaches a |
| 660 | // working internet-facing proxy by shape. This refusal is the |
| 661 | // structural guarantee that survives a future arm, column or |
| 662 | // placement finding the row: no map entry, so no proxy; and no map |
| 663 | // entry means Allowed says no, so no ACME issuance either. |
| 664 | if t.IsSidecar() { |
| 665 | continue |
| 666 | } |
| 667 | // Only serve ready routes: a 'creating'/'failed' entry has no working |
| 668 | // backend, so proxying it would just stall → 502 — AND, since Allowed |
| 669 | // is the autocert HostPolicy, serving it would burn an ACME issuance |
| 670 | // on a host that can't answer yet. |
| 671 | if t.Status != "ready" { |
| 672 | continue |
| 673 | } |
| 674 | var proxy http.Handler |
| 675 | // viaTunnel: this host's bytes leave the box over a reverse channel, |
| 676 | // which is what disqualifies it from hibernation below. |
| 677 | viaTunnel := false |
| 678 | // genManaged: an activator-managed instance reached through its |
| 679 | // current generation (appSocketProxy), generations on only. |
| 680 | genManaged := false |
| 681 | fleet, isFleetPlaced := registry.FleetPlacement(t.Region) |
| 682 | switch { |
| 683 | // Both reverse proxies get the Carlos-Purge response hook |
| 684 | // (purge_header.go): the app declares its own cache invalidations on |
| 685 | // the response it is already returning, and the edge applies them and |
| 686 | // strips the header before any client sees it. The hook closes over e |
| 687 | // rather than e.cache, because EnableCache runs AFTER New's startup |
| 688 | // Reload — capturing e.cache here would capture nil and the channel |
| 689 | // would be permanently inert. |
| 690 | // |
| 691 | // The whole ROUTE goes in, not just its host: Carlos-Purge-Host lets a |
| 692 | // response redirect its declaration at a sibling route, and the edge |
| 693 | // validates that against this route's App and Account. Those two are |
| 694 | // captured (the tenancy the edge resolved this request to); the sibling |
| 695 | // lookup is not — it re-reads e.current() per response, so a route |
| 696 | // added after this proxy was built still validates. |
| 697 | // |
| 698 | // Static and status routes get no hook: their responses are the |
| 699 | // edge's own, built from bucket bytes by static.go/status.go, with no |
| 700 | // upstream in a position to declare anything. |
| 701 | // |
| 702 | // The tunnel case comes FIRST, ahead of socket/addr, because a route |
| 703 | // can carry both: `carlos route -tunnel` is a narrow UPDATE that does |
| 704 | // not clear socket/addr, so an operator moving a local app onto a |
| 705 | // remote host leaves the old column set — and the binding they just |
| 706 | // typed is the one they meant. It is guarded on e.tunnels rather than |
| 707 | // skipping unconditionally so that an edge with no hub configured |
| 708 | // keeps serving such a route from its old socket/addr (degrade to the |
| 709 | // previous backend, never take the host off the air); a tunnel-only |
| 710 | // route on such an edge matches nothing and falls to |
| 711 | // `default: continue`, un-served and un-certed, until the hub lands. |
| 712 | // |
| 713 | // That degrade path only covers a HUB-LESS edge (e.tunnels == nil). |
| 714 | // A hub-configured edge whose client set doesn't (yet) include this |
| 715 | // route's channel name still takes this branch — e.tunnels != nil is |
| 716 | // all this case checks — and the built proxy 502s every request, even |
| 717 | // though t.Addr may hold a perfectly working backend. Deliberate: once |
| 718 | // a hub exists, a `-tunnel` binding is the newest operator intent and |
| 719 | // wins outright rather than silently falling back to a stale Addr. |
| 720 | case t.Tunnel != "" && e.tunnels != nil: |
| 721 | p := tunnelProxy(t.Tunnel, e.tunnels) |
| 722 | p.ModifyResponse = e.responseModifier(t) |
| 723 | proxy = p |
| 724 | viaTunnel = true |
| 725 | // Fleet placement (Region == "fleet/<name>", registry.FleetPlacement) is |
| 726 | // a customer-fleet route: its bytes leave the box over the SAME reverse |
| 727 | // channel a `-tunnel` binding uses, just resolved dynamically per |
| 728 | // request rather than to one fixed name — see fleetProxy. Guarded on |
| 729 | // e.tunnels for the identical reason the Tunnel case above is: a |
| 730 | // fleet-placed route on a hub-less edge degrades to its old Socket/Addr |
| 731 | // (or goes unserved) rather than the box silently going dark for a |
| 732 | // config gap that isn't its fault. viaTunnel is shared with the Tunnel |
| 733 | // case on purpose — a fleet-served instance is exactly as un-wakeable |
| 734 | // locally as a `-tunnel`-bound one; see the hibernation comment below. |
| 735 | case isFleetPlaced && e.tunnels != nil: |
| 736 | p := fleetProxy(fleet, t.FleetBox, e.tunnels) |
| 737 | p.ModifyResponse = e.responseModifier(t) |
| 738 | proxy = p |
| 739 | viaTunnel = true |
| 740 | case t.Socket != "": |
| 741 | // An activator-managed instance's process binds a generation |
| 742 | // socket, not t.Socket, once generations are on (spec §7.1): |
| 743 | // every new connection asks the activator for the current one. |
| 744 | // t.Kind/t.Hibernate are the activator-managed test below |
| 745 | // (hib); viaTunnel is false in this arm by construction. |
| 746 | var p *httputil.ReverseProxy |
| 747 | if e.gens != nil && t.Kind == "instance" && t.Hibernate { |
| 748 | p, genManaged = appSocketProxy(t.Host, e.gens, e.genPools), true |
| 749 | } else { |
| 750 | p = unixProxy(t.Socket) |
| 751 | } |
| 752 | p.ModifyResponse = e.responseModifier(t) |
| 753 | proxy = p |
| 754 | // TCP upstreams. The gate is AddrList (registry), not t.Addr, so a |
| 755 | // multi-upstream route — which writes the Addrs column and leaves the |
| 756 | // legacy Addr empty — is served at all; AddrList folds the legacy |
| 757 | // single Addr into a one-element list, so nothing about the old shape |
| 758 | // moves. Position in the dispatch order is unchanged (tunnel > socket > |
| 759 | // addrs > static > status). |
| 760 | // |
| 761 | // Health is what selects the proxy, not the addr COUNT: |
| 762 | // |
| 763 | // - no health config → tcpProxy, byte-identical to the pre-#152 |
| 764 | // path. An operator who never asked for probing gets exactly the |
| 765 | // behavior they had, including the stdlib 502 on a dead backend. |
| 766 | // - health config → multiProxy, whether the route names one upstream |
| 767 | // or five. A single PROBED upstream is on the dashboard and answers |
| 768 | // 503 + Retry-After when it is down, which is the point of asking |
| 769 | // for a probe on a one-box route. |
| 770 | // |
| 771 | // A route with several addrs but no health config takes the unprobed |
| 772 | // path on addrs[0]: without a probe there is no live set to balance |
| 773 | // over, and inventing one would mean load-balancing onto upstreams |
| 774 | // nothing is checking. It is a config gap, not a serving failure — |
| 775 | // mint always writes both columns together (provision), so this is |
| 776 | // only reachable via a hand-written row. |
| 777 | case len(t.AddrList()) > 0: |
| 778 | addrs := t.AddrList() |
| 779 | var p *httputil.ReverseProxy |
| 780 | switch { |
| 781 | case t.Health == "": |
| 782 | p = tcpProxy(addrs[0]) |
| 783 | default: |
| 784 | var h provision.Health |
| 785 | if err := json.Unmarshal([]byte(t.Health), &h); err != nil { |
| 786 | // Never a dead route: a health column we cannot parse costs |
| 787 | // the route its probing, not its service. Logged rather |
| 788 | // than swallowed — it is a config error someone has to fix, |
| 789 | // and a route silently losing its HA is worse than a log |
| 790 | // line — but throttled, because a bad row is a STANDING |
| 791 | // condition that would otherwise print on every reload for |
| 792 | // as long as it exists. |
| 793 | e.logThrottled("health-parse "+t.Host, fmt.Sprintf( |
| 794 | "route %s: unparseable health config (%v) — serving %s unprobed", t.Host, err, addrs[0])) |
| 795 | p = tcpProxy(addrs[0]) |
| 796 | break |
| 797 | } |
| 798 | // Raw parse, deliberately un-normalized: Sync runs every spec |
| 799 | // through Health.WithDefaults itself, and normalizing here too |
| 800 | // would just be a second place to keep in step with it. |
| 801 | probeSpecs[t.Host] = probeSpec{Host: t.Host, Addrs: addrs, Health: h} |
| 802 | p = multiProxy(t.Host, e.probers) |
| 803 | } |
| 804 | p.ModifyResponse = e.responseModifier(t) |
| 805 | proxy = p |
| 806 | case t.Kind == "static": |
| 807 | proxy = e.staticHandlerFor(t) |
| 808 | case t.Kind == "status": |
| 809 | proxy = e.statusHandlerFor(t) |
| 810 | default: |
| 811 | continue // no backend to reach |
| 812 | } |
| 813 | // The deploy watch's proof and the human "verify with the binary you |
| 814 | // built" tell (carlos-deploy spec §7): the version this route is |
| 815 | // serving, from the row the adopt pass stamps AFTER the restart. |
| 816 | // Server:-header-grade disclosure, ruled public 2026-08-08. |
| 817 | // |
| 818 | // Freshness contract: on every UNCACHED response, this is live — |
| 819 | // Reload rebuilds these handlers on every registry change, and |
| 820 | // Fingerprint is content-accurate since #15 closed, so an uncached |
| 821 | // header can't go stale. A CACHE HIT bypasses this handler entirely |
| 822 | // (ServeHTTP's GetFor/writeCached path, ahead of the wrapped proxy) |
| 823 | // and replays whatever header maybeStore froze at store time, so it |
| 824 | // can lag a promote by up to the cache's TTL (clamped to |
| 825 | // DefaultCacheMaxTTL, 5 minutes) or until a Carlos-Purge evicts the |
| 826 | // entry — Reload never purges e.cache. That lag is one-directional |
| 827 | // and fails safe for the watch: a stale entry only ever carries the |
| 828 | // OLD version, so it can read as "not yet" but never false-positive |
| 829 | // as the new one. A watcher that needs the fresh answer within the |
| 830 | // TTL window must bust the cache (e.g. a unique query string). |
| 831 | |
| 832 | // Static and status routes are excluded structurally, not because a |
| 833 | // Version is expected on them today (nothing adopts either, so the |
| 834 | // column is empty): the static handler stamps its OWN per-request |
| 835 | // version and must be the only voice on that header, and both classes |
| 836 | // have exits that are deliberately unstamped — static's "never |
| 837 | // resolved" 502 and its pre-EnableStatic 404. A stale row Version |
| 838 | // wrapping those would put a version on a response that is serving |
| 839 | // none, which is exactly the false "verified" the header exists to |
| 840 | // prevent. |
| 841 | // |
| 842 | // A genManaged route is excluded too: its header comes from the |
| 843 | // generation that served the request (genRoundTripper, spec §7.2), |
| 844 | // and the registry's Version names only what is installed. |
| 845 | if t.Version != "" && t.Kind != "static" && t.Kind != "status" && !genManaged { |
| 846 | proxy = versionHeader(t.Version, proxy) |
| 847 | } |
| 848 | // Hibernation is per-host: only an instance whose route carries the |
| 849 | // hibernate flag is activator-managed; unflagged instances and sibling |
| 850 | // services are proxied straight. With an activator wired but no route |
| 851 | // flagged, the edge therefore behaves exactly as it does without one — |
| 852 | // so the binary + env can ship fleet-wide first, and the flag flips |
| 853 | // one instance at a time. |
| 854 | // |
| 855 | // A tunnel-served host is never activator-managed, however it is |
| 856 | // flagged: the process lives on the far side of the channel and this |
| 857 | // edge cannot start it, so waking would mean launching a SECOND, |
| 858 | // local copy of the app beside the remote one. Leaving such a host out |
| 859 | // of `managed` is also what retires the local child a repointed route |
| 860 | // left running (RetireExcept, below) — the app moved to the remote |
| 861 | // host, so the process it left behind here should go. A fleet-placed |
| 862 | // route is the identical case (viaTunnel is set for both): whichever |
| 863 | // fleet box ends up serving it wakes its OWN local instance — that |
| 864 | // box's own activator, a separate process on a separate machine — so |
| 865 | // this platform edge must never manage it either. The lease race is |
| 866 | // the backstop if the fallback rule (fleetProxy) ever routes two |
| 867 | // requests to two different boxes for the same route in a gap. |
| 868 | hib := e.act != nil && t.Kind == "instance" && t.Hibernate && !viaTunnel |
| 869 | if hib { |
| 870 | managed[t.Host] = true |
| 871 | } |
| 872 | // A hibernate-flagged instance route on a box with no activator has |
| 873 | // nothing on this box that would ever spawn its process: the row is |
| 874 | // real and the socket path is derived, but only an armed activator |
| 875 | // starts the child that binds it, so a request that reaches this |
| 876 | // route here falls into unixProxy's dial and comes back as the |
| 877 | // stdlib ReverseProxy's bare, bodiless 502 — a status that reads as |
| 878 | // "your backend is broken" about a backend nothing here will start. |
| 879 | // |
| 880 | // Diagnostic only, and deliberately careful about what it claims. It |
| 881 | // says what this box knows — an eligible route, no activator — and |
| 882 | // not that every request 502s, because it cannot see whether |
| 883 | // something else is answering: a process started outside the |
| 884 | // platform could be bound to that socket already. The one deployment |
| 885 | // shape it was written for is a box whose /etc/carlos/host.env was |
| 886 | // hand-assembled and never given CARLOS_HIBERNATE_BUCKET, which the |
| 887 | // region-host bring-up docs make easy to do. |
| 888 | // |
| 889 | // The advice names both roles rather than guessing which one is |
| 890 | // running, because the fix differs and only one of them has one: |
| 891 | // `carlos edge` standalone passes a nil activator regardless of the |
| 892 | // environment (RunWithActivator), so an unconditional "set this |
| 893 | // variable" would send that operator after something that cannot |
| 894 | // help them. Saying both costs one clause and is true either way. |
| 895 | // |
| 896 | // Throttled: a standing condition, true on every reload for as long |
| 897 | // as the row and the missing activator both exist. Keyed per host — |
| 898 | // a value the box itself controls, per logThrottled's rule. Not a |
| 899 | // refusal to serve: dropping the route would only trade a 502 for a |
| 900 | // 404, and the row may yet be served once an activator arms. |
| 901 | if e.act == nil && t.Kind == "instance" && t.Hibernate && !viaTunnel { |
| 902 | e.logThrottled("no-activator "+t.Host, fmt.Sprintf( |
| 903 | "route %s is a hibernating instance but this box has no activator, so nothing here will create %s or start its process. If this box runs `carlos agent`, set CARLOS_HIBERNATE_BUCKET in /etc/carlos/host.env and restart carlos-edge; `carlos edge` on its own never runs instances.", |
| 904 | t.Host, t.Socket)) |
| 905 | } |
| 906 | m[t.Host] = upstream{proxy: proxy, route: t, hibernating: hib, viaTunnel: viaTunnel, genManaged: genManaged} |
| 907 | } |
| 908 | // Path routes, folded BEFORE aliases so an alias of a prefix-routed |
| 909 | // hostname inherits its map along with everything else it copies — an |
| 910 | // alias is meant to be indistinguishable from its primary, and a |
| 911 | // hostname that routed by prefix under one name but not the other would |
| 912 | // be a trap. |
| 913 | // |
| 914 | // Targets are recorded as hostnames, not resolved here: every route is |
| 915 | // already in m by this point, but resolving now would freeze a copy of |
| 916 | // the target's entry, and the target's own row may still be rewritten by |
| 917 | // the alias fold below. |
| 918 | paths, err := e.reg.ListPaths() |
| 919 | if err != nil { |
| 920 | return err |
| 921 | } |
| 922 | for _, p := range paths { |
| 923 | up, ok := m[p.Host] |
| 924 | if !ok { |
| 925 | // The hostname has no route on this box. The reconcile pass |
| 926 | // should not have produced this row; nothing to attach it to. |
| 927 | continue |
| 928 | } |
| 929 | if up.paths == nil { |
| 930 | up.paths = make(map[string]string) |
| 931 | } |
| 932 | up.paths[p.Prefix] = p.Target |
| 933 | m[p.Host] = up |
| 934 | } |
| 935 | // Attached hostnames (#151): each alias resolves to the primary's OWN |
| 936 | // entry — same proxy value, same route (so activator keying, version |
| 937 | // header, and purge tenancy are all the primary's). Routes always win |
| 938 | // a collision; an alias whose primary isn't in m (not ready, not |
| 939 | // local) is skipped — never a 502 route to nowhere. Errors abort the |
| 940 | // reload like every other registry read here: a transient failure must |
| 941 | // leave the old table serving. |
| 942 | aliases, err := e.reg.ListAliases() |
| 943 | if err != nil { |
| 944 | return err |
| 945 | } |
| 946 | for _, a := range aliases { |
| 947 | if _, taken := m[a.Alias]; taken { |
| 948 | log.Printf("edge: alias %s shadowed by a real route — skipped", a.Alias) |
| 949 | continue |
| 950 | } |
| 951 | up, ok := m[a.Host] |
| 952 | if !ok { |
| 953 | continue |
| 954 | } |
| 955 | up.alias = true |
| 956 | m[a.Alias] = up |
| 957 | } |
| 958 | // Fleet channel auth: registry-backed, so this refreshes on the SAME poll |
| 959 | // that rebuilds routes — Task 1's Fingerprint moves on every fleet |
| 960 | // mutation (including token rotation), and the edge's existing 2s |
| 961 | // Watch->Reload loop (below, Watch) is the whole convergence bound for a |
| 962 | // rotated or detached box. No hub configured (e.tunnels == nil, the vast |
| 963 | // majority of boxes today) means no fleet channels either — same |
| 964 | // nil-means-disabled convention as act/cache/staticStore. Errors here |
| 965 | // abort the WHOLE reload (routes included), matching e.reg.List() above: |
| 966 | // a transient registry read failure must leave both tables exactly as |
| 967 | // they were, not blank out fleet auth while routes move on. |
| 968 | if e.tunnels != nil { |
| 969 | auth, err := e.reg.FleetChannelAuth() |
| 970 | if err != nil { |
| 971 | return err |
| 972 | } |
| 973 | e.tunnels.SetDynamicAuth(auth) |
| 974 | // The store door's tenancy map, from the SAME pass: a box's |
| 975 | // credentials and the account its reads are scoped to must never come |
| 976 | // from two different reads of the registry. ListFleets goes over a |
| 977 | // fresh connection (like FleetChannelAuth above), so a fleet the |
| 978 | // console created moments ago is visible here rather than stuck |
| 979 | // behind a stale WAL snapshot on the long-lived handle. Same |
| 980 | // all-or-nothing error contract as everything else in Reload. |
| 981 | fleets, err := e.reg.ListFleets() |
| 982 | if err != nil { |
| 983 | return err |
| 984 | } |
| 985 | accounts := make(map[string]string, len(fleets)) |
| 986 | for _, f := range fleets { |
| 987 | accounts[f.Name] = f.Account |
| 988 | } |
| 989 | e.fleetAccounts.Store(&accounts) |
| 990 | } |
| 991 | // Probers BEFORE the map swap — the opposite order to RetireExcept below, |
| 992 | // and deliberately so. multiProxy answers 503 + Retry-After when the |
| 993 | // prober has nothing live for its host, and a prober this Sync has not |
| 994 | // installed yet is indistinguishable from one whose upstreams are all |
| 995 | // down. Swapping first would therefore open a window — however short — |
| 996 | // in which a request for a newly added (or newly re-specced) multi-upstream |
| 997 | // route reaches a handler whose live set is empty and gets a 503 for a |
| 998 | // perfectly healthy fleet. Syncing first cannot cause the mirror-image |
| 999 | // problem, because a prober is optimistic at birth (prober.go rule 1): the |
| 1000 | // new host's upstreams are all live the instant Sync returns, before any |
| 1001 | // request can reach the handler that reads them. |
| 1002 | // |
| 1003 | // The reverse case — a route REMOVED by this reload — loses its prober |
| 1004 | // before its handler, so a request landing in that window sees an empty |
| 1005 | // live set and gets 503 instead of proxying a route that is about to stop |
| 1006 | // existing. That is the trade, and it is the right way round: a 503 for a |
| 1007 | // host being deleted is a moment early, a 503 for a host being added is |
| 1008 | // simply wrong. |
| 1009 | e.probers.Sync(probeSpecs) |
| 1010 | e.upstreams.Store(&m) |
| 1011 | // Reconcile the activator with the new route set: a host un-flagged (or |
| 1012 | // removed) since the last reload may still have a live child holding its |
| 1013 | // socket and lease — retire it so a systemd unit can take over cleanly. |
| 1014 | // After the map swap above, so no new request re-enters the activator for |
| 1015 | // a host being retired. |
| 1016 | if e.act != nil { |
| 1017 | e.act.RetireExcept(managed) |
| 1018 | } |
| 1019 | if fn := e.reloadHook.Load(); fn != nil { |
| 1020 | (*fn)() |
| 1021 | } |
| 1022 | return nil |
| 1023 | } |
| 1024 | |
| 1025 | // OnReload registers fn to run after every successful Reload — after the new |
| 1026 | // route set is swapped in, so fn sees the post-reload table via current(). |
| 1027 | func (e *Edge) OnReload(fn func()) { |
| 1028 | e.reloadHook.Store(&fn) |
| 1029 | } |
| 1030 | |
| 1031 | // logThrottled logs msg the first time key occurs and every 300th time after |
| 1032 | // that — the same first-then-every-300th shape as the registry watcher below, |
| 1033 | // generalized out of logPurgeIssue (purge_header.go) when Reload gained a |
| 1034 | // second line with the same problem. |
| 1035 | // |
| 1036 | // The problem both share: the condition is a STANDING one, not an event. A |
| 1037 | // malformed Carlos-Purge header repeats on every response the app writes; an |
| 1038 | // unparseable health column repeats on every reload for as long as the row |
| 1039 | // exists. Either would fill a journal with the same sentence. The first |
| 1040 | // occurrence is always logged, so nothing is silently swallowed, and the |
| 1041 | // running count travels with each line so the rate stays visible. |
| 1042 | // |
| 1043 | // Keys must be built from values the box itself controls (a route's own host, |
| 1044 | // a complaint class) — never from remote input, or the map becomes a memory |
| 1045 | // growth vector. |
| 1046 | func (e *Edge) logThrottled(key, msg string) { |
| 1047 | e.logMu.Lock() |
| 1048 | if e.logN == nil { |
| 1049 | e.logN = make(map[string]int) |
| 1050 | } |
| 1051 | e.logN[key]++ |
| 1052 | n := e.logN[key] |
| 1053 | e.logMu.Unlock() |
| 1054 | if n == 1 || n%300 == 0 { |
| 1055 | log.Printf("edge: %s (occurrence #%d)", msg, n) |
| 1056 | } |
| 1057 | } |
| 1058 | |
| 1059 | func (e *Edge) current() map[string]upstream { |
| 1060 | if p := e.upstreams.Load(); p != nil { |
| 1061 | return *p |
| 1062 | } |
| 1063 | return nil |
| 1064 | } |
| 1065 | |
| 1066 | // currentHosts lists the hosts of platform-minted routes — rows the console |
| 1067 | // minted directly (Platform) or the provisioning reconciler minted from a |
| 1068 | // bucket record (Provisioned) — the wildcard manager's view of the route |
| 1069 | // table for parent derivation. Not every served host: parents are derived |
| 1070 | // from these hosts, and if a custom domain shaped like x.y.<apps-domain> |
| 1071 | // ever entered the route table by some other path, it would mint a wildcard |
| 1072 | // parent and burn a Let's Encrypt issuance on something the wildcard was |
| 1073 | // never meant to cover. Provisioned rows are admitted since 2026-08-24: a |
| 1074 | // box whose routes are ALL records-provisioned (every multi-region edge, and |
| 1075 | // any new account on a single box) derived no parents at all and could not |
| 1076 | // serve a single mirrored wildcard — found on the first three-region |
| 1077 | // flagship roll. A provisioned CUSTOM domain (a static record like |
| 1078 | // story.carlosframework.com) is harmless here: parentsFor only mints a |
| 1079 | // parent for hosts exactly two labels under the apps domain, and custom |
| 1080 | // domains under the apps domain are refused at the console. Static parents |
| 1081 | // (e.g. tito's go.tito.io) come from CARLOS_WILDCARD_DOMAINS via env, so |
| 1082 | // they are unaffected by this filter. |
| 1083 | // |
| 1084 | // Attached hostnames (#151) are excluded by the !up.alias half: an alias |
| 1085 | // entry carries the PRIMARY's route row verbatim, Platform flag included, so |
| 1086 | // without the flag a customer domain would inherit its platform primary's |
| 1087 | // trust and mint a parent for a zone the platform doesn't own. |
| 1088 | func (e *Edge) currentHosts() []string { |
| 1089 | m := e.current() |
| 1090 | out := make([]string, 0, len(m)) |
| 1091 | for h, up := range m { |
| 1092 | if (up.route.Platform || up.route.Provisioned) && !up.alias { |
| 1093 | out = append(out, h) |
| 1094 | } |
| 1095 | } |
| 1096 | return out |
| 1097 | } |
| 1098 | |
| 1099 | // deploymentHosts is currentHosts plus every exact platform-class name the |
| 1100 | // home map says another box homes — the input the wildcard manager's parent |
| 1101 | // set is built from (#300). Both halves apply the same class bound (the |
| 1102 | // publisher marks what currentHosts would admit); a box with no home map |
| 1103 | // (no deployment store) is its own deployment and gets currentHosts alone. |
| 1104 | // Not to be confused with deploymentHost (singular): that is the cert |
| 1105 | // gates' admission predicate and admits more (aliases, the tunnel host). |
| 1106 | func (e *Edge) deploymentHosts() []string { |
| 1107 | own := e.currentHosts() |
| 1108 | hm := e.homes.Load() |
| 1109 | if hm == nil { |
| 1110 | return own |
| 1111 | } |
| 1112 | seen := make(map[string]bool, len(own)) |
| 1113 | for _, h := range own { |
| 1114 | seen[h] = true |
| 1115 | } |
| 1116 | for _, h := range hm.Hosts() { |
| 1117 | if !seen[h] { |
| 1118 | seen[h] = true |
| 1119 | own = append(own, h) |
| 1120 | } |
| 1121 | } |
| 1122 | return own |
| 1123 | } |
| 1124 | |
| 1125 | // UpstreamStates is the probe table as the rest of the process may see it: |
| 1126 | // route host -> each upstream's current state, in the route's addr order, |
| 1127 | // for probed (multi-upstream) routes only. A route with no health config |
| 1128 | // holds no prober and is absent, as is every host on an edge that probes |
| 1129 | // nothing at all — the common shape, which reports an empty map. |
| 1130 | // |
| 1131 | // The agent's status heartbeat is the caller (internal/host wires this in from |
| 1132 | // the edge-ready callback, the same seam tunnelLoop arrives through) and it |
| 1133 | // calls this once per tick, taking the WHOLE table each time rather than |
| 1134 | // diffing: everything returned is a copy proberSet.State built under the |
| 1135 | // per-host lock, so serializing it while probes keep running is safe, and a |
| 1136 | // route that stopped being probed is simply gone from the next call rather |
| 1137 | // than lingering as a stale entry someone has to remember to evict. |
| 1138 | func (e *Edge) UpstreamStates() map[string][]UpstreamState { |
| 1139 | return e.probers.State() |
| 1140 | } |
| 1141 | |
| 1142 | // Allowed reports whether host is configured — used as the autocert |
| 1143 | // HostPolicy, so a certificate is only ever requested for a host we serve. |
| 1144 | func (e *Edge) Allowed(host string) bool { |
| 1145 | // Never certable directly: refuse the wildcard label itself rather than |
| 1146 | // resting on autocert's own IDN rejection of "*" to keep the bounded set. |
| 1147 | if strings.HasPrefix(host, "*.") { |
| 1148 | return false |
| 1149 | } |
| 1150 | // The reserved tunnel hostname is not a registry route, but remote hosts |
| 1151 | // dial it over TLS on this same 443 listener — so it must be Allowed, and |
| 1152 | // autocert will mint for it. That is intended: it is a hostname the |
| 1153 | // operator configured on this box, not a stranger-supplied SNI. |
| 1154 | if e.tunnels != nil && host == e.tunnels.Host() { |
| 1155 | return true |
| 1156 | } |
| 1157 | // Deliberately the EXACT map only — not lookupUpstream's wildcard |
| 1158 | // fallback. A catch-all alias ("*.<domain>") is a bounded, registry-backed |
| 1159 | // name; autocert's per-host HostPolicy gates a NEW per-host ACME order, |
| 1160 | // which is unbounded (labels are attacker-chosen) and shared across every |
| 1161 | // tenant on the box (one ACME account, one rate-limit bucket). Coverage |
| 1162 | // for a one-label host under a catch-all comes from the WILDCARD cert |
| 1163 | // (wrapGetCertificate calls certFor before ever reaching autocert); this |
| 1164 | // exact check only decides per-host HTTP-01 fallback for ROUTED hosts — |
| 1165 | // widening it to the wildcard fallback would let a stranger's SNI |
| 1166 | // <anything>.<domain> drive a fresh order and exhaust the box's shared |
| 1167 | // ACME quota for every tenant on it. |
| 1168 | _, ok := e.current()[host] |
| 1169 | return ok |
| 1170 | } |
| 1171 | |
| 1172 | // deploymentHost reports whether host is served HERE (Allowed) or is exactly |
| 1173 | // named by another box's edges file (homeMap.KnownHost) — the admission bound |
| 1174 | // for the three cert-adjacent gates in certsync.go: issuerHostPolicy's order |
| 1175 | // gate and the HTTP-01 and tls-alpn-01 read-throughs. |
| 1176 | // |
| 1177 | // The widening exists because Allowed alone was the wrong bound the moment a |
| 1178 | // deployment grew a second edge (#277): DNS for a home-boxed route can point |
| 1179 | // at any edge, the backhaul then dials the home box with the real SNI, the |
| 1180 | // home box orders — and the CA validates against whichever edge DNS names, |
| 1181 | // which refused every challenge for a host outside its OWN registry. Five |
| 1182 | // such failures burn the name's Let's Encrypt per-hostname budget in minutes |
| 1183 | // while the route is down. Both halves are registry-backed and |
| 1184 | // deployment-controlled: the home map is folded from control/edges/<box>.json, |
| 1185 | // which each box writes from its own registry (internal/host's edgesFilePass), |
| 1186 | // so a stranger cannot add a name to it without a control-bucket write. |
| 1187 | // |
| 1188 | // EXACT only, on both halves: Allowed refuses the wildcard key and consults |
| 1189 | // the exact route map, KnownHost refuses "*." and skips Home()'s one-label |
| 1190 | // fallback. That is PR #180's rule, unchanged and merely widened from "this |
| 1191 | // box's routes" to "this deployment's exactly-named routes" — an |
| 1192 | // attacker-chosen label under a catch-all must never drive an ACME order or a |
| 1193 | // bucket read. |
| 1194 | // |
| 1195 | // Both halves take the same normalized form: serverName/requestHost lowercase |
| 1196 | // and punycode into the shape registry rows, and therefore edges files, are |
| 1197 | // written in. e.homes may hold nil before the map arms; KnownHost is |
| 1198 | // nil-receiver-safe, matching Home's pattern. |
| 1199 | func (e *Edge) deploymentHost(host string) bool { |
| 1200 | return e.Allowed(host) || e.homes.Load().KnownHost(host) |
| 1201 | } |
| 1202 | |
| 1203 | // lookupIn resolves host to its upstream entry within a given map — the |
| 1204 | // HOST half of the request-routing lookup (NOT Allowed's autocert HostPolicy |
| 1205 | // — see the comment there). ServeHTTP does not call it directly: it calls |
| 1206 | // lookupUpstreamPath, which wraps this and then resolves a leading path |
| 1207 | // segment for the hostnames that carry a prefix map. An exact match (a real route, or a plain #151 |
| 1208 | // alias landed in the same map) always wins; on a miss, a one-label wildcard |
| 1209 | // key folds in: "*.<parent>" is the shape Task 3's reconcile pass derives |
| 1210 | // from a claim's catch-all row, and it lands in the SAME map as every other |
| 1211 | // alias (up.alias = true, primary's route + proxy verbatim) so this fallback |
| 1212 | // resolves to the primary's own entry — the identical map value the |
| 1213 | // #151/#152 agreed rule requires, not a value that merely behaves the same. |
| 1214 | // A wildcard matches exactly one label, mirroring the TLS cert: host[i:] |
| 1215 | // keeps the LEADING dot of the FIRST label only, so a two-label host |
| 1216 | // (a.b.woodstar.app) composes "*.b.woodstar.app" and misses, and the apex |
| 1217 | // itself (woodstar.app) composes "*.app" — never its own "*.woodstar.app" — |
| 1218 | // so it can never match its own wildcard. |
| 1219 | func lookupIn(m map[string]upstream, host string) (upstream, bool) { |
| 1220 | if up, ok := m[host]; ok { |
| 1221 | return up, true |
| 1222 | } |
| 1223 | if i := strings.IndexByte(host, '.'); i > 0 { |
| 1224 | if up, ok := m["*"+host[i:]]; ok { |
| 1225 | return up, true |
| 1226 | } |
| 1227 | } |
| 1228 | return upstream{}, false |
| 1229 | } |
| 1230 | |
| 1231 | // lookupUpstream resolves a hostname alone. Kept as the primitive because not |
| 1232 | // every caller is answering "where does this REQUEST go" — the :80->:443 |
| 1233 | // routability check in httpFallbackHandler asks only whether this box serves |
| 1234 | // the hostname at all, and giving that question a path would make a |
| 1235 | // hostname's routability depend on which URL happened to ask. |
| 1236 | // servesExactly reports whether this box's table has host as an exact key, |
| 1237 | // as opposed to answering it only through a "*.<parent>" fallback. |
| 1238 | func (e *Edge) servesExactly(host string) bool { |
| 1239 | _, ok := e.current()[host] |
| 1240 | return ok |
| 1241 | } |
| 1242 | |
| 1243 | func (e *Edge) lookupUpstream(host string) (upstream, bool) { |
| 1244 | return lookupIn(e.current(), host) |
| 1245 | } |
| 1246 | |
| 1247 | // lookupUpstreamPath resolves a request: the hostname exactly as |
| 1248 | // lookupUpstream does, then — only for a hostname carrying a prefix map — the |
| 1249 | // first path segment. |
| 1250 | // |
| 1251 | // Both lookups read ONE snapshot of the map. Calling lookupUpstream and then |
| 1252 | // current() again would let a reload land between them and resolve a prefix |
| 1253 | // against a different table than the host came from. |
| 1254 | // |
| 1255 | // Every miss falls through to the hostname's own route rather than failing: |
| 1256 | // an unlisted prefix, a dangling target, a request for "/". A prefix map is a |
| 1257 | // migration in progress, and the un-migrated answer is always the safe one. |
| 1258 | // |
| 1259 | // ATTRIBUTION: a path hit returns the TARGET's upstream entry wholesale, so |
| 1260 | // everything downstream that reads up.route reads the target's row, not the |
| 1261 | // requested hostname's. Four consequences, named here because this is where a |
| 1262 | // reader will look for them: |
| 1263 | // |
| 1264 | // 1. Usage metering bills the TARGET's route, account and app (the |
| 1265 | // UsageMeter wrap in ServeHTTP keys on up.route.Host). Correct: the |
| 1266 | // target's backend did the work and burned the bytes. Pinned by |
| 1267 | // TestPathRoutedRequestMetersAgainstTheTarget — a later "fix" that |
| 1268 | // attributes to the requested hostname would double-count nothing and |
| 1269 | // bill the wrong app. |
| 1270 | // 2. The X-Carlos-Version response header reports the TARGET's release. |
| 1271 | // Correct: it names the code that produced the response. |
| 1272 | // 3. Purge tenancy is the TARGET's app and account (purgeModifier is built |
| 1273 | // from up.route), while the response CACHE keys on the REQUESTED host. |
| 1274 | // Those two disagree, and closing that is what the path-routing branch |
| 1275 | // in purgeHostInTenancy exists for — see below. |
| 1276 | // 4. Activator keying and hibernation are the TARGET's (up.hibernating and |
| 1277 | // up.route come from the same entry), so a request for a flipped prefix |
| 1278 | // wakes the target instance, not the requested hostname's. Correct: the |
| 1279 | // target is the process that has to be awake to answer. |
| 1280 | // |
| 1281 | // How (3) is closed, and what an app must do. The cache keys on the requested |
| 1282 | // host (CacheKey{Host: host} in ServeHTTP) while purgeModifier is built from |
| 1283 | // the target's route, so a path-routed app emitting a bare Carlos-Purge still |
| 1284 | // purges under go.tito.io while its entry is stored under ti.to, and drops |
| 1285 | // nothing. The fix is the documented escape hatch: name the hostname the |
| 1286 | // request arrived on, Carlos-Purge-Host: ti.to. The override check used to |
| 1287 | // refuse that — it required the override to name a host in the SAME app, and |
| 1288 | // path routing exists precisely to cross an app boundary — and now allows it |
| 1289 | // when a prefix on the named host selects this route. Account is still |
| 1290 | // required. Aliases never hit any of this: an alias key maps to the primary's |
| 1291 | // own route, so the app comparison passes trivially. |
| 1292 | // |
| 1293 | // So an app can emit Carlos-Purge-Host set to the Host header it received, |
| 1294 | // unconditionally, from shared code. The inbound Host is preserved end to end, |
| 1295 | // so on an ordinary request that equals rt.Host and takes resolvePurgeHosts' |
| 1296 | // own-host branch; on a path-routed one it names the front hostname and takes |
| 1297 | // the path-routing branch. Nothing in the app has to know which it is. |
| 1298 | func (e *Edge) lookupUpstreamPath(host, urlPath string) (upstream, bool) { |
| 1299 | m := e.current() |
| 1300 | up, ok := lookupIn(m, host) |
| 1301 | if !ok || up.paths == nil { |
| 1302 | return up, ok |
| 1303 | } |
| 1304 | // BOTH FORMS, cleaned first. The prefix map is consulted for the |
| 1305 | // path.Clean'd first segment and then, if that missed and the two differ, |
| 1306 | // for the raw one. When both name a prefix, the cleaned one wins. |
| 1307 | // |
| 1308 | // Neither form alone is safe, and they fail in OPPOSITE directions. The |
| 1309 | // invariant being protected is one-way: during a reversible dual-run |
| 1310 | // migration, the app an account was flipped AWAY from must never answer |
| 1311 | // for that account. Serving a not-yet-flipped account from the NEW app is |
| 1312 | // the benign direction — at worst a 404 while the migration runs. |
| 1313 | // |
| 1314 | // - RAW alone misses "//paulca/foo". Rails and most other backends |
| 1315 | // collapse that to "/paulca/foo" and serve it, so the request falls |
| 1316 | // through to the legacy app, which then answers for a flipped |
| 1317 | // account. One character, silent. |
| 1318 | // - CLEANED alone misses "/paulca/../bob", which resolves to "bob". A |
| 1319 | // backend that does NOT collapse ".." — or that routes on the first |
| 1320 | // segment with a glob like "/:account/*rest" — then serves paulca's |
| 1321 | // page from the legacy app. The same bypass, reached the other way. |
| 1322 | // |
| 1323 | // Taking a hit from either form closes both: "//paulca/foo" hits on the |
| 1324 | // cleaned form, "/paulca/../bob" hits on the raw one, and a flipped |
| 1325 | // account reaches its new app whatever the backend does with the odd |
| 1326 | // shape. The cost is that "/paulca/../bob" reaches the NEW app carrying |
| 1327 | // "/paulca/../bob", which a normalising target resolves to "/bob" and |
| 1328 | // 404s — an account it was not given. Benign direction, and deliberate. |
| 1329 | // |
| 1330 | // Residue, measured rather than assumed. None of it crosses the |
| 1331 | // invariant above, and it is written down so nobody has to re-derive it: |
| 1332 | // |
| 1333 | // - "/%2e%2e/bob": r.URL.Path is already percent-DECODED, so this |
| 1334 | // cleans to "/bob" and matches "bob". The upstream receives the raw |
| 1335 | // "/%2e%2e/bob" and will most likely 404 it. Harmless. |
| 1336 | // - "/paulca%2ffoo": decodes to "/paulca/foo", so both forms match |
| 1337 | // "paulca" and the request reaches paulca's new app — while a strict |
| 1338 | // backend treats the upstream path as the SINGLE segment |
| 1339 | // "paulca/foo". A mis-flip toward the NEW app: the safe direction. |
| 1340 | // - A prefix literally named "." or ".." cannot exist: validPathPrefix |
| 1341 | // (internal/domains/claims.go) refuses both, so the raw form of |
| 1342 | // "/../bob" ("..") can never match anything. |
| 1343 | // |
| 1344 | // Do not collapse this back to a single form. Whichever one you keep, |
| 1345 | // you reopen the bypass the other one closes. |
| 1346 | // |
| 1347 | // The scope of the guarantee, stated so it is not over-read: the two |
| 1348 | // forms model percent-decoding and dot-segment removal, and the |
| 1349 | // invariant holds across every combination of a backend doing or not |
| 1350 | // doing those two. It is NOT a claim that no normalisation exists. A |
| 1351 | // backend that folds case ("/PAULCA/foo"), strips servlet-style path |
| 1352 | // parameters ("/paulca;jsessionid=x/foo"), accepts backslash |
| 1353 | // separators, strips trailing dots, or double-decodes could resolve a |
| 1354 | // path neither form matches into a flipped account's subtree. Rails |
| 1355 | // does none of these — Rack keeps ";" in PATH_INFO and routing is |
| 1356 | // case-sensitive — so Tito is not exposed, but a future consumer with |
| 1357 | // different normalisation would need this reasoning redone rather than |
| 1358 | // inherited. |
| 1359 | // |
| 1360 | // No guard is needed for the empty segment ("/", "//", ""): validPathPrefix |
| 1361 | // refuses an empty prefix, so no row can be keyed on one and both lookups |
| 1362 | // below simply miss, falling through to the host's own route. |
| 1363 | seg := firstSegmentClean(urlPath) |
| 1364 | raw := firstSegment(urlPath) |
| 1365 | target, ok := up.paths[seg] |
| 1366 | if !ok && raw != seg { |
| 1367 | target, ok = up.paths[raw] |
| 1368 | } |
| 1369 | if !ok { |
| 1370 | return up, true |
| 1371 | } |
| 1372 | // m[target], not lookupIn(m, target): the TARGET is resolved exactly, |
| 1373 | // while the host half above also accepts a one-label wildcard key. The |
| 1374 | // asymmetry is deliberate and it is documented rather than removed — |
| 1375 | // route_paths targets are hostnames the reconcile pass verified against a |
| 1376 | // real local route (aliases.go), and a real route is always an exact key, |
| 1377 | // so a target reachable ONLY through "*.parent" cannot be produced by any |
| 1378 | // supported writer. If one ever were, it would miss here and fall through |
| 1379 | // to the host's own route, which is this function's safe direction |
| 1380 | // anyway. |
| 1381 | tu, ok := m[target] |
| 1382 | if !ok { |
| 1383 | return up, true |
| 1384 | } |
| 1385 | // ONE HOP, deliberately: if tu itself carries a prefix map, it is NOT |
| 1386 | // re-resolved against this request's path. A prefix selects a route, and |
| 1387 | // the selected route's own map belongs to ITS hostname's traffic, not to |
| 1388 | // this one's. Do not add recursion here — chaining buys nothing the |
| 1389 | // publisher cannot express by pointing the prefix at the final target |
| 1390 | // directly, and it would introduce a cycle to detect. |
| 1391 | return tu, true |
| 1392 | } |
| 1393 | |
| 1394 | // firstSegment returns the first path segment of an absolute URL path, |
| 1395 | // without slashes, splitting the path EXACTLY as it arrived: |
| 1396 | // "/paulca/awesomeconf" -> "paulca". "" for "/", for the empty string, and |
| 1397 | // for anything not starting with "/". |
| 1398 | // |
| 1399 | // Returning the segment rather than testing a string prefix is what makes |
| 1400 | // matching segment-bounded: "/paulcarson" yields "paulcarson", which is not |
| 1401 | // equal to "paulca", so a longer account name can never be dragged across a |
| 1402 | // migration by a shorter one. |
| 1403 | // |
| 1404 | // This is the RAW half of the matcher. See lookupUpstreamPath's match block |
| 1405 | // for why both halves exist and why neither is safe alone. |
| 1406 | func firstSegment(p string) string { |
| 1407 | if p == "" || p[0] != '/' { |
| 1408 | return "" |
| 1409 | } |
| 1410 | p = p[1:] |
| 1411 | if i := strings.IndexByte(p, '/'); i >= 0 { |
| 1412 | p = p[:i] |
| 1413 | } |
| 1414 | return p |
| 1415 | } |
| 1416 | |
| 1417 | // firstSegmentClean is firstSegment over the path.Clean'd path, so leading |
| 1418 | // empty segments and dot segments resolve before the split: |
| 1419 | // |
| 1420 | // "//paulca/foo" -> "paulca" (leading empty segments collapse) |
| 1421 | // "/./paulca" -> "paulca" (dot segment collapses) |
| 1422 | // "/paulca/../bob" -> "bob" (".." resolves) |
| 1423 | // "/paulca/.." -> "" (resolves to "/", the host's own root) |
| 1424 | // "/paulca/" -> "paulca" (a trailing slash is not a second segment) |
| 1425 | // |
| 1426 | // This is the CLEANED half of the matcher; the raw half is firstSegment. |
| 1427 | func firstSegmentClean(p string) string { |
| 1428 | if p == "" || p[0] != '/' { |
| 1429 | return "" |
| 1430 | } |
| 1431 | // Clean of a rooted path is rooted, and is "/" when everything resolves |
| 1432 | // away — so the slice below is always safe and yields "" for the root. |
| 1433 | return firstSegment(path.Clean(p)) |
| 1434 | } |
| 1435 | |
| 1436 | func (e *Edge) ServeHTTP(w http.ResponseWriter, r *http.Request) { |
| 1437 | host := r.Host |
| 1438 | if h, _, err := net.SplitHostPort(host); err == nil { |
| 1439 | host = h |
| 1440 | } |
| 1441 | // GeoIP step 1 (geoip spec §8.1): capture and strip every geo header |
| 1442 | // before ANY branch. Nothing trusts the captured values until step 3, |
| 1443 | // after acceptHop and the tunnel mark have said whether the transport is |
| 1444 | // one we believe. hopPresent is read here because acceptHop deletes the |
| 1445 | // hop header: a request that ARRIVED claiming to be a hop and was not |
| 1446 | // verified must get no geo at all (its RemoteAddr is the sending edge). |
| 1447 | capturedGeo := geoip.Capture(r.Header) |
| 1448 | hopPresent := r.Header.Get(hopHeader) != "" |
| 1449 | // The edge-local door's mark (door.go), read before anything answers. A |
| 1450 | // door caller never reaches the steering probe, the tunnel hub or the |
| 1451 | // store door: those names are unknown hosts to it in any spelling, and |
| 1452 | // the two branches below are skipped for it outright. Only serveDoor on |
| 1453 | // this box sets the mark; nothing carries it across a hop. |
| 1454 | door, isDoor := doorCallerFrom(r.Context()) |
| 1455 | if isDoor && e.doorReservedHost(host) { |
| 1456 | e.unknownHost(w, r) |
| 1457 | return |
| 1458 | } |
| 1459 | // The deployment's steering health check (steeringprobe.go), answered |
| 1460 | // FIRST — ahead of the tunnel hub, the hop, routing and, decisively, the |
| 1461 | // unknown-host fallback. A -fallback edge redirects an unknown host with |
| 1462 | // a 302, which Route53 scores as healthy: reached in that order, the one |
| 1463 | // mechanism that withdraws a dead edge would report every edge alive |
| 1464 | // forever. Inert (one comparison against "") unless this deployment |
| 1465 | // steers. |
| 1466 | if !isDoor && e.steeringProbe(w, r, host) { |
| 1467 | return |
| 1468 | } |
| 1469 | // The reserved tunnel hostname is answered by the hub itself — before the |
| 1470 | // route lookup, so registration cannot be intercepted by a route, and |
| 1471 | // after the host:port split, so a dialer sending "tunnel.host:443" still |
| 1472 | // lands here. It wins over a registry route of the same name: the operator |
| 1473 | // owns both names, and a shadowed route is visible (its host stops |
| 1474 | // answering) where a shadowed hub would be a locked-out fleet. |
| 1475 | // |
| 1476 | // Being ahead of the cache path is load-bearing too, not just tidy: the |
| 1477 | // hub ends its handshake in a hijack, so it must be handed the REAL |
| 1478 | // ResponseWriter, never a recordingResponseWriter wrapping it. |
| 1479 | if !isDoor && e.tunnels != nil && host == e.tunnels.Host() { |
| 1480 | // The fleet store door (storedoor.go) shares this hostname, under |
| 1481 | // /store/ — the same host gate as registration, which is the whole |
| 1482 | // reason it is reachable at all: it is not a registry route, so no |
| 1483 | // other host can ever reach it. /register and everything else stay |
| 1484 | // the hub's. Ahead of the hub rather than behind it because the hub |
| 1485 | // 404s any path that is not /register. |
| 1486 | if e.door != nil && strings.HasPrefix(r.URL.Path, storeDoorPrefix) { |
| 1487 | e.door.ServeHTTP(w, r) |
| 1488 | return |
| 1489 | } |
| 1490 | e.tunnels.ServeHTTP(w, r) |
| 1491 | return |
| 1492 | } |
| 1493 | // Backhaul hop acceptance (homemap.go). Deliberately placed HERE — after |
| 1494 | // the reserved tunnel hostname's branch above, which stays byte-for-byte |
| 1495 | // what it was (that hostname is not a registry route, so it can never |
| 1496 | // appear in any box's edges file and can never be a hop's target), and |
| 1497 | // before the route lookup, so that a verified hop naming a host we do not |
| 1498 | // serve is answered by the stale-map refusal rather than the ordinary |
| 1499 | // unknown-host answer. |
| 1500 | // |
| 1501 | // On a request carrying no X-Carlos-Hop this is one Header.Get and |
| 1502 | // nothing else. |
| 1503 | r, hopped := e.acceptHop(r, host) |
| 1504 | // GeoIP steps 3-4 (geoip spec §8.1, §8.2): decide once, keep it on the |
| 1505 | // context for setGeo and analytics, and expose the app-visible headers on |
| 1506 | // r.Header so the cache's Vary matching sees exactly what the app will |
| 1507 | // get. Before every branch below: backhaul, door scope, gate, upgrade, |
| 1508 | // cache. |
| 1509 | geoAns := e.decideGeo(r, capturedGeo, hopPresent, hopped, isDoor) |
| 1510 | r = r.WithContext(context.WithValue(r.Context(), geoResultKey{}, geoAns)) |
| 1511 | geoAns.WriteApp(r.Header) |
| 1512 | // The forwarded-for proof's key (forwardproof.go), after the hop has |
| 1513 | // settled which X-Forwarded-For this edge will send on. |
| 1514 | r = e.withForwardKey(r) |
| 1515 | |
| 1516 | up, ok := e.lookupUpstreamPath(host, r.URL.Path) |
| 1517 | // A local match through a one-label "*.<parent>" key (a route |
| 1518 | // catch-all) does not beat ANOTHER box's exact host. That is the same |
| 1519 | // precedence one box keeps (lookupIn: exact first, then the star), |
| 1520 | // extended across the deployment: without it, a parent's route |
| 1521 | // catch-all on this box would swallow a name another box serves |
| 1522 | // exactly — an instance catch-all's derived name, or any alias homed |
| 1523 | // elsewhere — for every request that landed here. An arrived hop and a |
| 1524 | // door call keep the local answer: neither may start a backhaul of its |
| 1525 | // own here. |
| 1526 | if ok && !hopped && !isDoor && !e.servesExactly(host) { |
| 1527 | if eip, box, remote := e.homes.Load().ExactHome(host); remote && e.backhaulTo(w, r, host, eip, box) { |
| 1528 | return |
| 1529 | } |
| 1530 | } |
| 1531 | if !ok { |
| 1532 | if hopped { |
| 1533 | // A PROVEN hop from a sibling edge, for a host that has moved (or |
| 1534 | // never lived) here: its map is stale. 503 + Retry-After: 2, and |
| 1535 | // this return is the structural half of "one hop, ever" — the |
| 1536 | // backhaul dial below lives further down this same branch, so an |
| 1537 | // accepted hop physically cannot reach it. (e.backhaul opens with |
| 1538 | // the same refusal anyway, so moving this branch cannot open a |
| 1539 | // forwarding loop either — see its doc.) |
| 1540 | e.homes.Load().staleHopRefusal(w, host) |
| 1541 | return |
| 1542 | } |
| 1543 | // A door caller reaches a host served elsewhere only when the owner |
| 1544 | // its home box published is the caller's app, and is dialled at |
| 1545 | // exactly the box that same Owner read named, so the check and the |
| 1546 | // dial cannot disagree. A hostname split between apps publishes no |
| 1547 | // owner and is refused here. The far edge sees an ordinary hop. |
| 1548 | if isDoor { |
| 1549 | if acct, app, eip, box, ok := e.homes.Load().Owner(host); ok && acct == door.Account && app == door.App { |
| 1550 | if e.backhaulTo(w, r, host, eip, box) { |
| 1551 | return |
| 1552 | } |
| 1553 | } |
| 1554 | e.unknownHost(w, r) |
| 1555 | return |
| 1556 | } |
| 1557 | // Nothing here serves this host — but another box in the deployment |
| 1558 | // may (backhaul, below). It answers false when no box homes the host, |
| 1559 | // or when nothing could prove the hop, and today's unknown-host answer |
| 1560 | // then follows exactly as it always has. |
| 1561 | if e.backhaul(w, r, host) { |
| 1562 | return |
| 1563 | } |
| 1564 | e.unknownHost(w, r) |
| 1565 | return |
| 1566 | } |
| 1567 | |
| 1568 | // A login POST re-resolves its upstream by the path in `next`, not by |
| 1569 | // its own (spec §4.2). On a hostname that routes by path prefix, |
| 1570 | // lookupUpstreamPath keys on the first segment; "/.carlos/gate" has no |
| 1571 | // segment that can ever be a prefix, so a login POST falls through to |
| 1572 | // the FRONT host's route — the password would be judged by the wrong |
| 1573 | // row (or proxied to an app that never asked for it, which the |
| 1574 | // ungated-route branch below then refuses), and the cookie would be |
| 1575 | // signed for the wrong host. `next` is the path the visitor was refused |
| 1576 | // on, so resolving by it lands on the route that put up the page. On a |
| 1577 | // host with no prefix map it resolves to the same route either way. |
| 1578 | // |
| 1579 | // Placed AFTER the lookup above and its !ok branch, which stay |
| 1580 | // byte-for-byte what they were, because this reads the request BODY: a |
| 1581 | // host this box does not serve may be backhauled to the box that homes |
| 1582 | // it (e.backhaul, above), and that reverse-proxies this very request — |
| 1583 | // draining its body here would send the home box a ContentLength it |
| 1584 | // cannot fill, and the hop would 502. Nothing is read until the host is |
| 1585 | // known to be ours. Placed before the meter so every step below, |
| 1586 | // billing included, sees one settled up. |
| 1587 | // |
| 1588 | // A body that will not parse — over gate.MaxFormBody, or malformed — is |
| 1589 | // answered 413 here and never reaches Admit, so it spends no limiter |
| 1590 | // token and no bcrypt comparison. Refusing junk must not cost a real |
| 1591 | // visitor their attempts. |
| 1592 | loginPost := r.Method == http.MethodPost && r.URL.Path == gate.LoginPath |
| 1593 | if loginPost { |
| 1594 | r.Body = http.MaxBytesReader(w, r.Body, gate.MaxFormBody) |
| 1595 | if err := r.ParseForm(); err != nil { |
| 1596 | http.Error(w, http.StatusText(http.StatusRequestEntityTooLarge), http.StatusRequestEntityTooLarge) |
| 1597 | return |
| 1598 | } |
| 1599 | // Resolve on the PATH of `next`, never on the raw string. The 401 |
| 1600 | // page's next is SafeNext(r.URL.RequestURI()), so it carries the |
| 1601 | // query the visitor was refused on and it is percent-ENCODED — |
| 1602 | // while the prefix map is keyed on decoded segments, the way |
| 1603 | // r.URL.Path arrives for an ordinary request. Splitting the raw |
| 1604 | // string makes firstSegment("/paulca?tab=1") "paulca?tab=1", which |
| 1605 | // matches nothing and drops the login onto the FRONT route: the |
| 1606 | // wrong hash judges the password and the cookie is signed for the |
| 1607 | // wrong host. Parsing first drops the query and the fragment and |
| 1608 | // decodes the segments, so both shapes land where the refusal did. |
| 1609 | // |
| 1610 | // A miss cannot happen — SafeNext yields a path on this host, and |
| 1611 | // the worst case "/" resolves to the host's own route, the one |
| 1612 | // already in up — but keeping the first up is the honest answer if |
| 1613 | // the table changed underneath. |
| 1614 | next := gate.SafeNext(r.PostForm.Get("next")) |
| 1615 | lookupPath := "/" |
| 1616 | if u, err := url.ParseRequestURI(next); err == nil && u.Path != "" { |
| 1617 | lookupPath = u.Path |
| 1618 | } |
| 1619 | if target, hit := e.lookupUpstreamPath(host, lookupPath); hit { |
| 1620 | up = target |
| 1621 | } |
| 1622 | } |
| 1623 | |
| 1624 | // The door's scope, checked against the FINAL upstream: after the login |
| 1625 | // re-resolution above, so a login POST whose next names another app's |
| 1626 | // prefix is refused before the gate judges its password, and before the |
| 1627 | // meter, the gate, the cache and the dispatch below, which all use this |
| 1628 | // same up. |
| 1629 | if isDoor && !doorOwnsRoute(door, up.route) { |
| 1630 | e.unknownHost(w, r) |
| 1631 | return |
| 1632 | } |
| 1633 | // The import gate (importgate.go): after the door's scope, before the |
| 1634 | // meter, the password gate, the cache and the activator. |
| 1635 | if !e.importGateAdmit(w, r, host, up.route) { |
| 1636 | return |
| 1637 | } |
| 1638 | |
| 1639 | // Observe every route this box serves except the platform's own status |
| 1640 | // host (not billable, and not a member's app) — wrapping w here, once, |
| 1641 | // before the upgrade/cache branching below, is what makes all three |
| 1642 | // dispatch shapes (hibernating, cacheable, plain) plus the upgrade path |
| 1643 | // pass through this single site: the defer fires after whichever branch |
| 1644 | // actually served the request. Deliberately placed after the |
| 1645 | // tunnel-hostname and store-door returns above (edge.go:531-544) — that |
| 1646 | // hostname isn't a registry route, so up.route isn't defined yet there, |
| 1647 | // and neither one is billable traffic. |
| 1648 | // |
| 1649 | // The observing writer is installed UNCONDITIONALLY now (spec |
| 1650 | // 2026-09-07 §3.3) — the usage meter is one of its two consumers, the |
| 1651 | // errors sink the other — because a box that meters nothing still has |
| 1652 | // errors to file. Meter and sink are each nil-safe, so a pure-proxy |
| 1653 | // `carlos edge` pays one wrapper and no more. |
| 1654 | // |
| 1655 | // hopped is the ATTRIBUTION half of the backhaul egress ruling |
| 1656 | // (2026-08-24): a request that arrived over a verified hop is metered |
| 1657 | // here exactly once, as it always was, and additionally annotated into |
| 1658 | // the row's subset hop counters — so every backhauled byte is attributed |
| 1659 | // to the route, account and app that caused it, on the box that holds |
| 1660 | // the authoritative row. The sending edge stays meterless by design: it |
| 1661 | // has no route row for the host and so no identity to bill. Nothing |
| 1662 | // about the wrapper stack moves for this — recorder above the |
| 1663 | // compressor, this counter below it — because the annotation changes |
| 1664 | // WHAT is written at drain, never WHERE the bytes are counted. |
| 1665 | var bytes int64 |
| 1666 | ob := &countingWriter{ResponseWriter: w, n: &bytes} |
| 1667 | w = ob |
| 1668 | if up.route.Kind != "status" { |
| 1669 | // The outcome rides the request so the two things only the inner |
| 1670 | // layers know reach this defer: the activator marks the wake-path |
| 1671 | // 503s it writes itself (MarkAnswered, answered.go) — already the |
| 1672 | // member's platform event, so not filed again (spec §3.2, "exactly |
| 1673 | // once") — and a proxy ErrorHandler notes WHY it answered 502/503 |
| 1674 | // (noteCause), which is what groups Availability by cause. |
| 1675 | var out *outcome |
| 1676 | r, out = withOutcome(r) |
| 1677 | defer func() { |
| 1678 | if m := e.usageMeter.Load(); m != nil { |
| 1679 | m.Record(up.route, 1, bytes, hopped) |
| 1680 | } |
| 1681 | if ob.status >= 500 && !out.answered { |
| 1682 | e.emitEdgeError(up.route, up.viaTunnel, r, ob.status, out.cause) |
| 1683 | } |
| 1684 | e.observeAnalytics(up.route, r, ob, bytes) |
| 1685 | }() |
| 1686 | } |
| 1687 | |
| 1688 | // The password gate (internal/gate; spec 2026-09-07-password-gate-design |
| 1689 | // §4.1). Placed HERE and the neighbours are load-bearing: after the |
| 1690 | // meter, so the login page's bytes bill to the route that caused them; |
| 1691 | // before the upgrade branch, so a WebSocket handshake is gated; before |
| 1692 | // the cache check, so a cache hit is only ever served to a request that |
| 1693 | // already passed (the LRU keys on host and path, not on cookies); and |
| 1694 | // before the activator, so a stranger — a crawler, a scanner, a guessed |
| 1695 | // hostname — never wakes a sleeping instance. Admit strips the cookie |
| 1696 | // on the way through, so the app never sees it. |
| 1697 | // |
| 1698 | // A login POST that lands on an UNGATED route is a 404 and is never |
| 1699 | // proxied (spec §4.2): the form is only ever rendered by a gated route, |
| 1700 | // so this is a stray or a probe, and forwarding a field named |
| 1701 | // "password" to an app that never asked for it is the one outcome to |
| 1702 | // rule out. Every other method leaves the path ordinary. |
| 1703 | if loginPost && up.route.Gate == "" { |
| 1704 | http.NotFound(w, r) |
| 1705 | return |
| 1706 | } |
| 1707 | if up.route.Gate != "" && !e.gate.Admit(w, r, up.route.Host, up.route.Gate) { |
| 1708 | return |
| 1709 | } |
| 1710 | |
| 1711 | // A Connection: Upgrade request (a WebSocket handshake, typically GET) |
| 1712 | // must reach the activator/proxy with the real ResponseWriter, completely |
| 1713 | // untouched by caching: the stdlib reverse proxy's upgrade path type- |
| 1714 | // asserts the ResponseWriter to http.Hijacker to take over the raw |
| 1715 | // connection, and once upgraded, the open WebSocket is also what keeps a |
| 1716 | // hibernating instance's idle clock from firing (the activator's inflight |
| 1717 | // counting — see internal/activator/activator.go's actEntry.inflight doc |
| 1718 | // comment). So this path skips caching entirely and stays byte-for-byte |
| 1719 | // identical to pre-cache behavior, rather than relying on |
| 1720 | // recordingResponseWriter's Hijack forwarding below. |
| 1721 | if isUpgradeRequest(r) { |
| 1722 | if up.hibernating { |
| 1723 | if e.act.ServeHTTP(w, r, up.route, up.proxy) { |
| 1724 | return |
| 1725 | } |
| 1726 | } |
| 1727 | up.proxy.ServeHTTP(w, r) |
| 1728 | return |
| 1729 | } |
| 1730 | |
| 1731 | // Response compression (compress.go). Placed AFTER the upgrade branch, so |
| 1732 | // an upgrading request keeps the exact writer stack it has always had — |
| 1733 | // purge_header.go's contract records what wrapping the raw ResponseWriter |
| 1734 | // on that path has already cost this package — and BEFORE the cache check, |
| 1735 | // so a cache hit compresses on replay like any other delivery. |
| 1736 | // |
| 1737 | // The recorders constructed below wrap THIS writer, so they record |
| 1738 | // identity bytes and the cache keeps storing identity bodies; the usage |
| 1739 | // meter's countingWriter is below it, so the metered count is the |
| 1740 | // compressed one. This defer is registered after the meter's, so LIFO |
| 1741 | // closes the gzip stream — trailer included — before Record reads it. |
| 1742 | if e.gzipOn { |
| 1743 | if gw := newGzipWriter(w, r); gw != nil { |
| 1744 | w = gw |
| 1745 | defer gw.finish() |
| 1746 | } |
| 1747 | } |
| 1748 | |
| 1749 | // Cache check happens before the hibernation branch below — a hit must |
| 1750 | // never reach the activator, since avoiding pointless wakes for |
| 1751 | // bot/crawler/repeat traffic is the entire point of this cache. |
| 1752 | // |
| 1753 | // Kind=="status" is exempt from the shared cache in both directions: the |
| 1754 | // status host caches for itself (status.go's per-key store cache) and its |
| 1755 | // /ledgers/ objects carry a long immutable Cache-Control so browsers and |
| 1756 | // CDNs cache them. Admitting them HERE would be a different thing |
| 1757 | // entirely — this LRU is process-global and shared across every host, and |
| 1758 | // a public ledger chain is unbounded in length, so an anonymous walker |
| 1759 | // could evict every other tenant's cached responses at will. An evicted |
| 1760 | // entry for a hibernating route means the next request wakes the instance |
| 1761 | // instead of being answered from cache, which is the invariant this cache |
| 1762 | // exists to hold: the status host caches for itself; its responses never |
| 1763 | // enter the shared response cache. |
| 1764 | // |
| 1765 | // A gated row's verify request is exempt in both directions too: it must |
| 1766 | // reach the import rather than a response cached for the legacy route, |
| 1767 | // and the import's answer must not outlive a rollback. |
| 1768 | cacheable := e.cache != nil && (r.Method == http.MethodGet || r.Method == http.MethodHead) && |
| 1769 | up.route.Kind != "status" && up.route.ImportGate == "" |
| 1770 | var key CacheKey |
| 1771 | if cacheable { |
| 1772 | key = CacheKey{Host: host, Method: r.Method, Path: r.URL.Path, RawQuery: r.URL.RawQuery} |
| 1773 | // GetFor, not get: the variant this request belongs to depends on the |
| 1774 | // response's Vary header, which doesn't exist yet — GetFor matches r |
| 1775 | // against the Vary names each stored variant recorded for itself, so |
| 1776 | // the entry it returns is one maybeStore stored under exactly these |
| 1777 | // request headers. |
| 1778 | if resp, hit := e.cache.GetFor(key, r); hit { |
| 1779 | writeCached(w, resp) |
| 1780 | return |
| 1781 | } |
| 1782 | } |
| 1783 | |
| 1784 | if up.hibernating { |
| 1785 | // Wake (or join an in-progress wake) before proxying; the activator |
| 1786 | // also stamps this request's activity for the idle clock. Note a bare |
| 1787 | // TLS handshake never wakes anything — only a proxied request does. |
| 1788 | // The response is always recorded (even when not cacheable, in which |
| 1789 | // case maybeStore is a no-op on rec.cacheable) so this branch stays a |
| 1790 | // single shape regardless of method. |
| 1791 | rec := &recordingResponseWriter{ResponseWriter: w, request: r, cacheable: cacheable, genManaged: up.genManaged, maxBody: e.cacheMaxBody()} |
| 1792 | if e.act.ServeHTTP(rec, r, up.route, up.proxy) { |
| 1793 | e.maybeStore(key, rec) |
| 1794 | return |
| 1795 | } |
| 1796 | up.proxy.ServeHTTP(rec, r) |
| 1797 | e.maybeStore(key, rec) |
| 1798 | return |
| 1799 | } |
| 1800 | |
| 1801 | if cacheable { |
| 1802 | rec := &recordingResponseWriter{ResponseWriter: w, request: r, cacheable: true, maxBody: e.cacheMaxBody()} |
| 1803 | up.proxy.ServeHTTP(rec, r) |
| 1804 | e.maybeStore(key, rec) |
| 1805 | return |
| 1806 | } |
| 1807 | up.proxy.ServeHTTP(w, r) |
| 1808 | } |
| 1809 | |
| 1810 | // isUpgradeRequest reports whether r is a protocol-upgrade request (e.g. a |
| 1811 | // WebSocket handshake) per the Connection header's token list (RFC 7230 |
| 1812 | // §6.7: comma-separated, case-insensitive — "Upgrade", "keep-alive, Upgrade", |
| 1813 | // etc. all qualify). Connection is list-valued (RFC 9110 §5.3), so this reads |
| 1814 | // every line via joinHeader rather than r.Header.Get, which returns only the |
| 1815 | // first — the same class of bug joinHeader's doc comment describes for Vary |
| 1816 | // and Cache-Control, here on the request's Connection header instead. |
| 1817 | func isUpgradeRequest(r *http.Request) bool { |
| 1818 | for _, token := range strings.Split(joinHeader(r.Header, "Connection"), ",") { |
| 1819 | if strings.EqualFold(strings.TrimSpace(token), "Upgrade") { |
| 1820 | return true |
| 1821 | } |
| 1822 | } |
| 1823 | return false |
| 1824 | } |
| 1825 | |
| 1826 | // cacheMaxBody reports the body-size ceiling recordingResponseWriter should |
| 1827 | // enforce, or 0 when caching is disabled (recordingResponseWriter is never |
| 1828 | // constructed with cacheable=true in that case, so the value is unused). |
| 1829 | func (e *Edge) cacheMaxBody() int { |
| 1830 | if e.cache == nil { |
| 1831 | return 0 |
| 1832 | } |
| 1833 | return e.cache.maxBodyBytes |
| 1834 | } |
| 1835 | |
| 1836 | // maybeStore inspects a recorded response and caches it if it qualifies: |
| 1837 | // 2xx status, a Cache-Control that names a positive shared-cache lifetime |
| 1838 | // (s-maxage, else max-age — see parseMaxAge), and none of the per-visitor |
| 1839 | // markers below. This cache is shared across every visitor of a |
| 1840 | // host and caching is unconditional per route (not opt-in per app), so each |
| 1841 | // refusal here is the only thing standing between one visitor's response and |
| 1842 | // everyone else's: |
| 1843 | // |
| 1844 | // - Cache-Control: private — the app itself saying "one visitor only". |
| 1845 | // - Set-Cookie — the response carries per-visitor state (a session id, a |
| 1846 | // CSRF token); replaying it would hand that cookie to later visitors. |
| 1847 | // As a trailer too (hasTrailerSetCookie): the cached header map replays |
| 1848 | // trailers on every hit. |
| 1849 | // - an Authorization header on the REQUEST — the response is presumed |
| 1850 | // private per RFC 9111 §3.5, unless Cache-Control explicitly says public, |
| 1851 | // which is the app asserting the response is safe to share. |
| 1852 | // - Vary: Cookie — varying by cookie would defeat correctness per-visitor. |
| 1853 | // - Vary: * — RFC 9111 §4.1 / RFC 9110 §12.5.5: the response depends on |
| 1854 | // something request headers can't capture, so it is never reusable from |
| 1855 | // a shared cache. |
| 1856 | // |
| 1857 | // The stored TTL is clamped to DefaultCacheMaxTTL so a lost purge always |
| 1858 | // self-heals within a bounded window. |
| 1859 | // |
| 1860 | // One refusal is NOT made here because it has to be made earlier, where the |
| 1861 | // route is still in hand: Kind=="status" responses are never admitted at |
| 1862 | // all. See the cacheable computation in ServeHTTP for why. |
| 1863 | func (e *Edge) maybeStore(key CacheKey, rec *recordingResponseWriter) { |
| 1864 | if e.cache == nil || !rec.cacheable || rec.truncated { |
| 1865 | return |
| 1866 | } |
| 1867 | if rec.status < 200 || rec.status >= 300 { |
| 1868 | return |
| 1869 | } |
| 1870 | cc := joinHeader(rec.Header(), "Cache-Control") |
| 1871 | ttl, ok := parseMaxAge(cc) |
| 1872 | if !ok || ttl <= 0 { |
| 1873 | return |
| 1874 | } |
| 1875 | if len(rec.Header().Values("Set-Cookie")) > 0 || hasTrailerSetCookie(rec.Header()) { |
| 1876 | return |
| 1877 | } |
| 1878 | if len(rec.request.Header.Values("Authorization")) > 0 && !hasCCDirective(cc, "public") { |
| 1879 | return |
| 1880 | } |
| 1881 | if ttl > DefaultCacheMaxTTL { |
| 1882 | ttl = DefaultCacheMaxTTL |
| 1883 | } |
| 1884 | // The refusals below read the RAW joined field value, before any |
| 1885 | // canonicalization, and compare case-insensitively: whatever normalizing |
| 1886 | // happens downstream, "Vary: *" and "Vary: cookie" must be refused in every |
| 1887 | // spelling and on every line they can arrive in. |
| 1888 | vary := joinHeader(rec.Header(), "Vary") |
| 1889 | if vary != "" { |
| 1890 | for _, h := range strings.Split(vary, ",") { |
| 1891 | h = strings.TrimSpace(h) |
| 1892 | if h == "*" || strings.EqualFold(h, "Cookie") { |
| 1893 | return |
| 1894 | } |
| 1895 | } |
| 1896 | // The same computation GetFor replays at lookup time — both call |
| 1897 | // varyValue, which canonicalizes the name list itself, so the store key |
| 1898 | // and the lookup key can never drift on either count. |
| 1899 | key.VaryValue = varyValue(rec.request, vary) |
| 1900 | } |
| 1901 | // vary is handed to Put as well as consumed here: Put canonicalizes it the |
| 1902 | // same way varyValue just did and stores it with the entry, as part of its |
| 1903 | // key, so a later GetFor can replay this exact selection against a later |
| 1904 | // request — and so a response that varies by different headers, or doesn't |
| 1905 | // vary at all, becomes a variant beside this one rather than a replacement |
| 1906 | // for it. |
| 1907 | resp := &CachedResponse{ |
| 1908 | StatusCode: rec.status, |
| 1909 | Header: rec.Header().Clone(), |
| 1910 | Body: rec.body, |
| 1911 | StoredAt: time.Now(), |
| 1912 | TTL: ttl, |
| 1913 | } |
| 1914 | // Tagged with the resolved serving route and its generation (spec |
| 1915 | // §7.2). A managed response that no generation is recorded for (it |
| 1916 | // reached the process before any generation was current) is not stored: |
| 1917 | // nothing could later prove it current. |
| 1918 | if rec.genManaged { |
| 1919 | route, n, ok := servedGen(rec.request) |
| 1920 | if !ok { |
| 1921 | return |
| 1922 | } |
| 1923 | resp.Route, resp.Gen = route, n |
| 1924 | } |
| 1925 | e.cache.Put(key, vary, resp) |
| 1926 | } |
| 1927 | |
| 1928 | // hasTrailerSetCookie reports whether h carries a Set-Cookie TRAILER. An |
| 1929 | // undeclared trailer reaches the recorder under the raw key |
| 1930 | // "Trailer:Set-Cookie" (httputil re-adds it with http.TrailerPrefix after the |
| 1931 | // body), which the Set-Cookie check above never sees, and writeCached would |
| 1932 | // replay it to every later visitor as a trailer. The key is not canonical |
| 1933 | // (the colon stops CanonicalMIMEHeaderKey), so it is matched by hand. |
| 1934 | func hasTrailerSetCookie(h http.Header) bool { |
| 1935 | for k, v := range h { |
| 1936 | if name, ok := strings.CutPrefix(k, http.TrailerPrefix); ok && strings.EqualFold(name, "Set-Cookie") && len(v) > 0 { |
| 1937 | return true |
| 1938 | } |
| 1939 | } |
| 1940 | return false |
| 1941 | } |
| 1942 | |
| 1943 | // joinHeader returns every line of one header field as a single comma-joined |
| 1944 | // value. RFC 9110 §5.3: a field may legally appear on multiple lines and that |
| 1945 | // is identical in meaning to one comma-joined line — which is exactly what |
| 1946 | // http.Header.Add produces, and exactly how layered middleware emits |
| 1947 | // list-valued fields (compression adds Vary: Accept-Encoding, the session |
| 1948 | // layer separately adds Vary: Cookie; one handler sets Cache-Control: |
| 1949 | // max-age=600 and an auth wrapper adds Cache-Control: private). |
| 1950 | // |
| 1951 | // Every Cache-Control and Vary question this file asks goes through here |
| 1952 | // because http.Header.Get returns ONLY the first line: reading either field |
| 1953 | // with Get made every later line invisible, so a second line saying "private" |
| 1954 | // or "Cookie" silently failed to refuse the response. Joining with ", " keeps |
| 1955 | // the result a well-formed field value for the comma-splitting parsers below. |
| 1956 | func joinHeader(h http.Header, name string) string { |
| 1957 | return strings.Join(h.Values(name), ", ") |
| 1958 | } |
| 1959 | |
| 1960 | // canonicalVaryNames reduces a Vary field value to the set of header names it |
| 1961 | // selects on, in one canonical form: split on commas (a Vary may be comma- |
| 1962 | // joined from several lines — see joinHeader), trimmed, empty tokens dropped, |
| 1963 | // case-canonicalized, deduplicated and sorted. |
| 1964 | // |
| 1965 | // Everything downstream — the Vary component of the cache key, the specificity |
| 1966 | // count, and the order varyValue renders values in — is derived from THIS list |
| 1967 | // rather than from the raw field value, so spellings that mean the same thing |
| 1968 | // share one cache slot instead of splitting the URL's variant allowance into |
| 1969 | // near-duplicates: "Vary: accept-encoding" and "Vary: Accept-Encoding" are one |
| 1970 | // scheme, and so are "Vary: A, B" and "Vary: B, A" (RFC 9110 §5.1: field names |
| 1971 | // are case-insensitive; §12.5.5 gives the Vary list no ordering meaning). |
| 1972 | // |
| 1973 | // A Vary of only separators ("Vary: , , ,") yields no names at all, which is |
| 1974 | // exactly a response that selects on nothing: it canonicalizes to "", takes the |
| 1975 | // no-Vary path through Put and matches(), and has specificity 0. |
| 1976 | func canonicalVaryNames(vary string) []string { |
| 1977 | var names []string |
| 1978 | for _, h := range strings.Split(vary, ",") { |
| 1979 | h = strings.TrimSpace(h) |
| 1980 | if h == "" { |
| 1981 | continue |
| 1982 | } |
| 1983 | h = textproto.CanonicalMIMEHeaderKey(h) |
| 1984 | if slices.Contains(names, h) { |
| 1985 | continue |
| 1986 | } |
| 1987 | names = append(names, h) |
| 1988 | } |
| 1989 | slices.Sort(names) |
| 1990 | return names |
| 1991 | } |
| 1992 | |
| 1993 | // canonicalVary renders canonicalVaryNames back as a field value. This is the |
| 1994 | // form stored in cacheFullKey.Vary, so key equality is spelling-independent. |
| 1995 | func canonicalVary(vary string) string { |
| 1996 | return strings.Join(canonicalVaryNames(vary), ", ") |
| 1997 | } |
| 1998 | |
| 1999 | // varyValue concatenates r's values for each header the response's Vary list |
| 2000 | // names, in canonical name order, so the same selecting headers with the same |
| 2001 | // request values always produce the same bucket (standard HTTP Vary semantics; |
| 2002 | // RFC 7234 §4.1). Both the store path (maybeStore) and the lookup path |
| 2003 | // (cacheEntry.matches) render through this one function, so the two can never |
| 2004 | // disagree about either the canonicalization or the rendering. |
| 2005 | // |
| 2006 | // Each value is LENGTH-PREFIXED — "Name=<len>:<value>;" — which is what makes |
| 2007 | // the rendering injective for a given name list. Without the prefix the |
| 2008 | // separators were forgeable from inside a header value: under Vary: A, B a |
| 2009 | // request sending A: "u;B=v", B: "w" rendered "A=u;B=v;B=w;", byte-identical to |
| 2010 | // A: "u", B: "v;B=w" — two different requests sharing one cache slot, so |
| 2011 | // whichever stored first had its body served to the other. A length prefix |
| 2012 | // cannot be forged from within the value it measures: the reader takes exactly |
| 2013 | // len bytes, so no content can be mistaken for structure. |
| 2014 | // |
| 2015 | // The request side reads all of a header's lines (joinHeader), so two requests |
| 2016 | // that differ only in how they split a varied-on header still land in the same |
| 2017 | // bucket, which is what RFC 9110 §5.3 says they mean. |
| 2018 | func varyValue(r *http.Request, vary string) string { |
| 2019 | var b strings.Builder |
| 2020 | for _, h := range canonicalVaryNames(vary) { |
| 2021 | v := joinHeader(r.Header, h) |
| 2022 | b.WriteString(h) |
| 2023 | b.WriteByte('=') |
| 2024 | b.WriteString(strconv.Itoa(len(v))) |
| 2025 | b.WriteByte(':') |
| 2026 | b.WriteString(v) |
| 2027 | b.WriteByte(';') |
| 2028 | } |
| 2029 | return b.String() |
| 2030 | } |
| 2031 | |
| 2032 | // parseMaxAge extracts the freshness lifetime a SHARED cache should use from a |
| 2033 | // Cache-Control header. Returns ok=false for "no-store", "no-cache", "private", |
| 2034 | // or a missing/malformed lifetime — any of which mean "don't cache this in a |
| 2035 | // shared cache." |
| 2036 | // |
| 2037 | // s-maxage wins over max-age when both are present (RFC 9111 §5.2.2.10: it |
| 2038 | // overrides max-age for a shared cache, and a private cache ignores it). That |
| 2039 | // is the only directive pair an origin has for telling the two caches apart, |
| 2040 | // and it is what makes a purge effective end to end: "s-maxage=120, max-age=0" |
| 2041 | // asks the edge to hold the response for two minutes and every browser to |
| 2042 | // revalidate, so a purge here is not left with a client-side tail it cannot |
| 2043 | // reach. Reading only max-age left an origin one number for both caches — a |
| 2044 | // long one bought edge caching at the cost of stale browsers after a purge, a |
| 2045 | // short one freshened browsers by giving up the traffic the cache exists to |
| 2046 | // absorb, and the correct pair above read as max-age=0 and turned edge caching |
| 2047 | // off entirely (issue #403; tito PRO-4758/PRO-4759). |
| 2048 | // |
| 2049 | // max-age alone still means exactly what it did. Nothing can have depended on |
| 2050 | // s-maxage being ignored, because ignoring it is what made it unusable. |
| 2051 | // |
| 2052 | // The whole header is scanned before any lifetime is honored, so a |
| 2053 | // "max-age=600, private" (directives in either order, and "private" in its |
| 2054 | // field-name form private="Set-Cookie") is refused rather than cached because |
| 2055 | // max-age happened to come first — and, for the same reason, the two spellings |
| 2056 | // "s-maxage=120, max-age=0" and "max-age=0, s-maxage=120" mean the same thing. |
| 2057 | // A malformed argument on EITHER directive refuses the response whichever order |
| 2058 | // it arrives in, rather than falling back to the other one: an origin that |
| 2059 | // cannot state its lifetime is not asking for a guess. |
| 2060 | // |
| 2061 | // s-maxage is deliberately NOT read as an override of the Authorization |
| 2062 | // refusal in maybeStore, though RFC 9111 §3.5 permits that: making a response |
| 2063 | // to an authenticated request shareable stays an explicit "public", because |
| 2064 | // declining to cache is the safe direction and the origin already has the word |
| 2065 | // for it. |
| 2066 | func parseMaxAge(cc string) (time.Duration, bool) { |
| 2067 | var ttl, shared time.Duration |
| 2068 | found, sharedFound := false, false |
| 2069 | for name, arg := range ccDirectives(cc) { |
| 2070 | switch name { |
| 2071 | case "no-store", "no-cache", "private": |
| 2072 | return 0, false |
| 2073 | case "max-age": |
| 2074 | secs, err := strconv.Atoi(strings.TrimSpace(arg)) |
| 2075 | if err != nil { |
| 2076 | return 0, false |
| 2077 | } |
| 2078 | ttl, found = time.Duration(secs)*time.Second, true |
| 2079 | case "s-maxage": |
| 2080 | secs, err := strconv.Atoi(strings.TrimSpace(arg)) |
| 2081 | if err != nil { |
| 2082 | return 0, false |
| 2083 | } |
| 2084 | shared, sharedFound = time.Duration(secs)*time.Second, true |
| 2085 | } |
| 2086 | } |
| 2087 | if sharedFound { |
| 2088 | return shared, true |
| 2089 | } |
| 2090 | if !found { |
| 2091 | return 0, false |
| 2092 | } |
| 2093 | return ttl, true |
| 2094 | } |
| 2095 | |
| 2096 | // hasCCDirective reports whether cc carries the named directive. |
| 2097 | func hasCCDirective(cc, want string) bool { |
| 2098 | for name := range ccDirectives(cc) { |
| 2099 | if name == want { |
| 2100 | return true |
| 2101 | } |
| 2102 | } |
| 2103 | return false |
| 2104 | } |
| 2105 | |
| 2106 | // ccDirectives iterates a Cache-Control header as (lowercased name, argument) |
| 2107 | // pairs; the argument is "" for a bare directive. One parse shape for every |
| 2108 | // directive question this file asks. |
| 2109 | func ccDirectives(cc string) func(func(string, string) bool) { |
| 2110 | return func(yield func(string, string) bool) { |
| 2111 | for _, part := range strings.Split(cc, ",") { |
| 2112 | name, arg, _ := strings.Cut(strings.TrimSpace(part), "=") |
| 2113 | if !yield(strings.ToLower(strings.TrimSpace(name)), arg) { |
| 2114 | return |
| 2115 | } |
| 2116 | } |
| 2117 | } |
| 2118 | } |
| 2119 | |
| 2120 | // writeCached replays a cached response verbatim onto w. |
| 2121 | func writeCached(w http.ResponseWriter, resp *CachedResponse) { |
| 2122 | h := w.Header() |
| 2123 | for k, v := range resp.Header { |
| 2124 | h[k] = v |
| 2125 | } |
| 2126 | w.WriteHeader(resp.StatusCode) |
| 2127 | w.Write(resp.Body) |
| 2128 | } |
| 2129 | |
| 2130 | // recordingResponseWriter captures a response as it's written so it can be |
| 2131 | // cached afterward, while still passing bytes through to the real client |
| 2132 | // immediately (no buffering delay for the requester). Recording stops |
| 2133 | // (silently, not an error) once the body exceeds maxBody — that response is |
| 2134 | // simply never cached, per maybeStore's rec.truncated check. |
| 2135 | type recordingResponseWriter struct { |
| 2136 | http.ResponseWriter |
| 2137 | request *http.Request |
| 2138 | cacheable bool |
| 2139 | // genManaged: the response came through appSocketProxy, so maybeStore |
| 2140 | // stores it only tagged with the generation that served it. |
| 2141 | genManaged bool |
| 2142 | maxBody int |
| 2143 | status int |
| 2144 | body []byte |
| 2145 | truncated bool |
| 2146 | wroteHdr bool |
| 2147 | } |
| 2148 | |
| 2149 | func (r *recordingResponseWriter) WriteHeader(status int) { |
| 2150 | // 1xx responses (103 Early Hints, forwarded by httputil.ReverseProxy) |
| 2151 | // precede the final status; latching one would record it as THE status |
| 2152 | // and maybeStore's 2xx check would then refuse the real response that |
| 2153 | // follows. Pass them through and keep waiting for the final one. |
| 2154 | if status < 200 { |
| 2155 | r.ResponseWriter.WriteHeader(status) |
| 2156 | return |
| 2157 | } |
| 2158 | if !r.wroteHdr { |
| 2159 | r.status = status |
| 2160 | r.wroteHdr = true |
| 2161 | } |
| 2162 | r.ResponseWriter.WriteHeader(status) |
| 2163 | } |
| 2164 | |
| 2165 | func (r *recordingResponseWriter) Write(p []byte) (int, error) { |
| 2166 | if !r.wroteHdr { |
| 2167 | r.WriteHeader(http.StatusOK) |
| 2168 | } |
| 2169 | if r.cacheable && !r.truncated { |
| 2170 | if len(r.body)+len(p) > r.maxBody { |
| 2171 | r.truncated = true |
| 2172 | r.body = nil |
| 2173 | } else { |
| 2174 | r.body = append(r.body, p...) |
| 2175 | } |
| 2176 | } |
| 2177 | return r.ResponseWriter.Write(p) |
| 2178 | } |
| 2179 | |
| 2180 | // Hijack forwards to the underlying ResponseWriter's http.Hijacker, when it |
| 2181 | // implements one — the same optional-interface pattern the stdlib itself |
| 2182 | // uses. ServeHTTP special-cases upgrade requests before ever constructing a |
| 2183 | // recordingResponseWriter (see isUpgradeRequest), so this method is not |
| 2184 | // expected to be called on the hot path today; it exists so the wrapper type |
| 2185 | // is never itself the reason a hijack fails, matching http.ResponseWriter |
| 2186 | // wrapper conventions generally. |
| 2187 | func (r *recordingResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { |
| 2188 | hj, ok := r.ResponseWriter.(http.Hijacker) |
| 2189 | if !ok { |
| 2190 | return nil, nil, fmt.Errorf("edge: underlying ResponseWriter does not support http.Hijacker") |
| 2191 | } |
| 2192 | return hj.Hijack() |
| 2193 | } |
| 2194 | |
| 2195 | // Flush forwards to the underlying ResponseWriter's http.Flusher, when it |
| 2196 | // implements one — needed so a recorded (non-upgrade) response can still |
| 2197 | // stream incrementally to the real client, e.g. Server-Sent Events, instead |
| 2198 | // of buffering until the handler returns. |
| 2199 | func (r *recordingResponseWriter) Flush() { |
| 2200 | if f, ok := r.ResponseWriter.(http.Flusher); ok { |
| 2201 | f.Flush() |
| 2202 | } |
| 2203 | } |
| 2204 | |
| 2205 | // unknownHost answers a request for a host we don't serve: a redirect to the |
| 2206 | // fallback (302 — a typo'd host must not be cached as permanently moved), or |
| 2207 | // the classic 404 when no fallback is configured. |
| 2208 | // |
| 2209 | // A host under one of this deployment's own domains is answered with the |
| 2210 | // pending page instead (pending.go): it may be a host created a moment ago |
| 2211 | // that this edge has not heard about yet, and a dead end for it strands the |
| 2212 | // visitor where a retry would have worked. |
| 2213 | func (e *Edge) unknownHost(w http.ResponseWriter, r *http.Request) { |
| 2214 | host := r.Host |
| 2215 | if h, _, err := net.SplitHostPort(host); err == nil { |
| 2216 | host = h |
| 2217 | } |
| 2218 | if e.underPlatformDomain(host) { |
| 2219 | e.pendingHost(w, r) |
| 2220 | return |
| 2221 | } |
| 2222 | if e.fallbackURL != "" { |
| 2223 | http.Redirect(w, r, e.fallbackURL, http.StatusFound) |
| 2224 | return |
| 2225 | } |
| 2226 | http.Error(w, "unknown host", http.StatusNotFound) |
| 2227 | } |
| 2228 | |
| 2229 | // httpFallbackHandler is the non-ACME :80 handler passed to |
| 2230 | // autocert.Manager.HTTPHandler (RunWithActivatorReady): known hosts get the |
| 2231 | // classic redirect to https (GET/HEAD only, 302); unknown hosts go to the |
| 2232 | // fallback URL instead of a redirect into a doomed TLS handshake (no cert |
| 2233 | // will ever exist for them). |
| 2234 | // |
| 2235 | // This gate uses lookupUpstream (and, since the backhaul, the home map), NOT |
| 2236 | // Allowed: "does something answer this host on 443?" is a ROUTING question |
| 2237 | // (lookupUpstream's job, including its one-label wildcard-alias fallback, plus |
| 2238 | // the deployment's other edges), not an issuance question (Allowed's job, |
| 2239 | // deliberately exact-only — see its own doc comment). A wildcard-alias |
| 2240 | // host like paul.woodstar.app resolves fine on 443 via the wildcard cert, so |
| 2241 | // plain http:// must redirect it rather than 404. This is safe to widen past |
| 2242 | // Allowed's exact match because a redirect grants nothing: it's a 302 to the |
| 2243 | // SAME host on 443, never a new ACME order — the attacker-supplied-label |
| 2244 | // hazard that keeps Allowed narrow does not apply here. |
| 2245 | // |
| 2246 | // The ACME HTTP-01 challenge path is untouched by this: m.HTTPHandler (the |
| 2247 | // caller) checks the /.well-known/acme-challenge/ prefix — and runs its OWN |
| 2248 | // hostPolicy, which IS Allowed — before ever falling through to this |
| 2249 | // handler, so a challenge request never reaches lookupUpstream at all. |
| 2250 | func (e *Edge) httpFallbackHandler() http.Handler { |
| 2251 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 2252 | host := r.Host |
| 2253 | if h, _, err := net.SplitHostPort(host); err == nil { |
| 2254 | host = h |
| 2255 | } |
| 2256 | // The load balancer's health check (region HA spec 2 §6, lbhealth.go): |
| 2257 | // any host, so it comes before every host-based branch. Inert with |
| 2258 | // region HA off. |
| 2259 | if e.lbHealthAnswer(w, r) { |
| 2260 | return |
| 2261 | } |
| 2262 | // The steering hostname answers here too (ServeHTTP's own first |
| 2263 | // branch, and the same reasoning): the health check is HTTPS today, |
| 2264 | // but a :80 probe — or an operator's curl — must not be handed this |
| 2265 | // edge's fallback redirect for a name that is deployment |
| 2266 | // infrastructure. |
| 2267 | if e.steeringProbe(w, r, host) { |
| 2268 | return |
| 2269 | } |
| 2270 | // The reserved tunnel hostname is answered by the hub itself on 443 |
| 2271 | // (ServeHTTP, and Allowed's same check for issuance) but is not a |
| 2272 | // registry route, so lookupUpstream alone never sees it — without this, |
| 2273 | // plain http://<tunnel-host>/ fell through to unknownHost instead of |
| 2274 | // redirecting like every other routable host. |
| 2275 | routable := e.tunnels != nil && host == e.tunnels.Host() |
| 2276 | if !routable { |
| 2277 | _, routable = e.lookupUpstream(host) |
| 2278 | } |
| 2279 | if !routable { |
| 2280 | // A host this box does not serve but another box HOMES: 443 |
| 2281 | // backhauls it (the hop, edge.go's backhaul), so :80 must send the |
| 2282 | // visitor there rather than 404 an app that is up — a first-time |
| 2283 | // visitor typing a bare hostname lands on whichever edge DNS |
| 2284 | // picked, which in a steered deployment is routinely not the home. |
| 2285 | // Home() is the same wildcard-capable lookup the dial uses, is |
| 2286 | // nil-safe (no deployment store ⇒ no answer), and excludes this |
| 2287 | // box's own hosts, so a single-box deployment is untouched. |
| 2288 | // |
| 2289 | // No hop logic here on purpose: the hop dials 443, so a verified |
| 2290 | // hop can never arrive on this handler at all. |
| 2291 | _, _, routable = e.homes.Load().Home(host) |
| 2292 | } |
| 2293 | if !routable { |
| 2294 | e.unknownHost(w, r) |
| 2295 | return |
| 2296 | } |
| 2297 | if r.Method != "GET" && r.Method != "HEAD" { |
| 2298 | http.Error(w, "Use HTTPS", http.StatusBadRequest) |
| 2299 | return |
| 2300 | } |
| 2301 | http.Redirect(w, r, "https://"+host+r.URL.RequestURI(), http.StatusFound) |
| 2302 | }) |
| 2303 | } |
| 2304 | |
| 2305 | // Watch polls the registry for writes from other processes (`carlos add` |
| 2306 | // etc., without needing rights to signal us) every 2s and reloads on change. |
| 2307 | // It polls a cheap fingerprint of the routes table rather than PRAGMA |
| 2308 | // data_version: the pragma proved blind to cross-process commits on Eleven's |
| 2309 | // production box (a modernc/sqlite WAL-visibility quirk under long-lived |
| 2310 | // connections). registry.Fingerprint opens a FRESH connection per poll — a |
| 2311 | // brand-new connection reads the latest committed state by construction |
| 2312 | // (see registry.List's comment for the stale-WAL-snapshot history). Opening |
| 2313 | // SQLite costs about a millisecond; at one poll every two seconds that is |
| 2314 | // nothing. Returns when stop closes; SIGHUP-forced reloads live in Run. |
| 2315 | func (e *Edge) Watch(stop <-chan struct{}) { |
| 2316 | log.Printf("edge: registry watcher armed (2s fingerprint poll)") |
| 2317 | // No baseline-only first tick: the first observed fingerprint also |
| 2318 | // triggers a reload, so a route added between the edge's startup reload |
| 2319 | // and this poll can't be silently baselined away. (Eleven's version has |
| 2320 | // that gap; one redundant reload at startup is the cost of closing it.) |
| 2321 | var last string |
| 2322 | errs := 0 |
| 2323 | reloadErrs := 0 |
| 2324 | tick := time.NewTicker(2 * time.Second) |
| 2325 | defer tick.Stop() |
| 2326 | for { |
| 2327 | select { |
| 2328 | case <-stop: |
| 2329 | return |
| 2330 | case <-tick.C: |
| 2331 | } |
| 2332 | fp, err := e.reg.Fingerprint() |
| 2333 | if err != nil { |
| 2334 | errs++ |
| 2335 | if errs == 1 || errs%300 == 0 { |
| 2336 | log.Printf("edge: registry-watch poll error (#%d): %v", errs, err) |
| 2337 | } |
| 2338 | continue |
| 2339 | } |
| 2340 | errs = 0 |
| 2341 | // The edge's own loop is alive, its routing table readable, and the |
| 2342 | // :443 path's shared lock free (region HA spec 2 §6, servingPathLive). |
| 2343 | // A wedged poll or a wedged request path stops beating. |
| 2344 | if e.lbHealth != nil && e.servingPathLive() { |
| 2345 | e.lbHealth.EdgeBeat(time.Now()) |
| 2346 | } |
| 2347 | if fp == last { |
| 2348 | continue |
| 2349 | } |
| 2350 | // last is baselined only on a SUCCESSFUL reload: a fleet detach is a |
| 2351 | // revocation, and if Reload fails on this tick, the fingerprint it |
| 2352 | // was meant to apply must not be discarded — baselining it here |
| 2353 | // (before Reload runs) would make the change invisible to every |
| 2354 | // later tick until some UNRELATED write moves the fingerprint again, |
| 2355 | // silently sitting on a revocation that never took effect. Leaving |
| 2356 | // last unchanged means the very next tick sees the same fp != last |
| 2357 | // and retries. |
| 2358 | if err := e.Reload(); err != nil { |
| 2359 | reloadErrs++ |
| 2360 | if reloadErrs == 1 || reloadErrs%300 == 0 { |
| 2361 | log.Printf("edge: registry-watch reload failed (#%d): %v", reloadErrs, err) |
| 2362 | } |
| 2363 | continue |
| 2364 | } |
| 2365 | reloadErrs = 0 |
| 2366 | last = fp |
| 2367 | log.Printf("edge: registry changed — reloaded (%d route(s))", len(e.current())) |
| 2368 | } |
| 2369 | } |
| 2370 | |
| 2371 | // versionHeader wraps next so every response also carries |
| 2372 | // X-Carlos-Version: ver — the deploy watch's proof (carlos-deploy spec §7) |
| 2373 | // that this route is serving the version it was told to. It wraps the |
| 2374 | // finished handler rather than reaching inside it, so it never interferes |
| 2375 | // with a ReverseProxy's ModifyResponse purge-header hook or the activator's |
| 2376 | // dispatch — both already ran (or will run inside next) by the time this |
| 2377 | // sets the header. |
| 2378 | func versionHeader(ver string, next http.Handler) http.Handler { |
| 2379 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 2380 | w.Header().Set("X-Carlos-Version", ver) |
| 2381 | next.ServeHTTP(w, r) |
| 2382 | }) |
| 2383 | } |
| 2384 | |
| 2385 | // UnixProxy is unixProxy for callers outside this package — the scheduled-work |
| 2386 | // runner in internal/host, which delivers a tick to a route's own socket and |
| 2387 | // must NOT take it through the edge's cache, compression or usage layers: a |
| 2388 | // tick is platform traffic, not a visitor's request. |
| 2389 | func UnixProxy(socketPath string) http.Handler { return unixProxy(socketPath) } |
| 2390 | |
| 2391 | // unixProxy reverse-proxies to a backend listening on a unix socket, |
| 2392 | // preserving the client's Host header (so the backend sees the real hostname) |
| 2393 | // and passing WebSocket upgrades straight through. |
| 2394 | func unixProxy(socketPath string) *httputil.ReverseProxy { |
| 2395 | dialer := &net.Dialer{} |
| 2396 | return &httputil.ReverseProxy{ |
| 2397 | Rewrite: func(pr *httputil.ProxyRequest) { |
| 2398 | setForwarded(pr) |
| 2399 | setGeo(pr, false) // geoip spec §8.3: after hop-header removal, from the request's one answer |
| 2400 | pr.Out.URL.Scheme = "http" |
| 2401 | pr.Out.URL.Host = "carlos" // placeholder; the unix dialer ignores it |
| 2402 | pr.Out.Host = pr.In.Host // backend sees the real hostname |
| 2403 | scrubInboundPurgeHeader(pr) |
| 2404 | }, |
| 2405 | Transport: &http.Transport{ |
| 2406 | DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { |
| 2407 | return dialUnixWithRetry(ctx, dialer, socketPath) |
| 2408 | }, |
| 2409 | // Bound idle keep-alives so proxies replaced on reload don't leak |
| 2410 | // their pooled connections + reader goroutines on the long-running |
| 2411 | // edge. |
| 2412 | MaxIdleConns: 32, |
| 2413 | IdleConnTimeout: 90 * time.Second, |
| 2414 | }, |
| 2415 | ErrorHandler: defaultProxyError(proxyCause), |
| 2416 | } |
| 2417 | } |
| 2418 | |
| 2419 | // tcpProxy reverse-proxies to a TCP upstream at http://addr with the same |
| 2420 | // Host preservation and idle bounds as unixProxy — this is what lets |
| 2421 | // whole-app tenants sit behind the edge without unix-socket support. |
| 2422 | func tcpProxy(addr string) *httputil.ReverseProxy { |
| 2423 | return &httputil.ReverseProxy{ |
| 2424 | Rewrite: func(pr *httputil.ProxyRequest) { |
| 2425 | setForwarded(pr) |
| 2426 | setGeo(pr, false) // geoip spec §8.3: after hop-header removal, from the request's one answer |
| 2427 | pr.Out.URL.Scheme = "http" |
| 2428 | pr.Out.URL.Host = addr |
| 2429 | pr.Out.Host = pr.In.Host // backend sees the real hostname |
| 2430 | scrubInboundPurgeHeader(pr) |
| 2431 | }, |
| 2432 | Transport: &http.Transport{ |
| 2433 | MaxIdleConns: 32, |
| 2434 | IdleConnTimeout: 90 * time.Second, |
| 2435 | }, |
| 2436 | ErrorHandler: defaultProxyError(proxyCause), |
| 2437 | } |
| 2438 | } |
| 2439 | |
| 2440 | // errNoLiveUpstream is the sentinel multiProxy's dialer returns when a probed |
| 2441 | // route has nothing left to try — an empty live set, or every live upstream |
| 2442 | // refusing the dial inside this one request. multiProxy's ErrorHandler turns |
| 2443 | // it into 503 + Retry-After: 2, the same "nobody is home right now, come back |
| 2444 | // shortly" answer errFleetDark gets, rather than the ReverseProxy default's |
| 2445 | // 502 (which reads as "your backend is broken" and is what an UNPROBED addr |
| 2446 | // route still correctly returns). |
| 2447 | // |
| 2448 | // It never escapes the route it came from: each multiProxy has its own |
| 2449 | // ErrorHandler, and the only thing that can produce this error is that |
| 2450 | // proxy's own DialContext. |
| 2451 | var errNoLiveUpstream = errors.New("edge: no live upstream") |
| 2452 | |
| 2453 | // multiDialHost is the placeholder authority multiProxy's Rewrite puts on the |
| 2454 | // outbound URL. Nothing ever resolves it: DialContext ignores the address |
| 2455 | // entirely and dials the live set instead. `.invalid` is the reserved TLD for |
| 2456 | // exactly this (RFC 2606), so if it ever DID reach a resolver — a wiring |
| 2457 | // mistake — it fails immediately and visibly rather than hitting somebody's |
| 2458 | // wildcard DNS. |
| 2459 | const multiDialHost = "multi.invalid" |
| 2460 | |
| 2461 | // multiDialTimeout bounds ONE dial attempt. Per attempt, not per request: a |
| 2462 | // route with three upstreams whose first two are black-holed must still reach |
| 2463 | // the third, and a single request-wide budget would spend it all on the first |
| 2464 | // dead box. Three seconds is long enough for a loaded box on a real network to |
| 2465 | // complete a handshake and short enough that a fully dark route answers its |
| 2466 | // 503 well inside any sane client timeout. |
| 2467 | const multiDialTimeout = 3 * time.Second |
| 2468 | |
| 2469 | // multiProxy reverse-proxies a probed addr route across its live upstreams |
| 2470 | // (prober.go). Same Host preservation and purge-header discipline as tcpProxy; |
| 2471 | // what it adds is where the bytes go. |
| 2472 | // |
| 2473 | // It takes the host and not the route's addr list because the addr list is not |
| 2474 | // its input: the prober owns it (Reload hands the same list to Sync), and the |
| 2475 | // only set this proxy may dial is the LIVE one. Closing over the configured |
| 2476 | // addrs as well would create a second, staler copy of the route's upstreams |
| 2477 | // and invite exactly the fallback the design forbids — dialing a box the probe |
| 2478 | // has already declared dead. |
| 2479 | // |
| 2480 | // Selection happens in DialContext, not in Rewrite, and that is the whole |
| 2481 | // design. A dial is the last moment at which NOTHING has been sent: the |
| 2482 | // request body is untouched and no response has begun, so a refused upstream |
| 2483 | // can be ejected and the next one tried inside the same request, with the |
| 2484 | // client none the wiser. Retrying one layer up (at RoundTrip) would mean |
| 2485 | // replaying a request that may already have had its body consumed, which is |
| 2486 | // only safe for a subset of requests — this way it is safe for all of them. |
| 2487 | // |
| 2488 | // The live set is read PER REQUEST rather than captured, mirroring |
| 2489 | // fleetProxy/tunnelProxy: an upstream that fails away (or rejoins) between two |
| 2490 | // requests is picked up with no Reload needed. |
| 2491 | // |
| 2492 | // Keep-alives to upstreams are off, which is a real cost paid for two real |
| 2493 | // properties. The stdlib pools connections by the outbound URL's authority, |
| 2494 | // and every request through this proxy carries the same placeholder one — so a |
| 2495 | // pooled connection would be reused regardless of WHICH upstream it goes to. |
| 2496 | // That would (a) pin a route to whichever upstream happened to answer first, |
| 2497 | // making the round-robin below decorative, and (b) survive an Eject: the |
| 2498 | // upstream the prober just took out of rotation would keep serving traffic |
| 2499 | // over the connection already in the pool. Correct failover is worth a TCP |
| 2500 | // handshake per request to a backend that is, in every deployment this exists |
| 2501 | // for, on the same network. The unprobed tcpProxy path keeps its pool |
| 2502 | // untouched. |
| 2503 | func multiProxy(host string, p *proberSet) *httputil.ReverseProxy { |
| 2504 | // Per-ROUTE cursor: closed over by this proxy alone, so two routes cannot |
| 2505 | // interleave their rotations (a global counter would make a two-upstream |
| 2506 | // route's balance depend on the request rate of every other route on the |
| 2507 | // box). Reload builds a new proxy, so the cursor resets with the route's |
| 2508 | // upstream set — which is what you want, since the set it was counting |
| 2509 | // over no longer exists. |
| 2510 | var cursor atomic.Uint64 |
| 2511 | dialer := &net.Dialer{Timeout: multiDialTimeout} |
| 2512 | return &httputil.ReverseProxy{ |
| 2513 | Rewrite: func(pr *httputil.ProxyRequest) { |
| 2514 | setForwarded(pr) |
| 2515 | setGeo(pr, false) // geoip spec §8.3: after hop-header removal, from the request's one answer |
| 2516 | pr.Out.URL.Scheme = "http" |
| 2517 | pr.Out.URL.Host = multiDialHost // placeholder; the dialer below ignores it |
| 2518 | pr.Out.Host = pr.In.Host // backend sees the real hostname |
| 2519 | scrubInboundPurgeHeader(pr) |
| 2520 | }, |
| 2521 | Transport: &http.Transport{ |
| 2522 | DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) { |
| 2523 | live := p.Live(host) |
| 2524 | if len(live) == 0 { |
| 2525 | return nil, errNoLiveUpstream |
| 2526 | } |
| 2527 | // Start where the last request left off, then walk the whole |
| 2528 | // snapshot once. Snapshot rather than re-reading Live per |
| 2529 | // attempt: each Eject below shrinks the live set, and re-reading |
| 2530 | // would skip untried upstreams as the list moved under us. |
| 2531 | // |
| 2532 | // The index arithmetic stays in uint64 all the way to the |
| 2533 | // modulo. Converting the counter to int first would wrap |
| 2534 | // NEGATIVE once it passed 2^31 on a 32-bit box (the Pi fleet), |
| 2535 | // and Go's modulo keeps the sign — so the very next request |
| 2536 | // after that wrap would index out of range and panic the edge. |
| 2537 | start := cursor.Add(1) - 1 |
| 2538 | n := uint64(len(live)) |
| 2539 | var lastErr error |
| 2540 | for i := range live { |
| 2541 | if err := ctx.Err(); err != nil { |
| 2542 | // The client went away (or the request deadline |
| 2543 | // expired) mid-loop. Stop, and do NOT eject: a dial we |
| 2544 | // abandoned says nothing about the upstream's health, |
| 2545 | // and marking it down here would take a healthy box out |
| 2546 | // of rotation because one user hit stop. |
| 2547 | return nil, err |
| 2548 | } |
| 2549 | addr := live[(start+uint64(i))%n] |
| 2550 | conn, err := dialer.DialContext(ctx, network, addr) |
| 2551 | if err == nil { |
| 2552 | return conn, nil |
| 2553 | } |
| 2554 | lastErr = err |
| 2555 | if ctx.Err() != nil { |
| 2556 | return nil, err // as above: our own cancellation, not the upstream's fault |
| 2557 | } |
| 2558 | // The fast path the probe cadence cannot match: a box that |
| 2559 | // died since the last tick is out of rotation now, for |
| 2560 | // every other in-flight request too, instead of collecting |
| 2561 | // failures until fail_after. |
| 2562 | p.Eject(host, addr) |
| 2563 | } |
| 2564 | return nil, fmt.Errorf("%w for %s: %d upstream(s) refused, last: %v", |
| 2565 | errNoLiveUpstream, host, len(live), lastErr) |
| 2566 | }, |
| 2567 | // No MaxIdleConns/IdleConnTimeout twin of tcpProxy's: with |
| 2568 | // keep-alives off nothing is ever pooled, so the idle bounds that |
| 2569 | // stop a replaced proxy leaking pooled connections have nothing to |
| 2570 | // bound. Every connection this transport opens belongs to exactly |
| 2571 | // one request and closes with it. |
| 2572 | DisableKeepAlives: true, // see the pooling note above |
| 2573 | }, |
| 2574 | ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { |
| 2575 | if errors.Is(err, errNoLiveUpstream) { |
| 2576 | noteCause(r, apperr.CauseNoUpstream) |
| 2577 | w.Header().Set("Retry-After", "2") |
| 2578 | http.Error(w, "no healthy upstream", http.StatusServiceUnavailable) |
| 2579 | return |
| 2580 | } |
| 2581 | // Anything else is a real proxy failure (upstream hung up |
| 2582 | // mid-response, malformed reply): same shape as |
| 2583 | // httputil.ReverseProxy's own default handler, log and 502. |
| 2584 | noteCause(r, proxyCause(r, err)) |
| 2585 | log.Printf("edge: multi-upstream %s proxy error: %v", host, err) |
| 2586 | w.WriteHeader(http.StatusBadGateway) |
| 2587 | }, |
| 2588 | } |
| 2589 | } |
| 2590 | |
| 2591 | // tunnelProxy reverse-proxies over a named reverse channel (TunnelHub). |
| 2592 | // Same Host preservation and purge-header discipline as tcpProxy — a |
| 2593 | // tunnel-backed host is exactly as untrusted as an addr-backed one. The |
| 2594 | // scheme is https because the h2 ClientConn requires it; no dial happens |
| 2595 | // (the channel already exists), so the value is otherwise inert. No channel |
| 2596 | // is resolved here either: hub.transportFor looks it up per request, so a |
| 2597 | // proxy built before a reconnect uses the fresh channel afterwards, and a |
| 2598 | // disconnected one fails fast into the stdlib proxy's 502 rather than hanging. |
| 2599 | func tunnelProxy(name string, hub *TunnelHub) *httputil.ReverseProxy { |
| 2600 | return &httputil.ReverseProxy{ |
| 2601 | Rewrite: func(pr *httputil.ProxyRequest) { |
| 2602 | setForwarded(pr) |
| 2603 | setGeo(pr, true) // geoip spec §8.3: after hop-header removal, from the request's one answer |
| 2604 | pr.Out.URL.Scheme = "https" |
| 2605 | pr.Out.URL.Host = pr.In.Host // h2 :authority = the real hostname |
| 2606 | pr.Out.Host = pr.In.Host |
| 2607 | scrubInboundPurgeHeader(pr) |
| 2608 | }, |
| 2609 | Transport: hub.transportFor(name), |
| 2610 | ErrorHandler: defaultProxyError(tunnelCause), |
| 2611 | } |
| 2612 | } |
| 2613 | |
| 2614 | // errFleetDark is the sentinel fleetProxy's Transport returns when fleet has |
| 2615 | // no connected channel at all — neither the route's bound box nor any |
| 2616 | // fallback. fleetProxy's ErrorHandler recognizes it and answers a clean 503 |
| 2617 | // with Retry-After: 2 instead of the ReverseProxy default (a 502 with a |
| 2618 | // "context deadline exceeded"-flavored body that reads like a broken backend |
| 2619 | // rather than "no box is home right now, try again shortly"). |
| 2620 | var errFleetDark = errors.New("edge: fleet has no connected channel") |
| 2621 | |
| 2622 | // fleetChannelFor picks which channel name to dial for one request against a |
| 2623 | // fleet-placed route, per the brief's binding/fallback rule: |
| 2624 | // |
| 2625 | // 1. boxLabel (the route's bound box, Route.FleetBox) if its channel |
| 2626 | // "<fleet>/<boxLabel>" is connected. |
| 2627 | // 2. else the first name in ConnectedOfFleet(fleet)'s sorted order — ANY |
| 2628 | // connected box of the fleet. That box's own activator wakes its |
| 2629 | // instance locally; a bound-but-different box racing it awake is the |
| 2630 | // lease's job to arbitrate, not this edge's. |
| 2631 | // 3. else "" — the fleet is dark. |
| 2632 | // |
| 2633 | // boxLabel may be "" (a route that wants a fleet but has no box assigned |
| 2634 | // yet), which just skips straight to the fallback step. |
| 2635 | func fleetChannelFor(fleet, boxLabel string, hub *TunnelHub) string { |
| 2636 | if boxLabel != "" { |
| 2637 | if bound := fleet + "/" + boxLabel; hub.State(bound) == "connected" { |
| 2638 | return bound |
| 2639 | } |
| 2640 | } |
| 2641 | if names := hub.ConnectedOfFleet(fleet); len(names) > 0 { |
| 2642 | return names[0] |
| 2643 | } |
| 2644 | return "" |
| 2645 | } |
| 2646 | |
| 2647 | // fleetProxy reverse-proxies a fleet-placed route (Region == "fleet/<fleet>", |
| 2648 | // registry.FleetPlacement) over whichever channel fleetChannelFor picks — |
| 2649 | // resolved PER REQUEST, mirroring tunnelProxy/transportFor above, so a bound |
| 2650 | // box reconnecting (or a fallback box's channel dropping mid-outage) between |
| 2651 | // two requests is picked up with no Reload needed. Same Host preservation |
| 2652 | // and purge-header discipline as tunnelProxy; the only difference is the |
| 2653 | // no-channel-at-all case, which answers 503 + Retry-After via ErrorHandler |
| 2654 | // instead of falling into ReverseProxy's default 502 (see errFleetDark). |
| 2655 | func fleetProxy(fleet, boxLabel string, hub *TunnelHub) *httputil.ReverseProxy { |
| 2656 | return &httputil.ReverseProxy{ |
| 2657 | Rewrite: func(pr *httputil.ProxyRequest) { |
| 2658 | setForwarded(pr) |
| 2659 | setGeo(pr, true) // geoip spec §8.3: after hop-header removal, from the request's one answer |
| 2660 | pr.Out.URL.Scheme = "https" |
| 2661 | pr.Out.URL.Host = pr.In.Host // h2 :authority = the real hostname |
| 2662 | pr.Out.Host = pr.In.Host |
| 2663 | scrubInboundPurgeHeader(pr) |
| 2664 | }, |
| 2665 | Transport: roundTripperFunc(func(req *http.Request) (*http.Response, error) { |
| 2666 | name := fleetChannelFor(fleet, boxLabel, hub) |
| 2667 | if name == "" { |
| 2668 | return nil, errFleetDark |
| 2669 | } |
| 2670 | return hub.transportFor(name).RoundTrip(req) |
| 2671 | }), |
| 2672 | ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { |
| 2673 | if errors.Is(err, errFleetDark) { |
| 2674 | noteCause(r, apperr.CauseFleetDark) |
| 2675 | w.Header().Set("Retry-After", "2") |
| 2676 | http.Error(w, "fleet unreachable: no connected box", http.StatusServiceUnavailable) |
| 2677 | return |
| 2678 | } |
| 2679 | // Any other transport failure (e.g. the picked channel died between |
| 2680 | // fleetChannelFor's check and the RoundTrip call) — same shape as |
| 2681 | // httputil.ReverseProxy's own default handler: log and 502. |
| 2682 | noteCause(r, tunnelCause(r, err)) |
| 2683 | log.Printf("edge: fleet %s proxy error: %v", fleet, err) |
| 2684 | w.WriteHeader(http.StatusBadGateway) |
| 2685 | }, |
| 2686 | } |
| 2687 | } |
| 2688 | |
| 2689 | // ---- backhaul: the SENDING half of one-hop edge-to-edge forwarding ---------- |
| 2690 | // |
| 2691 | // homemap.go holds the map (which OTHER box homes a host) and the RECEIVING |
| 2692 | // half (what proves an inbound hop, and what a stale one is answered with). |
| 2693 | // This is the other end: a request that landed on this edge for a host homed |
| 2694 | // elsewhere is forwarded to that box's edge — over ordinary, publicly-verified |
| 2695 | // TLS, exactly once — instead of dead-ending in the unknown-host answer. |
| 2696 | // |
| 2697 | // Three things are deliberately NOT done here, and each is a decision rather |
| 2698 | // than an omission: |
| 2699 | // |
| 2700 | // - NO METERING. The one usage meter (ServeHTTP, above) sits BELOW a |
| 2701 | // successful lookupUpstream, so a backhauled request misses it here by |
| 2702 | // construction while the HOME edge — which does serve the route — records |
| 2703 | // the request and its bytes exactly once. A second Record at this site |
| 2704 | // would double-count every backhauled request against the tenant. What |
| 2705 | // that leaves genuinely open is EGRESS ATTRIBUTION: the bytes cross the |
| 2706 | // internet twice and the second leg is billed to this box's account while |
| 2707 | // the meter runs on the home box's route. That is a unit-economics |
| 2708 | // question — the plan's Task 7 ruling — and not one to guess at from |
| 2709 | // inside a proxy. |
| 2710 | // |
| 2711 | // - NO CACHING AND NO COMPRESSION. This branch returns before ServeHTTP's |
| 2712 | // cache and gzip wrapping, so a backhauled response is neither stored nor |
| 2713 | // re-encoded here. That is load-bearing, not lazy: a tenant's Carlos-Purge |
| 2714 | // is honoured and stripped by the HOME edge's purgeModifier |
| 2715 | // (purge_header.go), so a copy cached on THIS box could never be |
| 2716 | // invalidated by the app that owns it. The home edge's own cache and |
| 2717 | // compression still apply and reach the client through the hop — as does |
| 2718 | // X-Carlos-Version, which the home edge stamps from the route's own row |
| 2719 | // (versionHeader is per-route, and this box has no row for the host), so |
| 2720 | // `carlos deploy`'s verification works through a backhaul with nothing |
| 2721 | // added here. |
| 2722 | // |
| 2723 | // - NO RETRY, AND NO SECOND TARGET. Home() answers with one box or none. |
| 2724 | // A failed dial is a 502 (see backhaulProxy's ErrorHandler), not a |
| 2725 | // scan for somewhere else to try: the process backing a home-boxed route |
| 2726 | // exists on exactly ONE box, so there is no second copy a retry could |
| 2727 | // reach. |
| 2728 | // |
| 2729 | // Single-box inertness: with one platform box Home() never offers a target |
| 2730 | // (own-box exclusion, homemap.go's foldHomes), so e.backhaul returns false on |
| 2731 | // every request and this whole path is dead code until a second box publishes |
| 2732 | // its edges file. |
| 2733 | |
| 2734 | // backhaulMaxInflight is the containment ruling of 2026-08-24: at most this |
| 2735 | // many backhauled requests may be in flight toward any ONE home box at a |
| 2736 | // time, counted on the SENDING edge. At the limit the sender answers 503 + |
| 2737 | // Retry-After: 2 (backhaul, below) rather than dialling. |
| 2738 | // |
| 2739 | // A concurrency bound and not a rate: cross-region egress is (concurrent |
| 2740 | // streams × per-stream bandwidth), so a request-rate cap would neither bound |
| 2741 | // a handful of long-lived streams — a hop carries WebSockets and has no |
| 2742 | // whole-body deadline by design — nor tolerate a legitimate burst. This |
| 2743 | // degrades the other way round: short requests flow at full rate forever, and |
| 2744 | // only slow, sustained streams pile into the cap. |
| 2745 | // |
| 2746 | // Sender-side because every backhauled byte traverses BOTH legs (home → |
| 2747 | // sender, sender → client), so one cap here bounds both boxes' egress at |
| 2748 | // once; the home box needs no twin, its inbound hops being bounded by the sum |
| 2749 | // of its siblings' caps. |
| 2750 | // |
| 2751 | // A var, not a const, so a test can shorten it — the same reason backhaulPort |
| 2752 | // is one. There is deliberately no "off" value: an unbounded cross-region |
| 2753 | // path is the thing this closes. |
| 2754 | // |
| 2755 | // Atomic rather than a plain int because it is READ on the request path by |
| 2756 | // every backhauling goroutine: it is written once today, before any listener |
| 2757 | // binds, but a future env re-read (or a second Edge composed in one process) |
| 2758 | // would turn a plain int into a real data race rather than a stale read. |
| 2759 | var backhaulMaxInflight atomic.Int64 |
| 2760 | |
| 2761 | const defaultBackhaulMaxInflight = 64 |
| 2762 | |
| 2763 | func init() { backhaulMaxInflight.Store(defaultBackhaulMaxInflight) } |
| 2764 | |
| 2765 | // backhaulInflightFromEnv reads CARLOS_BACKHAUL_MAX_INFLIGHT. Unset, |
| 2766 | // unparseable or ≤ 0 all mean the default: a typo'd knob must not silently |
| 2767 | // uncap (or, worse, zero-cap) a deployment's backhaul. |
| 2768 | func backhaulInflightFromEnv() int { |
| 2769 | raw := strings.TrimSpace(os.Getenv("CARLOS_BACKHAUL_MAX_INFLIGHT")) |
| 2770 | if raw == "" { |
| 2771 | return defaultBackhaulMaxInflight |
| 2772 | } |
| 2773 | n, err := strconv.Atoi(raw) |
| 2774 | if err != nil || n <= 0 { |
| 2775 | log.Printf("edge: CARLOS_BACKHAUL_MAX_INFLIGHT=%q is not a positive integer — using the default of %d concurrent backhauled requests per home box", raw, defaultBackhaulMaxInflight) |
| 2776 | return defaultBackhaulMaxInflight |
| 2777 | } |
| 2778 | return n |
| 2779 | } |
| 2780 | |
| 2781 | // backhaulAcquire reserves one in-flight slot toward box, or reports |
| 2782 | // saturation. releaseSlot must be called exactly once when the hop's |
| 2783 | // ServeHTTP returns — a streaming or upgraded response holds its slot for the |
| 2784 | // life of the stream, deliberately: a long-lived stream IS sustained |
| 2785 | // cross-region egress, which is the thing being bounded. |
| 2786 | // |
| 2787 | // The counter is per home BOX and the map is never keyed by anything a |
| 2788 | // stranger controls (backhaulInflight's own doc). The count is incremented |
| 2789 | // FIRST and rolled back on refusal, so the cap can never be exceeded: two |
| 2790 | // requests racing at the limit both see the excess and both refuse, and a |
| 2791 | // request arriving inside another's rollback window can be refused while a |
| 2792 | // slot is in fact free. Both errors are in the safe direction — a refusal is |
| 2793 | // a 503 the client retries in two seconds, where an over-admission would be |
| 2794 | // cross-region bytes the cap exists to prevent. |
| 2795 | func (e *Edge) backhaulAcquire(box string) (releaseSlot func(), ok bool) { |
| 2796 | // Load first, like backhaulTransport: LoadOrStore would allocate a |
| 2797 | // counter per backhauled request just to throw it away on the hit path. |
| 2798 | v, hit := e.backhaulInflight.Load(box) |
| 2799 | if !hit { |
| 2800 | v, _ = e.backhaulInflight.LoadOrStore(box, new(atomic.Int64)) |
| 2801 | } |
| 2802 | n := v.(*atomic.Int64) |
| 2803 | if n.Add(1) > backhaulMaxInflight.Load() { |
| 2804 | n.Add(-1) |
| 2805 | return nil, false |
| 2806 | } |
| 2807 | // Once-guarded because a slot released twice would let the cap drift |
| 2808 | // upwards for the life of the process — the one failure mode of a |
| 2809 | // hand-released semaphore, and cheaper to make impossible than to audit. |
| 2810 | var once sync.Once |
| 2811 | return func() { once.Do(func() { n.Add(-1) }) }, true |
| 2812 | } |
| 2813 | |
| 2814 | // backhaulPort is the port a home edge answers on. Always 443 in production — |
| 2815 | // an edges file publishes a bare EIP (Task 1) and every edge's front door is |
| 2816 | // 443 — and a var only so a test can point the dial at an httptest server, the |
| 2817 | // same reason homeMapInterval is one. |
| 2818 | var backhaulPort = "443" |
| 2819 | |
| 2820 | // backhaulRootCAs is the trust anchor set the hop's TLS handshake verifies the |
| 2821 | // home edge's certificate against. nil — the system trust store, standard |
| 2822 | // verification — is the production answer and the only one: the home edge |
| 2823 | // presents the SAME publicly-trusted certificate it presents to any browser |
| 2824 | // (Part 2's bucket-distributed certs), so a hop needs no private CA, no |
| 2825 | // pinning, and above all no skipped verification. A var only so a test can |
| 2826 | // trust a fake home edge's self-signed cert. |
| 2827 | // |
| 2828 | // The bootstrap window this implies is stated rather than papered over: a box |
| 2829 | // that has just joined and whose cert for a host has not yet been distributed |
| 2830 | // presents something the dialer will not verify, so the hop fails the |
| 2831 | // handshake and the client gets a 502 until the cert mirror catches up. That |
| 2832 | // is the correct failure — the alternative, dialing without verification, |
| 2833 | // would make the hop's TLS decorative — and it heals on its own within a |
| 2834 | // mirror poll. |
| 2835 | var backhaulRootCAs *x509.CertPool |
| 2836 | |
| 2837 | // The hop's timeouts, per the plan. Deliberately no whole-body deadline: a |
| 2838 | // backhauled response may be a stream or an upgraded WebSocket that is |
| 2839 | // supposed to stay open for hours (the fleetapi client's ruling, same |
| 2840 | // reasoning), so the only bounds are on getting connected and on the home edge |
| 2841 | // beginning to answer. |
| 2842 | const ( |
| 2843 | backhaulDialTimeout = 5 * time.Second |
| 2844 | backhaulHeaderTimeout = 30 * time.Second |
| 2845 | ) |
| 2846 | |
| 2847 | // backhaul forwards a request for a host this box does not serve to the box |
| 2848 | // that does, and reports whether it answered. false means "not mine to |
| 2849 | // forward" and the caller falls through to the unchanged unknown-host answer: |
| 2850 | // either no box homes the host, or nothing could prove the hop. |
| 2851 | // |
| 2852 | // The hopVerified guard is FIRST, before the map is even consulted. ServeHTTP |
| 2853 | // already refuses an arrived hop above this call (that is the structural half |
| 2854 | // of "one hop, ever"), so this line is unreachable today — which is exactly |
| 2855 | // why it is here: if this call is ever moved, or that branch reordered, a hop |
| 2856 | // that arrived from a sibling edge fails closed with the stale-map refusal |
| 2857 | // instead of being forwarded onward. A forwarding loop must take TWO mistakes, |
| 2858 | // not one. |
| 2859 | func (e *Edge) backhaul(w http.ResponseWriter, r *http.Request, host string) bool { |
| 2860 | m := e.homes.Load() |
| 2861 | if hopVerified(r.Context()) { |
| 2862 | m.staleHopRefusal(w, host) |
| 2863 | return true |
| 2864 | } |
| 2865 | eip, box, ok := m.Home(host) |
| 2866 | if !ok { |
| 2867 | return false |
| 2868 | } |
| 2869 | return e.backhaulTo(w, r, host, eip, box) |
| 2870 | } |
| 2871 | |
| 2872 | // backhaulTo is backhaul's dial half: sign, contain, and forward to the box at |
| 2873 | // eip. Split out so the edge-local door dials exactly the target its own |
| 2874 | // Owner read returned (door.go); backhaul is Home plus this, unchanged. It |
| 2875 | // answers false only when no mesh key is mirrored, and it repeats backhaul's |
| 2876 | // arrived-hop guard so neither caller can forward a hop onward. |
| 2877 | func (e *Edge) backhaulTo(w http.ResponseWriter, r *http.Request, host, eip, box string) bool { |
| 2878 | m := e.homes.Load() |
| 2879 | if hopVerified(r.Context()) { |
| 2880 | m.staleHopRefusal(w, host) |
| 2881 | return true |
| 2882 | } |
| 2883 | // Sign with the mirror's cached pair, .next preferred (signingKey, mesh.go) |
| 2884 | // — never a store read on the request path. NO KEY MEANS NO DIAL: signHop |
| 2885 | // would happily HMAC under an empty key, the home edge would refuse the |
| 2886 | // result, and the request would have crossed a region to be served as an |
| 2887 | // ordinary client with its forwarding chain stripped. Falling through to |
| 2888 | // the unknown-host answer is both cheaper and honest about the deployment |
| 2889 | // being mid-bootstrap. |
| 2890 | key := signingKey(m.meshKeys()) |
| 2891 | if len(key) == 0 { |
| 2892 | e.logThrottled("backhaul-no-mesh-key", fmt.Sprintf("%s is homed on %s but no mesh key is mirrored — not dialling (the request gets the unknown-host answer until the key lands)", host, box)) |
| 2893 | return false |
| 2894 | } |
| 2895 | // Containment (the egress ruling, 2026-08-24), between the last guard and |
| 2896 | // the dial: a saturated home box is answered here, with no dial and no |
| 2897 | // bytes crossing a region. 503 + Retry-After: 2 is this package's |
| 2898 | // established "come back shortly" (staleHopRefusal, fleetProxy) and is |
| 2899 | // deliberately NOT the unknown-host answer — that is a 302 on a fallback |
| 2900 | // edge, which would lie to a legitimate client about where its app lives. |
| 2901 | // The log key is per home BOX, like "backhaul-error "+box below and for |
| 2902 | // the same bounded-key reason. |
| 2903 | releaseSlot, ok := e.backhaulAcquire(box) |
| 2904 | if !ok { |
| 2905 | e.logThrottled("backhaul-saturated "+box, |
| 2906 | fmt.Sprintf("backhaul to %s (%s) is at its in-flight cap (%d) — answering 503 + Retry-After: 2", box, eip, backhaulMaxInflight.Load())) |
| 2907 | w.Header().Set("Retry-After", "2") |
| 2908 | http.Error(w, "temporarily busy", http.StatusServiceUnavailable) |
| 2909 | return true |
| 2910 | } |
| 2911 | // Held until the hop's ServeHTTP returns, which for a stream or an |
| 2912 | // upgraded connection is the life of the stream (backhaulAcquire). |
| 2913 | defer releaseSlot() |
| 2914 | // The MAC is over the CONCRETE request host — never the wildcard map key |
| 2915 | // that may have matched it — because the home edge verifies against its own |
| 2916 | // literal request host (hopMAC canonicalizes both, mesh.go). |
| 2917 | auth := signHop(key, host, time.Now()) |
| 2918 | backhaulProxy(host, eip, box, auth, e.backhaulTransport(eip), e.logThrottled).ServeHTTP(w, r) |
| 2919 | return true |
| 2920 | } |
| 2921 | |
| 2922 | // backhaulTransport is the cached outbound transport for one home box. The |
| 2923 | // TRANSPORT is what is cached and the ReverseProxy above is what is rebuilt |
| 2924 | // per request: a proxy value is two closures, while a transport owns the |
| 2925 | // connection pool, and a cross-region TCP+TLS handshake on every backhauled |
| 2926 | // page is a real, visible cost. |
| 2927 | // |
| 2928 | // Keyed by EIP and NOT by host, which is both the cheaper and the safer half |
| 2929 | // of that trade: |
| 2930 | // |
| 2931 | // - Nothing about the connection is per-host. The dial goes to the box, and |
| 2932 | // the TLS ServerName is derived per request from the outbound URL's |
| 2933 | // authority — backhaulProxy sets it to the real hostname — so one |
| 2934 | // transport serves every host homed on one box while each still gets its |
| 2935 | // own SNI, its own certificate verification, and (the stdlib pools by that |
| 2936 | // same authority) its own pooled connections. |
| 2937 | // - Per host, this map would be unbounded. A catch-all row in another box's |
| 2938 | // edges file homes every subdomain under it, so a stranger asking for a |
| 2939 | // stream of invented hostnames would mint a transport apiece. Per box it |
| 2940 | // is bounded by the deployment, and the pooled connections inside it are |
| 2941 | // bounded by MaxIdleConns and IdleConnTimeout as usual. |
| 2942 | // |
| 2943 | // A host whose home MOVES therefore lands on the new box's transport, with the |
| 2944 | // old box's pooled connections left to idle out — no equivalent of the pinning |
| 2945 | // multiProxy pays DisableKeepAlives to avoid, because the authority a |
| 2946 | // connection is pooled under names the host and the transport it lives in |
| 2947 | // names the box. |
| 2948 | // |
| 2949 | // HTTP/2 is not attempted, and that is on purpose: ForceAttemptHTTP2 stays |
| 2950 | // false (the stdlib disables the automatic upgrade anyway once TLSClientConfig |
| 2951 | // or a custom dialer is set), so the hop speaks HTTP/1.1 and |
| 2952 | // httputil.ReverseProxy's protocol-switch path — the one that carries a |
| 2953 | // WebSocket through — works exactly as it does for every other proxy in this |
| 2954 | // file. |
| 2955 | func (e *Edge) backhaulTransport(eip string) *http.Transport { |
| 2956 | if tr, ok := e.backhaulTransports.Load(eip); ok { |
| 2957 | return tr.(*http.Transport) |
| 2958 | } |
| 2959 | dialer := &net.Dialer{Timeout: backhaulDialTimeout} |
| 2960 | tr := &http.Transport{ |
| 2961 | // The address is ignored, multiProxy-style: it is the outbound URL's |
| 2962 | // authority, which is the HOST (for SNI, verification and pooling) and |
| 2963 | // never where the bytes go. The home edge is at its EIP. |
| 2964 | DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) { |
| 2965 | return dialer.DialContext(ctx, network, net.JoinHostPort(eip, backhaulPort)) |
| 2966 | }, |
| 2967 | // No ServerName: the stdlib fills it from the outbound URL's authority, |
| 2968 | // which is exactly the host being asked for. RootCAs nil is the system |
| 2969 | // trust store — standard verification, see backhaulRootCAs. |
| 2970 | TLSClientConfig: &tls.Config{ |
| 2971 | RootCAs: backhaulRootCAs, |
| 2972 | MinVersion: tls.VersionTLS12, |
| 2973 | }, |
| 2974 | TLSHandshakeTimeout: backhaulDialTimeout, |
| 2975 | ResponseHeaderTimeout: backhaulHeaderTimeout, |
| 2976 | // PER-HOST as well as in total, which is where this diverges from |
| 2977 | // unixProxy/tcpProxy's bare MaxIdleConns: their pool key is one constant |
| 2978 | // localhost authority, so the stdlib default of 2 idle conns per host is |
| 2979 | // really 2 per BACKEND. Here the key is the hostname, so the default |
| 2980 | // would cap each backhauled host at 2 — and the third concurrent request |
| 2981 | // to a busy host would pay the cross-region TCP+TLS handshake this cache |
| 2982 | // exists to avoid. Equal to MaxIdleConns so the total is the only bound. |
| 2983 | MaxIdleConns: 32, |
| 2984 | MaxIdleConnsPerHost: 32, |
| 2985 | IdleConnTimeout: 90 * time.Second, |
| 2986 | } |
| 2987 | // Two requests for a box nobody has dialled yet can both build one; the |
| 2988 | // loser's transport is simply dropped, having opened nothing and started no |
| 2989 | // goroutine. |
| 2990 | if existing, loaded := e.backhaulTransports.LoadOrStore(eip, tr); loaded { |
| 2991 | return existing.(*http.Transport) |
| 2992 | } |
| 2993 | return tr |
| 2994 | } |
| 2995 | |
| 2996 | // backhaulProxy is the hop itself: an ordinary reverse proxy to the home box, |
| 2997 | // carrying the client's Host through unchanged so the home edge routes on the |
| 2998 | // real hostname, and carrying the two hop headers that make it a hop rather |
| 2999 | // than an anonymous request from a stranger with a data-centre IP. |
| 3000 | // |
| 3001 | // The outbound URL names the HOST, not the EIP, and the transport's dialer is |
| 3002 | // what turns that into bytes at the home box (backhaulTransport). The |
| 3003 | // authority is therefore doing three jobs at once and all three want the |
| 3004 | // hostname: the TLS ServerName the home edge selects a certificate by, the |
| 3005 | // name that certificate is verified against, and the key the connection is |
| 3006 | // pooled under. |
| 3007 | // |
| 3008 | // It takes the signature ready-made instead of minting one inside Rewrite, so |
| 3009 | // that the "no key, no dial" decision (backhaul, above) and the signing happen |
| 3010 | // at the SAME read of the mirror's key pair — a Rewrite that signed for itself |
| 3011 | // could still reach for a key that had gone away since the guard. |
| 3012 | // |
| 3013 | // X-Forwarded-For is setForwarded's single-element chain and nothing more: the |
| 3014 | // client that dialled THIS edge, with neither this edge's own address nor the |
| 3015 | // home EIP appended. That is the contract homemap.go's setForwarded documents |
| 3016 | // from the receiving end — the home edge restores this chain verbatim on a |
| 3017 | // verified hop, so the app's last XFF element is the real client, exactly as |
| 3018 | // it would be if the request had landed on the home box directly. (The restore |
| 3019 | // branch cannot fire on THIS side: backhaul only runs when the request did not |
| 3020 | // arrive as a verified hop.) |
| 3021 | // |
| 3022 | // logThrottled is e.logThrottled, threaded through rather than closed over |
| 3023 | // directly: this is a free function (built fresh per request, unlike the |
| 3024 | // cached backhaulTransport) so it takes no *Edge, and a dial failure is a |
| 3025 | // STANDING condition for as long as a home box is unreachable — an |
| 3026 | // unthrottled log.Printf here would fill the journal with one line per |
| 3027 | // failed request for the whole outage. Keyed "backhaul-error "+box (per home |
| 3028 | // box, not per host — a wildcard home behind one unreachable box would |
| 3029 | // otherwise mint a fresh throttle bucket per invented hostname). |
| 3030 | func backhaulProxy(host, eip, box, auth string, tr http.RoundTripper, logThrottled func(key, msg string)) *httputil.ReverseProxy { |
| 3031 | return &httputil.ReverseProxy{ |
| 3032 | Rewrite: func(pr *httputil.ProxyRequest) { |
| 3033 | setForwarded(pr) |
| 3034 | setGeo(pr, true) // geoip spec §8.3: after hop-header removal, from the request's one answer |
| 3035 | // No forwarded-for proof across a hop. The proof is for the |
| 3036 | // upstream the home edge dials, and the home edge signs that |
| 3037 | // itself. An older home edge would pass this one through |
| 3038 | // untouched, and the console would take it as the sign that its |
| 3039 | // own edge signs, while its local traffic carried none |
| 3040 | // (edgeproof.go's rollout rule, in internal/console). |
| 3041 | pr.Out.Header.Del(edgeproof.Header) |
| 3042 | pr.Out.URL.Scheme = "https" |
| 3043 | pr.Out.URL.Host = host // SNI + verification + pool key; the dial is the transport's |
| 3044 | pr.Out.Host = pr.In.Host // the home edge routes on the real hostname |
| 3045 | pr.Out.Header.Set(hopHeader, "1") |
| 3046 | pr.Out.Header.Set(hopAuthHeader, auth) |
| 3047 | // The same discipline every other Rewrite here keeps, and needed for |
| 3048 | // the same reason one hop further along: the home edge scrubs it |
| 3049 | // again before the app, but a header this edge forwards is a header |
| 3050 | // this edge vouched for. |
| 3051 | scrubInboundPurgeHeader(pr) |
| 3052 | }, |
| 3053 | Transport: tr, |
| 3054 | ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { |
| 3055 | // 502, not the 503 + Retry-After that multiProxy and fleetProxy |
| 3056 | // answer with: those two KNOW nobody is home (an empty live set, no |
| 3057 | // connected channel), while this one believed a specific box was |
| 3058 | // there and could not reach it. A failed dial, a TLS handshake |
| 3059 | // against a cert the mirror has not distributed yet, or a home edge |
| 3060 | // that hung up mid-response are all genuinely bad gateways — and the |
| 3061 | // cert case heals itself on the next mirror poll. |
| 3062 | logThrottled("backhaul-error "+box, fmt.Sprintf("backhaul %s → %s (%s): %v", host, box, eip, err)) |
| 3063 | w.WriteHeader(http.StatusBadGateway) |
| 3064 | }, |
| 3065 | } |
| 3066 | } |
| 3067 | |
| 3068 | // dialUnixWithRetry masks the brief gap during an instance restart: while the |
| 3069 | // old process has exited and the new one hasn't rebound the socket, dials fail |
| 3070 | // with ENOENT/ECONNREFUSED. Retry those for a short window so no in-flight |
| 3071 | // request is dropped; any other error fails fast. The window must exceed a |
| 3072 | // deploy's socket-absent gap — the instance drain (10s) plus rebind — so an |
| 3073 | // in-flight request is never 502'd mid-deploy; a sustained-down backend hangs |
| 3074 | // this long before 502 (rare: systemd Restart=always makes "down" transient). |
| 3075 | func dialUnixWithRetry(ctx context.Context, d *net.Dialer, socketPath string) (net.Conn, error) { |
| 3076 | return dialUnixRetrying(ctx, d, func() (string, bool) { return socketPath, true }) |
| 3077 | } |
| 3078 | |
| 3079 | func isRetryableDialErr(err error) bool { |
| 3080 | return errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.ENOENT) |
| 3081 | } |
| 3082 | |
| 3083 | // Run is the `carlos edge` verb: the shared front door for every instance, |
| 3084 | // sibling service, and static site on the box. |
| 3085 | func Run(args []string) { |
| 3086 | RunWithActivator(args, nil, NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes)) |
| 3087 | } |
| 3088 | |
| 3089 | // RunWithActivator is Run with an injected hibernation activator — the seam |
| 3090 | // `carlos agent` (internal/host) uses to compose edge + activator in one |
| 3091 | // process. act may be nil: pure proxy, exactly Run's behavior. cache may |
| 3092 | // also be nil, disabling response caching entirely. |
| 3093 | func RunWithActivator(args []string, act Activator, cache *Cache) { |
| 3094 | RunWithActivatorReady(args, act, cache, nil) |
| 3095 | } |
| 3096 | |
| 3097 | // RunWithActivatorReady is RunWithActivator plus a ready hook: ready (when |
| 3098 | // non-nil) is called with the fully configured Edge — every EnableXxx done, |
| 3099 | // tunnel hub armed if configured — immediately before the process blocks on |
| 3100 | // its listeners. It is the seam an in-process caller uses to reach the live |
| 3101 | // Edge (e.g. to read tunnel state for status reporting) without racing the |
| 3102 | // configuration it needs. It runs on the calling goroutine, so a hook that |
| 3103 | // blocks holds the front door shut: hand off to a goroutine and return. |
| 3104 | // |
| 3105 | // opts configure the Edge the moment it exists — before any EnableXxx, and |
| 3106 | // long before the TLS listener — so a knob the composing caller owns |
| 3107 | // (WithIssuerGate) is in place for the construction decisions that read it. |
| 3108 | // The ready hook is the wrong home for those: it fires once, at the end, for |
| 3109 | // state a caller wants to REACH. |
| 3110 | func RunWithActivatorReady(args []string, act Activator, cache *Cache, ready func(*Edge), opts ...RunOption) { |
| 3111 | dataDir := envOr("CARLOS_DATA_DIR", "/data") |
| 3112 | fs := flag.NewFlagSet("edge", flag.ExitOnError) |
| 3113 | registryPath := fs.String("registry", filepath.Join(dataDir, "registry.db"), "registry SQLite path (default $CARLOS_DATA_DIR/registry.db)") |
| 3114 | acmeCache := fs.String("acme-cache", filepath.Join(dataDir, "acme"), "autocert certificate cache dir (default $CARLOS_DATA_DIR/acme)") |
| 3115 | httpsAddr := fs.String("https", ":443", "HTTPS listen address") |
| 3116 | httpAddr := fs.String("http", ":80", "HTTP listen address (ACME HTTP-01 challenge + redirect to https; in --dev mode, the only listener)") |
| 3117 | fallbackURL := fs.String("fallback", "", "redirect requests for unknown hosts here, e.g. an apex marketing site; empty keeps the plain 404") |
| 3118 | dev := fs.Bool("dev", false, "plain-HTTP mode: serve routes on --http with no TLS/ACME (local development only)") |
| 3119 | fs.Usage = cliflag.Usage(fs, "") |
| 3120 | _ = fs.Parse(args) |
| 3121 | |
| 3122 | reg, err := registry.Open(*registryPath) |
| 3123 | if err != nil { |
| 3124 | log.Fatalf("edge: open registry %s: %v", *registryPath, err) |
| 3125 | } |
| 3126 | |
| 3127 | // act is nil for `carlos edge` — the pure-proxy role; `carlos agent` |
| 3128 | // injects the hibernation activator (edge + activator in one process). |
| 3129 | e, err := New(reg, act) |
| 3130 | if err != nil { |
| 3131 | log.Fatalf("edge: load routes: %v", err) |
| 3132 | } |
| 3133 | for _, opt := range opts { |
| 3134 | opt(e) |
| 3135 | } |
| 3136 | e.fallbackURL = *fallbackURL |
| 3137 | e.setPlatformDomains(platformDomainsFromEnv(os.Getenv)) |
| 3138 | e.EnableCache(cache) |
| 3139 | |
| 3140 | // Static serving lights up when the environment carries a deployment |
| 3141 | // store and the release pubkey; otherwise static routes 404 until it does. |
| 3142 | if err := enableStaticFromEnv(e, dataDir); err != nil { |
| 3143 | log.Fatalf("edge: static serving: %v", err) |
| 3144 | } |
| 3145 | |
| 3146 | // Reverse channels light up when the environment names a reserved |
| 3147 | // hostname and at least one client; otherwise tunnel routes stay skipped. |
| 3148 | if err := enableTunnelsFromEnv(e); err != nil { |
| 3149 | log.Fatalf("edge: tunnels: %v", err) |
| 3150 | } |
| 3151 | |
| 3152 | // The backhaul in-flight cap, read from the environment once here rather |
| 3153 | // than per request (backhaulMaxInflight). Inert on a single-box |
| 3154 | // deployment, where nothing is ever backhauled at all. |
| 3155 | backhaulMaxInflight.Store(int64(backhaulInflightFromEnv())) |
| 3156 | |
| 3157 | // GeoIP (geoip spec §5.1): a tunnel client takes the hub's answer and |
| 3158 | // mirrors nothing; an edge with the deployment store mirrors; anything |
| 3159 | // else has no database and only strips forged headers. |
| 3160 | switch { |
| 3161 | case e.geoTunnelClient: |
| 3162 | e.geoMode = "tunnel" |
| 3163 | case e.staticStore != nil: |
| 3164 | e.geoMode = "mirror" |
| 3165 | e.geoMirror = newGeoMirror(e.staticStore, filepath.Join(dataDir, "carlos", "geoip"), e.geo) |
| 3166 | e.geoMirror.Boot() |
| 3167 | default: |
| 3168 | e.geoMode = "none" |
| 3169 | } |
| 3170 | if e.geoMirror != nil { |
| 3171 | // Booted and started here, before ready(e) publishes the edge and |
| 3172 | // before the dev-mode return, so the heartbeat never reads a |
| 3173 | // half-set geoMode and a dev edge mirrors too (plan review R1-5). |
| 3174 | go e.geoMirror.run(make(chan struct{})) // process-lifetime, like the cert mirror |
| 3175 | } |
| 3176 | |
| 3177 | go e.Watch(make(chan struct{})) // process-lifetime; never stopped |
| 3178 | |
| 3179 | // Reload routes on SIGHUP too (an operator's explicit "now"). |
| 3180 | sighup := make(chan os.Signal, 1) |
| 3181 | signal.Notify(sighup, syscall.SIGHUP) |
| 3182 | go func() { |
| 3183 | for range sighup { |
| 3184 | if err := e.Reload(); err != nil { |
| 3185 | log.Printf("edge: reload failed: %v", err) |
| 3186 | continue |
| 3187 | } |
| 3188 | log.Printf("edge: reloaded (%d route(s))", len(e.current())) |
| 3189 | } |
| 3190 | }() |
| 3191 | |
| 3192 | stop := make(chan os.Signal, 1) |
| 3193 | signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT) |
| 3194 | |
| 3195 | // The Edge is fully configured here — every EnableXxx applied and the |
| 3196 | // route table rebuilt behind them — and nothing is listening yet. |
| 3197 | if ready != nil { |
| 3198 | ready(e) |
| 3199 | } |
| 3200 | |
| 3201 | if *dev { |
| 3202 | if e.doorPath != "" { |
| 3203 | log.Printf("edge: edge-local door off in dev mode") |
| 3204 | } |
| 3205 | // ReadHeaderTimeout guards against Slowloris-style slow-header |
| 3206 | // connection exhaustion. No ReadTimeout/WriteTimeout — those would |
| 3207 | // break long-lived proxied WebSockets. |
| 3208 | srv := &http.Server{Addr: *httpAddr, Handler: e, ReadHeaderTimeout: 10 * time.Second} |
| 3209 | go func() { |
| 3210 | log.Printf("edge: DEV MODE — plain HTTP, serving %d route(s) on %s (registry %s)", |
| 3211 | len(e.current()), *httpAddr, *registryPath) |
| 3212 | if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { |
| 3213 | log.Fatalf("edge: http listener %s: %v", *httpAddr, err) |
| 3214 | } |
| 3215 | }() |
| 3216 | <-stop |
| 3217 | log.Printf("edge: draining and shutting down") |
| 3218 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
| 3219 | defer cancel() |
| 3220 | _ = srv.Shutdown(ctx) |
| 3221 | return |
| 3222 | } |
| 3223 | |
| 3224 | certCache, err := newCertCache(*acmeCache) |
| 3225 | if err != nil { |
| 3226 | log.Fatalf("edge: %v", err) |
| 3227 | } |
| 3228 | // The RAW local cache, kept before the publishing wrapper goes on. The |
| 3229 | // mirror writes INWARDS through this handle and reads it as its last |
| 3230 | // serving resort; going through the wrapper instead would have the issuer |
| 3231 | // re-publishing its own material to the bucket on every poll. |
| 3232 | localCerts := certCache |
| 3233 | // Publish every issuance artifact to the control bucket, and gate |
| 3234 | // issuance on being the box that may write there (certsync.go): per-host |
| 3235 | // certs, the ACME account keys, HTTP-01 tokens, and — through the same |
| 3236 | // handle below — the DNS-01 wildcards. One call because it is one |
| 3237 | // decision: a box with no store to publish to also has no mirror to |
| 3238 | // serve from, so it stays ungated and keeps issuing for itself. |
| 3239 | certCache = e.armCertIssuance(certCache) |
| 3240 | |
| 3241 | // The other side of that decision (certsync.go): the mirror of |
| 3242 | // control/certs/, which is what makes the gate safe to shut. nil when |
| 3243 | // there is no deployment store — armCertIssuance has then cleared the |
| 3244 | // gate too, and this box issues and serves entirely for itself. |
| 3245 | mirror := e.newCertMirror(localCerts) |
| 3246 | if mirror != nil { |
| 3247 | // One synchronous pass before anything listens: a box that has just |
| 3248 | // booted must not spend its first 30 seconds handshaking with nothing, |
| 3249 | // and the wildcard manager's first pass (below) reads what this lands |
| 3250 | // in the local cache. |
| 3251 | ctx, cancel := context.WithTimeout(context.Background(), certMirrorBootTimeout) |
| 3252 | if err := mirror.Refresh(ctx); err != nil { |
| 3253 | log.Printf("edge: first cert mirror refresh failed: %v — serving from this box's own cache until a later poll lands", err) |
| 3254 | } |
| 3255 | cancel() |
| 3256 | } |
| 3257 | |
| 3258 | // The home map and the mesh keys (homemap.go), armed beside the cert |
| 3259 | // mirror and on the same terms: nil when there is no deployment store to |
| 3260 | // mirror from, one synchronous refresh before anything listens so the |
| 3261 | // first request does not meet an empty map, then the 30s loop. Stored |
| 3262 | // before either listener binds, so ServeHTTP never sees a half-armed edge. |
| 3263 | if hm := e.newHomeMap(); hm != nil { |
| 3264 | ctx, cancel := context.WithTimeout(context.Background(), homeMapBootTimeout) |
| 3265 | if err := hm.Refresh(ctx); err != nil { |
| 3266 | log.Printf("edge: first home map refresh failed: %v — hosts homed on another box are unreachable from here until a later poll lands", err) |
| 3267 | } |
| 3268 | cancel() |
| 3269 | e.homes.Store(hm) |
| 3270 | go hm.run(make(chan struct{})) // process-lifetime; never stopped, like the cert mirror's |
| 3271 | } |
| 3272 | |
| 3273 | // The edge-local door (door.go, #175 P7), armed right after the home map |
| 3274 | // it dials through: its own key mirror, one synchronous refresh, its |
| 3275 | // loop, then the socket. Off, with the reason in the journal, when there |
| 3276 | // is no store, no release key or no public address. Closed in the |
| 3277 | // shutdown below, ahead of the public listeners. |
| 3278 | _ = e.armDoor(context.Background()) |
| 3279 | |
| 3280 | m := &autocert.Manager{ |
| 3281 | Prompt: autocert.AcceptTOS, |
| 3282 | Cache: certCache, |
| 3283 | // Registry membership AND the issuance gate — see issuerHostPolicy |
| 3284 | // (certsync.go) for why the gate cannot be relaxed for cached names, |
| 3285 | // and for why a transient double-holder is deliberately not fenced. |
| 3286 | HostPolicy: e.issuerHostPolicy(), |
| 3287 | } |
| 3288 | tlsCfg := m.TLSConfig() |
| 3289 | // The handshake chain, built inside out: autocert, then the mirror (which |
| 3290 | // also carries the direct local-cache last resort behind it), then the |
| 3291 | // wildcard. See certMirror.wrapGetCertificate for why the mirror sits in |
| 3292 | // FRONT of autocert rather than behind it. |
| 3293 | getCert := tlsCfg.GetCertificate |
| 3294 | if mirror != nil { |
| 3295 | getCert = mirror.wrapGetCertificate(getCert) |
| 3296 | } |
| 3297 | // The per-host path, captured before the wildcard wrap goes on top of it: |
| 3298 | // this is what the cert warm below calls. Calling the WRAPPED chain would |
| 3299 | // let the wildcard answer for a covered name and order nothing. |
| 3300 | perHostCert := getCert |
| 3301 | // Wildcard certs (wildcard.go, ACME DNS-01): covered names — one label |
| 3302 | // under a per-account or configured parent — are served the wildcard; |
| 3303 | // everything else (custom domains, canary hosts, parents not yet |
| 3304 | // obtained) stays on autocert's per-host HTTP-01 above. nil = not |
| 3305 | // configured = exactly the pre-wildcard edge. |
| 3306 | var wm *wildcardManager |
| 3307 | if w := newWildcardFromEnv(e, certCache); w != nil { |
| 3308 | wm = w |
| 3309 | getCert = wrapGetCertificate(w, getCert) |
| 3310 | e.OnReload(w.refresh) |
| 3311 | attachWildcardToMirror(mirror, w) |
| 3312 | // A remote box's edges file landing (or leaving) can add or drop a |
| 3313 | // parent (#300): re-derive on the home map's poll, not the 12h tick. |
| 3314 | e.homes.Load().setHostsWake(w.refresh) |
| 3315 | w.refresh() |
| 3316 | go w.run() |
| 3317 | log.Printf("edge: wildcard certs on (static %v, apps domain %q, delegation zone %q)", |
| 3318 | splitDomains(os.Getenv("CARLOS_WILDCARD_DOMAINS")), os.Getenv("PLATFORM_APPS_DOMAIN"), w.delegationZone) |
| 3319 | } |
| 3320 | // Cert warm (#299 stage 3, certwarm.go): the platform ORDERS and RENEWS |
| 3321 | // custom domains' certs instead of waiting for a visitor's handshake. |
| 3322 | // |
| 3323 | // It calls perHostCert — the chain as it stood BEFORE the wildcard wrap — |
| 3324 | // because the warm's set already excludes every wildcard-covered name, |
| 3325 | // and a wrapped call would be answered by the wildcard and order nothing. |
| 3326 | // |
| 3327 | // Armed only with a deployment store: an agent-less `carlos edge` is |
| 3328 | // ungated (issuerGate nil means yes), and would otherwise start placing |
| 3329 | // proactive ACME orders on a hobbyist's box. |
| 3330 | if e.staticStore != nil { |
| 3331 | appsDomain := strings.ToLower(strings.TrimSpace(os.Getenv("PLATFORM_APPS_DOMAIN"))) |
| 3332 | parents := func() []string { return nil } |
| 3333 | if wm != nil { |
| 3334 | parents = wm.currentParents |
| 3335 | } |
| 3336 | envIPs := splitDomains(os.Getenv("PLATFORM_EDGE_IPS")) |
| 3337 | store := e.staticStore |
| 3338 | edgeIPs := func() []string { |
| 3339 | ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) |
| 3340 | defer cancel() |
| 3341 | return edgesfile.EdgeIPs(ctx, store, envIPs, time.Now()) |
| 3342 | } |
| 3343 | warm := newCertWarm( |
| 3344 | func() []string { return e.warmSet(appsDomain, parents) }, |
| 3345 | perHostCert, |
| 3346 | dnsGate(warmResolver, edgeIPs), |
| 3347 | e.issuerGate, |
| 3348 | store, |
| 3349 | ) |
| 3350 | go warm.run() |
| 3351 | log.Printf("edge: cert warm on (%s pass) — custom domains are ordered and renewed proactively, not on first handshake", warmInterval) |
| 3352 | } |
| 3353 | tlsCfg.GetCertificate = getCert |
| 3354 | if mirror != nil { |
| 3355 | // Started last: every field the loop reads (wake above) is in place. |
| 3356 | go mirror.run(make(chan struct{})) // process-lifetime; never stopped |
| 3357 | } |
| 3358 | |
| 3359 | // The non-ACME :80 handler. See httpFallbackHandler's doc comment for the |
| 3360 | // Allowed-vs-lookupUpstream split. |
| 3361 | httpFallback := e.httpFallbackHandler() |
| 3362 | |
| 3363 | // :80's ACME path, outermost first: the mirror answers any edge's HTTP-01 |
| 3364 | // challenge, autocert answers its own live orders, and everything else |
| 3365 | // falls through to the redirect handler. |
| 3366 | acmeAndFallback := m.HTTPHandler(httpFallback) |
| 3367 | if mirror != nil { |
| 3368 | acmeAndFallback = mirror.challengeHandler(acmeAndFallback) |
| 3369 | } |
| 3370 | |
| 3371 | // ReadHeaderTimeout guards the public front door against Slowloris-style |
| 3372 | // slow-header connection exhaustion. No ReadTimeout/WriteTimeout — those |
| 3373 | // would break long-lived proxied WebSockets. |
| 3374 | httpSrv := &http.Server{Addr: *httpAddr, Handler: acmeAndFallback, ReadHeaderTimeout: 10 * time.Second} |
| 3375 | httpsSrv := &http.Server{Addr: *httpsAddr, Handler: e, TLSConfig: tlsCfg, ReadHeaderTimeout: 10 * time.Second} |
| 3376 | |
| 3377 | // :80 serves the ACME HTTP-01 challenge and redirects everything else. |
| 3378 | go func() { |
| 3379 | if err := httpSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed { |
| 3380 | log.Fatalf("edge: http listener %s: %v", *httpAddr, err) |
| 3381 | } |
| 3382 | }() |
| 3383 | go func() { |
| 3384 | log.Printf("edge: serving %d route(s) on %s (registry %s, acme cache %s)", |
| 3385 | len(e.current()), *httpsAddr, *registryPath, *acmeCache) |
| 3386 | if err := httpsSrv.ListenAndServeTLS("", ""); err != nil && err != http.ErrServerClosed { |
| 3387 | log.Fatalf("edge: https listener %s: %v", *httpsAddr, err) |
| 3388 | } |
| 3389 | }() |
| 3390 | |
| 3391 | // Graceful shutdown on SIGTERM/SIGINT (e.g. an edge binary upgrade): stop |
| 3392 | // accepting and drain in-flight HTTP before exiting. Proxied WebSockets are |
| 3393 | // hijacked (not drained); they reconnect through the new edge. |
| 3394 | <-stop |
| 3395 | log.Printf("edge: draining and shutting down") |
| 3396 | if e.closeDoor != nil { |
| 3397 | e.closeDoor() |
| 3398 | } |
| 3399 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
| 3400 | defer cancel() |
| 3401 | _ = httpsSrv.Shutdown(ctx) |
| 3402 | _ = httpSrv.Shutdown(ctx) |
| 3403 | } |
| 3404 | |
| 3405 | // enableStaticFromEnv configures static serving when the environment names a |
| 3406 | // deployment store (CARLOS_DEPLOYMENT_BUCKET for S3, else |
| 3407 | // CARLOS_DEPLOYMENT_DIR for a directory) and CARLOS_RELEASE_PUBKEY. It |
| 3408 | // deliberately duplicates the CLI's ten-line store-open helper — the edge |
| 3409 | // can't import package main. Missing config is not an error: static routes |
| 3410 | // simply keep answering 404, exactly the pre-A8 behavior. |
| 3411 | func enableStaticFromEnv(e *Edge, dataDir string) error { |
| 3412 | pubB64 := os.Getenv("CARLOS_RELEASE_PUBKEY") |
| 3413 | bucket := os.Getenv("CARLOS_DEPLOYMENT_BUCKET") |
| 3414 | dir := os.Getenv("CARLOS_DEPLOYMENT_DIR") |
| 3415 | if pubB64 == "" || (bucket == "" && dir == "") { |
| 3416 | return nil |
| 3417 | } |
| 3418 | pub, err := release.ParsePublicKey(pubB64, "CARLOS_RELEASE_PUBKEY") |
| 3419 | if err != nil { |
| 3420 | return err |
| 3421 | } |
| 3422 | var store release.Store |
| 3423 | if bucket != "" { |
| 3424 | s, err := release.NewDeploymentS3Store(context.Background(), bucket) |
| 3425 | if err != nil { |
| 3426 | return err |
| 3427 | } |
| 3428 | store = s |
| 3429 | } else { |
| 3430 | ds := release.NewDirStore(dir) |
| 3431 | ds.EnableChangeIndex() |
| 3432 | store = ds |
| 3433 | } |
| 3434 | e.EnableStatic(store, pub, filepath.Join(dataDir, "static")) |
| 3435 | // The process's one ratchet for this directory (Task 8): the agent's |
| 3436 | // adoption passes and this reader share it. |
| 3437 | e.staticRatchet = release.SharedDigestRatchet(release.EdgeRatchetDir(dataDir), |
| 3438 | []string{release.RootRatchetDir(release.OptDir())}, release.RequirePointerDigest(os.Getenv)) |
| 3439 | // Rebuild the route table so static routes that loaded as 404 stubs in |
| 3440 | // New's startup reload get real handlers. |
| 3441 | return e.Reload() |
| 3442 | } |
| 3443 | |
| 3444 | // enableTunnelsFromEnv arms the reverse-channel hub when the environment |
| 3445 | // names a reserved hostname (CARLOS_TUNNEL_HOST); CARLOS_TUNNEL_CLIENTS |
| 3446 | // ("name=token[,name=token]") names its STATIC authorized clients, which may |
| 3447 | // legitimately be empty. No CARLOS_TUNNEL_HOST is not an error: the box |
| 3448 | // simply serves no tunnels at all, and routes with Tunnel set keep whatever |
| 3449 | // local backend they have (Reload's tunnel case is guarded on the hub) — no |
| 3450 | // fleet channels either, since those need a hub to land on just as much as a |
| 3451 | // static client does. |
| 3452 | // |
| 3453 | // A host with no static clients configured IS still valid, unlike before |
| 3454 | // fleet channels existed: the hub also serves customer-fleet boxes, whose |
| 3455 | // credentials come from the registry (Reload -> SetDynamicAuth), not this |
| 3456 | // env var — a box that only ever hosts fleet channels legitimately runs with |
| 3457 | // CARLOS_TUNNEL_CLIENTS unset. The registration endpoint this arms is not |
| 3458 | // "no one can ever authenticate against" in that case; it is "no STATIC |
| 3459 | // client can," which the operator chose on purpose. |
| 3460 | // |
| 3461 | // Static channel names are validated here, at the only place one enters the |
| 3462 | // process from config: a name the CLI would refuse (registry.ValidTunnelName, |
| 3463 | // what `carlos route -tunnel` enforces) can never match a route's Tunnel |
| 3464 | // column, so such a client would authenticate, connect, and silently carry no |
| 3465 | // traffic at all. Fail the boot instead, naming the entry. Dynamic (fleet) |
| 3466 | // channel names are validated separately, at the registry writer |
| 3467 | // (AddFleetBox) — they are never parsed from this env var. |
| 3468 | func enableTunnelsFromEnv(e *Edge) error { |
| 3469 | host := os.Getenv("CARLOS_TUNNEL_HOST") |
| 3470 | if host == "" { |
| 3471 | return nil |
| 3472 | } |
| 3473 | clients, err := parseTunnelClients(os.Getenv("CARLOS_TUNNEL_CLIENTS")) |
| 3474 | if err != nil { |
| 3475 | return err |
| 3476 | } |
| 3477 | for _, name := range slices.Sorted(maps.Keys(clients)) { |
| 3478 | if !registry.ValidTunnelName(name) { |
| 3479 | return fmt.Errorf("CARLOS_TUNNEL_CLIENTS: %q is not a valid tunnel name — want [a-z0-9-]{1,32}, and it must match a route's --tunnel to carry anything", name) |
| 3480 | } |
| 3481 | } |
| 3482 | e.EnableTunnels(NewTunnelHub(host, clients)) |
| 3483 | // Rebuild the route table so tunnel-backed routes that were skipped (or |
| 3484 | // left on their old socket/addr) by the startup reload pick up the hub, |
| 3485 | // and so fleet channel auth (SetDynamicAuth) gets its first load. |
| 3486 | if err := e.Reload(); err != nil { |
| 3487 | return fmt.Errorf("reload after EnableTunnels: %w", err) |
| 3488 | } |
| 3489 | log.Printf("edge: tunnel hub armed on %s (%d static client(s))", host, len(clients)) |
| 3490 | if len(clients) == 0 { |
| 3491 | // Zero static clients is legitimate (doc comment above), but it is |
| 3492 | // ALSO exactly what a typo'd CARLOS_TUNNEL_CLIENTS var name produces |
| 3493 | // (the env var silently reads empty rather than failing to parse) — |
| 3494 | // an operator staring at logs needs this distinguishable from "I |
| 3495 | // meant to run fleet-channels-only," not inferred from a client |
| 3496 | // count buried in the line above. |
| 3497 | log.Printf("edge: WARNING: no static tunnel clients configured — fleet channels only") |
| 3498 | } |
| 3499 | return nil |
| 3500 | } |
| 3501 | |
| 3502 | func envOr(key, fallback string) string { |
| 3503 | if v := os.Getenv(key); v != "" { |
| 3504 | return v |
| 3505 | } |
| 3506 | return fallback |
| 3507 | } |
| 3508 | |
amadan