amadan

carlos / platform Public

edge-unknown-host-174

Squashed from edge-unknown-host-174 into main.
claude 055ed3b6aa7f88740f76bb47ac076c41786b72e8 parent 0d2fdd4
9 files changed, +869 −26
  • internal/edge/edge.go +27 −0
  • internal/edge/homemap.go +7 −5
  • internal/edge/pending.go +222 −0
  • internal/edge/pending_script_test.go +136 −0
  • internal/edge/pending_test.go +167 −0
  • internal/edge/steeringprobe.go +3 −1
  • internal/edge/wildcard.go +3 −1
  • internal/host/edgesfile.go +152 −19
  • internal/host/edgesfile_schedule_test.go +152 −0

CI cancelled — superseded by a newer push — run details

diff --git a/internal/edge/edge.go b/internal/edge/edge.go
index 063435f7a..f47dd053e 100644
--- a/internal/edge/edge.go
+++ b/internal/edge/edge.go
@@ -108,7 +108,20 @@ type Edge struct {
// removed instance); a never-served host fails the TLS handshake first,
// because HostPolicy refuses to mint certs for unknown hosts (anything else
// would let strangers burn the ACME rate limit). The :80 path always works.
+ //
+ // Hosts under a platform domain are the exception (pending.go, #174): they
+ // get the 503 pending page, which reaches this URL only through its
+ // script, after the retry window. curl, crawlers and no-JS clients see the
+ // 503 and never the redirect.
fallbackURL string
+ // platformDomains and pendingBody are the "not here YET" answer
+ // (pending.go, #174): an unknown host strictly under one of these gets a
+ // 503 retry page instead of unknownHost's 404 or fallback redirect. Both
+ // are set together by setPlatformDomains, at start; nil domains (every
+ // test that builds an Edge with New, and a deployment with neither env
+ // set) keep the old answer for every host.
+ platformDomains []string
+ pendingBody []byte
// steeringHost is this deployment's DNS-steering hostname
// (edge.<apps-domain>), or "" when the deployment does not steer — which
// is every deployment today. Set once at construction from the
@@ -1890,7 +1903,20 @@ func (r *recordingResponseWriter) Flush() {
// unknownHost answers a request for a host we don't serve: a redirect to the
// fallback (302 — a typo'd host must not be cached as permanently moved), or
// the classic 404 when no fallback is configured.
+//
+// A host under one of this deployment's own domains is answered with the
+// pending page instead (pending.go): it may be a host created a moment ago
+// that this edge has not heard about yet, and a dead end for it strands the
+// visitor where a retry would have worked.
func (e *Edge) unknownHost(w http.ResponseWriter, r *http.Request) {
+ host := r.Host
+ if h, _, err := net.SplitHostPort(host); err == nil {
+ host = h
+ }
+ if e.underPlatformDomain(host) {
+ e.pendingHost(w, r)
+ return
+ }
if e.fallbackURL != "" {
http.Redirect(w, r, e.fallbackURL, http.StatusFound)
return
@@ -2775,6 +2801,7 @@ func RunWithActivatorReady(args []string, act Activator, cache *Cache, ready fun
opt(e)
}
e.fallbackURL = *fallbackURL
+ e.setPlatformDomains(platformDomainsFromEnv(os.Getenv))
e.EnableCache(cache)
// Static serving lights up when the environment carries a deployment
diff --git a/internal/edge/homemap.go b/internal/edge/homemap.go
index 4057747db..482d6951d 100644
--- a/internal/edge/homemap.go
+++ b/internal/edge/homemap.go
@@ -815,10 +815,12 @@ func stripForwardedHeaders(r *http.Request) {
// is the same "come back shortly" answer a dark fleet gets (fleetProxy).
// Retry-After: 2 is a keep-trying hint, not a heal estimate: there is no 2s
// serial-converge path here (edges never bump control/serial), so what
-// actually heals a stale map is the edges-file cycle — the sending box's own
-// 5min+jitter publish loop (internal/host's edgesFileLoop) followed by this
-// box's 30s mirror poll — which can leave a sender retrying into this refusal
-// for minutes during a home move, not seconds.
+// actually heals a stale map is the edges-file cycle: the home boxes
+// republish within a couple of seconds of their registry changing
+// (internal/host's edgesFileLoop, since #174; before that only on a
+// 5min+jitter timer), and the sending box then picks that up on its 30s
+// mirror poll. So a sender can retry into this refusal for up to about half
+// a minute during a home move.
//
// The body says nothing. It is reachable only by a party holding the mesh key
// — an unauthenticated stranger asking for the same host gets the ordinary,
@@ -830,7 +832,7 @@ func stripForwardedHeaders(r *http.Request) {
//
// Silent until now was leaving a home move that gets stuck mid-way with
// nothing on this box's OWN journal to show it — a sender can retry into
-// this refusal for minutes (see above) and the only visibility was on the
+// this refusal for a while (see above) and the only visibility was on the
// SENDING side. This logs first-and-every-tenth per (box, host), the same
// cadence logPartialPoll (above) uses for its own standing condition; box is
// always m.self (there is exactly one homeMap per process, and it never
diff --git a/internal/edge/pending.go b/internal/edge/pending.go
new file mode 100644
index 000000000..bdc6dc99d
--- /dev/null
+++ b/internal/edge/pending.go
@@ -0,0 +1,222 @@
+package edge
+
+import (
+ "bytes"
+ "html/template"
+ "net/http"
+ "strings"
+)
+
+// The "not here YET" answer (#174): what an edge says for an unknown host
+// under one of this deployment's own domains.
+//
+// A host the platform just created is not servable everywhere at once. Its
+// home box starts serving it when the route lands in that box's registry;
+// every other edge learns where to backhaul it from the home box's edges file
+// (homemap.go), which reaches it seconds to half a minute later. A visitor
+// steered to one of those edges in that window used to get unknownHost's 404
+// (or the fallback redirect): a dead end for an address that would have
+// worked a minute later. titogo hit it on every account's first sandbox: the
+// app's readiness probe asked the home edge, got a 200, and redirected the
+// browser to a different edge that did not know the host yet.
+//
+// So a host under a platform domain gets 503 + Retry-After and a page that
+// retries by itself, and a host anywhere else keeps the old answer exactly.
+// The edge cannot tell "created a moment ago" from "never existed" — no edge
+// knows about a host before its home box does — so a typo'd or retired
+// subdomain gets the retry page too. That is why the retries are bounded:
+// after pendingRetryWindow the page gives up and shows the not-found text,
+// or goes to the fallback URL on an edge configured with one. A retired
+// address therefore still lands where it always did, pendingRetryWindow
+// later.
+//
+// Platform domains are the ones the operator configured: the static wildcard
+// parents (CARLOS_WILDCARD_DOMAINS, which may be a tenant's parent the
+// operator set up, like go.tito.io) and the apps domain
+// (PLATFORM_APPS_DOMAIN), read once at start. A customer's own domain stays on
+// the plain 404, because a customer domain pointing at the edge before its
+// alias is attached is a configuration state, not propagation lag.
+//
+// The retry state (when this tab first saw the page, and when it last did)
+// lives in sessionStorage, and in window.name where sessionStorage throws
+// (cookies blocked, some embedded webviews). Both survive a reload in one
+// tab and nothing else. If neither works, the page cannot tell its first
+// load from its hundredth, so it gives up on the first: a visitor with no
+// storage sees the not-found text or the fallback at once, never an endless
+// reload. The reload itself is
+// location.replace, so a form POST that landed here is not resubmitted and
+// the waiting page does not fill the back button.
+//
+// Crawlers see 503 for as long as a retired platform subdomain is asked
+// for, so they never de-index it; noindex keeps it out of results
+// meanwhile. That is the accepted cost of not knowing "new" from "gone".
+//
+// The page never echoes the host or anything else from the request, so it
+// carries nothing a request could inject. What changes with the request is
+// the status line only.
+
+const (
+ // pendingRetryAfter is the Retry-After an API client gets, and the
+ // page's own retry interval, in seconds.
+ pendingRetryAfter = "5"
+ // pendingRetryWindow is how long the page keeps retrying, in
+ // milliseconds, measured from the visitor's first sight of it in this
+ // tab. Propagation is at most about half a minute once the home box
+ // serves the host; 90s leaves room for the home box itself still
+ // converging.
+ pendingRetryWindowMS = 90_000
+ // pendingEpisodeGapMS ends one episode of waiting. A tab that last saw
+ // the page longer ago than this (because the host came up and it moved
+ // on, say) starts a fresh 90s window the next time, rather than giving
+ // up at once on a stale start time. Three retry intervals, so a slow
+ // reload never splits one episode in two.
+ pendingEpisodeGapMS = 15_000
+)
+
+// The page's words. Kept together so a copy change is one place.
+const (
+ pendingTitle = "Almost ready"
+ pendingHeading = "This address is still being set up"
+ pendingBody = "It usually takes less than a minute. This page will try again on its own."
+ pendingNoscript = "Reload this page in a minute."
+ goneHeading = "Nothing is answering at this address"
+ goneBody = "Check the address and try again."
+)
+
+var pendingPage = template.Must(template.New("pending").Parse(`<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<meta name="robots" content="noindex">
+<title>{{.Title}}</title>
+<style>
+body{font:16px/1.5 system-ui,sans-serif;max-width:32rem;margin:20vh auto 0;padding:0 1.5rem;color:#222}
+h1{font-size:1.25rem;font-weight:600;margin:0 0 .5rem}
+p{margin:0;color:#555}
+</style>
+</head>
+<body>
+<h1 id="h">{{.Heading}}</h1>
+<p id="b">{{.Body}}</p>
+<noscript><p>{{.Noscript}}</p></noscript>
+<script>
+(function(){
+ var k="carlos-pending:"+location.host, now=Date.now(), st=null;
+ function load(){
+ try{return JSON.parse(sessionStorage.getItem(k))}catch(e){}
+ try{var w=JSON.parse(window.name);if(w&&w.k===k)return w}catch(e){}
+ return null;
+ }
+ function save(v){
+ try{sessionStorage.setItem(k,JSON.stringify(v));return true}catch(e){}
+ try{v.k=k;window.name=JSON.stringify(v);return load()!==null}catch(e){}
+ return false;
+ }
+ st=load();
+ if(!st||!(now-st.last<={{.GapMS}}))st={first:now};
+ st.last=now;
+ if(now-st.first<{{.WindowMS}}&&save(st)){
+ setTimeout(function(){location.replace(location.href)},{{.RetryMS}});
+ return;
+ }
+ save({first:0,last:0});
+ var f={{.Fallback}};
+ if(f){location.replace(f);return}
+ document.title={{.GoneHeading}};
+ document.getElementById("h").textContent={{.GoneHeading}};
+ document.getElementById("b").textContent={{.GoneBody}};
+})();
+</script>
+</body>
+</html>
+`))
+
+// renderPendingPage renders the page once per edge: its only inputs are
+// constants and the fallback URL, which is fixed at start.
+func renderPendingPage(fallback string) []byte {
+ var buf bytes.Buffer
+ err := pendingPage.Execute(&buf, map[string]any{
+ "Title": pendingTitle,
+ "Heading": pendingHeading,
+ "Body": pendingBody,
+ "Noscript": pendingNoscript,
+ "GoneHeading": goneHeading,
+ "GoneBody": goneBody,
+ "WindowMS": pendingRetryWindowMS,
+ "RetryMS": 5000,
+ "GapMS": pendingEpisodeGapMS,
+ "Fallback": fallback,
+ })
+ if err != nil {
+ // The template and its inputs are fixed at build time; a failure
+ // here is a programming error, caught by the package's tests.
+ panic(err)
+ }
+ return buf.Bytes()
+}
+
+// platformDomainsFromEnv reads the domains the pending answer covers: the
+// static wildcard parents and the apps domain, lowercased and deduplicated.
+func platformDomainsFromEnv(getenv func(string) string) []string {
+ var out []string
+ for _, d := range append(splitDomains(getenv("CARLOS_WILDCARD_DOMAINS")),
+ splitDomains(getenv("PLATFORM_APPS_DOMAIN"))...) {
+ d = strings.TrimSuffix(d, ".")
+ if d != "" && !containsString(out, d) {
+ out = append(out, d)
+ }
+ }
+ return out
+}
+
+func containsString(list []string, s string) bool {
+ for _, v := range list {
+ if v == s {
+ return true
+ }
+ }
+ return false
+}
+
+// setPlatformDomains arms the pending answer for hosts under domains, and
+// renders its page against the fallback URL, which must already be set.
+func (e *Edge) setPlatformDomains(domains []string) {
+ e.platformDomains = domains
+ e.pendingBody = renderPendingPage(e.fallbackURL)
+}
+
+// underPlatformDomain reports whether host sits strictly BELOW one of the
+// platform domains. The domain itself is excluded: an unrouted apex is an
+// operator's configuration, not a host waiting to propagate.
+//
+// Only a host in the form the route table keys on counts: lowercase, no
+// trailing dot. Routing matches r.Host byte for byte (lookupIn), so
+// "Acme.go.tito.io" or "acme.go.tito.io." misses even when acme.go.tito.io
+// is served, and telling that visitor "still being set up" for 90 seconds
+// would be wrong. It keeps the plain unknown-host answer it always had.
+func (e *Edge) underPlatformDomain(host string) bool {
+ if host == "" || host != strings.ToLower(host) || strings.HasSuffix(host, ".") {
+ return false
+ }
+ for _, d := range e.platformDomains {
+ if strings.HasSuffix(host, "."+d) {
+ return true
+ }
+ }
+ return false
+}
+
+// pendingHost writes the 503 answer. HEAD gets the headers alone.
+func (e *Edge) pendingHost(w http.ResponseWriter, r *http.Request) {
+ h := w.Header()
+ h.Set("Content-Type", "text/html; charset=utf-8")
+ // no-store: this answer is about a moment, and a cache holding it would
+ // keep a visitor on it after the host became servable.
+ h.Set("Cache-Control", "no-store")
+ h.Set("Retry-After", pendingRetryAfter)
+ w.WriteHeader(http.StatusServiceUnavailable)
+ if r.Method != http.MethodHead {
+ w.Write(e.pendingBody) //nolint:errcheck
+ }
+}
diff --git a/internal/edge/pending_script_test.go b/internal/edge/pending_script_test.go
new file mode 100644
index 000000000..bb22421f4
--- /dev/null
+++ b/internal/edge/pending_script_test.go
@@ -0,0 +1,136 @@
+package edge
+
+import (
+ "encoding/json"
+ "os/exec"
+ "regexp"
+ "testing"
+)
+
+// The pending page's retry script, run for real under node against a fake
+// browser: storage, window.name, location, the clock and setTimeout. The
+// substring checks in pending_test.go pin the markup; these pin what the
+// script DOES. Skipped where node is not installed.
+
+type jsTab struct {
+ Storage string `json:"storage"` // "ok", "throws"
+ WindowName string `json:"windowName"`
+ WindowNames bool `json:"windowNameWorks"`
+ Session string `json:"session"` // prior sessionStorage value for the key
+ Now int64 `json:"now"`
+}
+
+type jsOutcome struct {
+ Reload bool `json:"reload"`
+ Replaced string `json:"replaced"`
+ Heading string `json:"heading"`
+ Session string `json:"session"`
+ Window string `json:"windowName"`
+}
+
+var scriptRE = regexp.MustCompile(`(?s)<script>(.*)</script>`)
+
+func runPendingScript(t *testing.T, fallback string, tab jsTab) jsOutcome {
+ t.Helper()
+ node, err := exec.LookPath("node")
+ if err != nil {
+ t.Skip("node not installed")
+ }
+ m := scriptRE.FindSubmatch(renderPendingPage(fallback))
+ if m == nil {
+ t.Fatal("no script in the pending page")
+ }
+ in, _ := json.Marshal(tab)
+ prog := `
+const tab = ` + string(in) + `;
+const out = {reload:false, replaced:"", heading:"", session:"", windowName:""};
+const store = {};
+if (tab.session) store["carlos-pending:x.go.tito.example"] = tab.session;
+global.sessionStorage = {
+ getItem(k){ if (tab.storage==="throws") throw new Error("SecurityError"); return k in store ? store[k] : null; },
+ setItem(k,v){ if (tab.storage==="throws") throw new Error("SecurityError"); store[k]=String(v); },
+};
+let wn = tab.windowName;
+global.window = {};
+Object.defineProperty(global.window, "name", {
+ get(){ return wn; },
+ set(v){ if (tab.windowNameWorks) wn = String(v); },
+});
+global.location = {
+ host: "x.go.tito.example", href: "https://x.go.tito.example/p",
+ replace(u){ if (u === this.href) out.reload = true; else out.replaced = u; },
+};
+Date.now = () => tab.now;
+global.setTimeout = (fn) => fn();
+const els = {h:{textContent:""}, b:{textContent:""}};
+global.document = { title:"", getElementById(id){ return els[id]; } };
+` + string(m[1]) + `
+out.heading = els.h.textContent;
+out.session = store["carlos-pending:x.go.tito.example"] || "";
+out.windowName = wn;
+console.log(JSON.stringify(out));
+`
+ cmd := exec.Command(node, "-e", prog)
+ b, err := cmd.CombinedOutput()
+ if err != nil {
+ t.Fatalf("node: %v\n%s", err, b)
+ }
+ var o jsOutcome
+ if err := json.Unmarshal(b, &o); err != nil {
+ t.Fatalf("node output %q: %v", b, err)
+ }
+ return o
+}
+
+func TestPendingScriptFirstLoadRetries(t *testing.T) {
+ o := runPendingScript(t, "", jsTab{Storage: "ok", Now: 1_000_000})
+ if !o.Reload || o.Heading != "" {
+ t.Fatalf("first load did not schedule a reload: %+v", o)
+ }
+}
+
+func TestPendingScriptGivesUpAfterTheWindow(t *testing.T) {
+ // Seen continuously (last 5s ago) since 95s ago.
+ o := runPendingScript(t, "", jsTab{Storage: "ok", Now: 1_000_000,
+ Session: `{"first":905000,"last":995000}`})
+ if o.Reload || o.Heading != goneHeading {
+ t.Fatalf("did not give up after the window: %+v", o)
+ }
+}
+
+func TestPendingScriptGivesUpToTheFallback(t *testing.T) {
+ o := runPendingScript(t, "https://www.example.com/", jsTab{Storage: "ok", Now: 1_000_000,
+ Session: `{"first":905000,"last":995000}`})
+ if o.Replaced != "https://www.example.com/" {
+ t.Fatalf("did not go to the fallback: %+v", o)
+ }
+}
+
+func TestPendingScriptStaleStartIsAFreshEpisode(t *testing.T) {
+ // First seen long ago, but not seen for a minute: a new episode.
+ o := runPendingScript(t, "", jsTab{Storage: "ok", Now: 1_000_000,
+ Session: `{"first":100000,"last":940000}`})
+ if !o.Reload {
+ t.Fatalf("a stale start time made the page give up at once: %+v", o)
+ }
+}
+
+func TestPendingScriptFallsBackToWindowName(t *testing.T) {
+ o := runPendingScript(t, "", jsTab{Storage: "throws", WindowNames: true, Now: 1_000_000})
+ if !o.Reload || o.Window == "" {
+ t.Fatalf("with sessionStorage throwing, window.name was not used: %+v", o)
+ }
+ // ...and window.name carries the window across reloads.
+ o = runPendingScript(t, "", jsTab{Storage: "throws", WindowNames: true, Now: 1_000_000,
+ WindowName: `{"first":905000,"last":995000,"k":"carlos-pending:x.go.tito.example"}`})
+ if o.Reload || o.Heading != goneHeading {
+ t.Fatalf("window.name state did not end the window: %+v", o)
+ }
+}
+
+func TestPendingScriptWithNoStorageNeverLoops(t *testing.T) {
+ o := runPendingScript(t, "", jsTab{Storage: "throws", WindowNames: false, Now: 1_000_000})
+ if o.Reload || o.Heading != goneHeading {
+ t.Fatalf("with no storage at all the page reloaded instead of giving up: %+v", o)
+ }
+}
diff --git a/internal/edge/pending_test.go b/internal/edge/pending_test.go
new file mode 100644
index 000000000..62b875eb6
--- /dev/null
+++ b/internal/edge/pending_test.go
@@ -0,0 +1,167 @@
+package edge
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+)
+
+// The "not here YET" answer (#174).
+
+// pendingEdge is an edge serving one route, with the pending answer armed
+// for go.tito.example and apps.example.
+func pendingEdge(t *testing.T, fallback string) *Edge {
+ t.Helper()
+ e := purgeHostsEdge(t, titoAccount(), func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ })
+ e.fallbackURL = fallback
+ e.setPlatformDomains(platformDomainsFromEnv(func(k string) string {
+ return map[string]string{
+ "CARLOS_WILDCARD_DOMAINS": "go.tito.example, GO.TITO.EXAMPLE",
+ "PLATFORM_APPS_DOMAIN": "apps.example.",
+ }[k]
+ }))
+ return e
+}
+
+func get(e *Edge, method, url string) *httptest.ResponseRecorder {
+ rec := httptest.NewRecorder()
+ e.ServeHTTP(rec, httptest.NewRequest(method, url, nil))
+ return rec
+}
+
+// TestPendingHostUnderAPlatformDomainIs503 is the headline: a host the edge
+// has not heard of yet, under a platform domain, gets a retryable answer
+// rather than a dead end.
+func TestPendingHostUnderAPlatformDomainIs503(t *testing.T) {
+ e := pendingEdge(t, "")
+ for _, url := range []string{
+ "https://sb-andyet.go.tito.example/dashboard",
+ "https://acct.apps.example/",
+ "https://app.acct.apps.example/x?y=1",
+ } {
+ rec := get(e, "GET", url)
+ if rec.Code != http.StatusServiceUnavailable {
+ t.Fatalf("%s: status %d, want 503", url, rec.Code)
+ }
+ if got := rec.Header().Get("Retry-After"); got != pendingRetryAfter {
+ t.Fatalf("%s: Retry-After %q, want %q", url, got, pendingRetryAfter)
+ }
+ if got := rec.Header().Get("Cache-Control"); got != "no-store" {
+ t.Fatalf("%s: Cache-Control %q, want no-store", url, got)
+ }
+ body := rec.Body.String()
+ if !strings.Contains(body, pendingHeading) || !strings.Contains(body, "location.replace(location.href)") {
+ t.Fatalf("%s: body is not the retrying pending page:\n%s", url, body)
+ }
+ }
+}
+
+// TestPendingPageNeverEchoesTheRequest — the host, path and query are
+// attacker-chosen for an unknown host; none of them may reach the page.
+func TestPendingPageNeverEchoesTheRequest(t *testing.T) {
+ e := pendingEdge(t, "")
+ rec := get(e, "GET", "https://xss-probe.go.tito.example/%3Cscript%3E?q=%3Cimg%3E")
+ body := rec.Body.String()
+ for _, s := range []string{"xss-probe", "<script>alert", "%3C", "<img"} {
+ if strings.Contains(body, s) {
+ t.Fatalf("the pending page echoed %q from the request:\n%s", s, body)
+ }
+ }
+}
+
+// TestPendingHostHEADHasNoBody — the headers alone.
+func TestPendingHostHEADHasNoBody(t *testing.T) {
+ e := pendingEdge(t, "")
+ rec := get(e, "HEAD", "https://sb-new.go.tito.example/")
+ if rec.Code != http.StatusServiceUnavailable || rec.Body.Len() != 0 {
+ t.Fatalf("HEAD: status %d with a %d-byte body, want 503 and none", rec.Code, rec.Body.Len())
+ }
+}
+
+// TestUnknownHostOutsidePlatformDomainsIsUnchanged — a customer domain, and a
+// platform domain's own apex, keep the old answers exactly: 404 without a
+// fallback, the 302 with one.
+func TestUnknownHostOutsidePlatformDomainsIsUnchanged(t *testing.T) {
+ for _, tc := range []struct {
+ fallback string
+ want int
+ }{{"", http.StatusNotFound}, {"https://www.example.com/", http.StatusFound}} {
+ e := pendingEdge(t, tc.fallback)
+ for _, url := range []string{
+ "https://customer.example.org/",
+ "https://apps.example/", // the apex itself (go.tito.example is a served route here)
+ "https://evilgo.tito.example/", // not a label boundary of go.tito.example
+ } {
+ if rec := get(e, "GET", url); rec.Code != tc.want {
+ t.Fatalf("fallback %q, %s: status %d, want %d", tc.fallback, url, rec.Code, tc.want)
+ }
+ }
+ }
+}
+
+// TestPendingPageGivesUpToTheFallback — an edge with a fallback URL still
+// sends a mistyped or retired platform subdomain there, after the retry
+// window, so the fallback keeps doing its job. The URL is embedded as a JS
+// string, escaped.
+func TestPendingPageGivesUpToTheFallback(t *testing.T) {
+ e := pendingEdge(t, `https://www.example.com/?a=1&b="x"</script>`)
+ body := get(e, "GET", "https://typo.go.tito.example/").Body.String()
+ if !strings.Contains(body, "location.replace(f)") {
+ t.Fatalf("the page does not go to the fallback when it gives up:\n%s", body)
+ }
+ if strings.Count(body, "</script>") != 1 {
+ t.Fatalf("the fallback URL broke out of the script element:\n%s", body)
+ }
+ if !strings.Contains(body, `"https://www.example.com/?a=1`+"\\u0026"+`b=\"x\"`+"\\u003c"+`/script`+"\\u003e"+`"`) {
+ t.Fatalf("the fallback URL is not embedded as an escaped JS string:\n%s", body)
+ }
+}
+
+// TestPendingAnswerIsOffWithoutPlatformDomains — New alone (no env) arms
+// nothing: every unknown host is the plain 404, as before #174.
+func TestPendingAnswerIsOffWithoutPlatformDomains(t *testing.T) {
+ e := purgeHostsEdge(t, titoAccount(), func(w http.ResponseWriter, r *http.Request) {})
+ if rec := get(e, "GET", "https://sb-new.go.tito.example/"); rec.Code != http.StatusNotFound {
+ t.Fatalf("status %d, want 404 with no platform domains configured", rec.Code)
+ }
+}
+
+// TestKnownHostUnderAPlatformDomainIsServed — the pending answer is only
+// ever the miss branch.
+func TestKnownHostUnderAPlatformDomainIsServed(t *testing.T) {
+ e := pendingEdge(t, "")
+ if rec := get(e, "GET", "https://acme.go.tito.example/"); rec.Code != http.StatusOK {
+ t.Fatalf("a served host under a platform domain got %d", rec.Code)
+ }
+}
+
+// TestPendingHostOnPlainHTTP — :80 for an unknown platform-domain host gets
+// the same answer rather than a redirect into a handshake.
+func TestPendingHostOnPlainHTTP(t *testing.T) {
+ e := pendingEdge(t, "")
+ rec := httptest.NewRecorder()
+ e.httpFallbackHandler().ServeHTTP(rec, httptest.NewRequest("GET", "http://sb-new.go.tito.example/", nil))
+ if rec.Code != http.StatusServiceUnavailable {
+ t.Fatalf(":80 status %d, want 503", rec.Code)
+ }
+}
+
+// TestPendingAnswerNeedsTheRouteTableForm — routing matches r.Host byte for
+// byte, so an uppercase or trailing-dot spelling of a platform host can miss
+// even when the host is served. Those keep the plain unknown-host answer
+// rather than 90 seconds of "still being set up".
+func TestPendingAnswerNeedsTheRouteTableForm(t *testing.T) {
+ e := pendingEdge(t, "")
+ for _, host := range []string{"Acme.go.tito.example", "acme.go.tito.example.", "SB-NEW.GO.TITO.EXAMPLE"} {
+ req := httptest.NewRequest("GET", "https://x/", nil)
+ req.Host = host
+ rec := httptest.NewRecorder()
+ e.ServeHTTP(rec, req)
+ if rec.Code != http.StatusNotFound {
+ t.Fatalf("%s: status %d, want the plain 404", host, rec.Code)
+ }
+ }
+}
diff --git a/internal/edge/steeringprobe.go b/internal/edge/steeringprobe.go
index ac5c8fa05..4af1bc09a 100644
--- a/internal/edge/steeringprobe.go
+++ b/internal/edge/steeringprobe.go
@@ -34,7 +34,9 @@ package edge
// 302, which Route53 counts as HEALTHY. A false-healthy check is worse
// than no check: it never withdraws anything, and nothing in the journal
// says so. Hence the answer below, placed AHEAD of every other serving
-// path.
+// path. (Since #174 the name, one label under the apps domain, would get
+// the pending page's 503 instead, which Route53 scores unhealthy on
+// every edge: wrong the other way. The probe still answers first.)
//
// Inert unless the deployment steers: steeringProbeHostFromEnv returns ""
// unless CARLOS_STEERING_ZONE_ID is set AND usable — the same read, with the
diff --git a/internal/edge/wildcard.go b/internal/edge/wildcard.go
index eb78a74e9..343de8e0f 100644
--- a/internal/edge/wildcard.go
+++ b/internal/edge/wildcard.go
@@ -798,7 +798,9 @@ func newWildcardFromEnv(e *Edge, cache autocert.Cache) *wildcardManager {
// COSTS: a stranger's one-label unknown host under the apps domain
// (anything.<apps-domain>) now COMPLETES the TLS handshake — certFor
// serves the wildcard before HostPolicy is ever consulted — and reaches
- // the ordinary unknown-host answer, a 404 or a -fallback 302. Before this
+ // the ordinary unknown-host answer: since #174 the pending page's 503
+ // (pending.go), which gives up to the not-found text or the -fallback
+ // URL after its retry window; before that a 404 or a -fallback 302. Before this
// parent existed it failed at the handshake instead, because autocert's
// policy refuses a name that is not a registry route. Nothing is
// disclosed that a routed host does not already disclose, and every
diff --git a/internal/host/edgesfile.go b/internal/host/edgesfile.go
index 8a6aab585..2c9c666dd 100644
--- a/internal/host/edgesfile.go
+++ b/internal/host/edgesfile.go
@@ -422,17 +422,40 @@ func edgesFileRegion(getenv func(string) string, probedRegion string) string {
}
// edgesFileLoop publishes this box's edges file every 5 minutes + jitter —
-// the same cadence and arming site as registrySnapshotLoop. Home moves ARE
-// placement events, not per-request events, but the comment this replaces
-// overstated how fast that heals: there is no 2s serial-converge path for
-// this file (edges never bump control/serial — homemap.go's staleHopRefusal
-// doc makes the identical correction on the reader's side), so what
-// actually heals a stale home map is this loop's own 5min+jitter cycle
-// followed by each remote edge's 30s mirror poll (internal/edge's
-// homeMap) — minutes during a home move, not seconds. Platform-box job,
-// retentionLoop's rules: skipped on fleet boxes, disabled with a log line
-// when there is no deployment bucket. Single-box deployments: the file is
-// published and nothing reads it yet — inert until Task 3's mirror ships.
+// the same cadence and arming site as registrySnapshotLoop — AND within one
+// watch interval of any change to this box's registry (#174).
+//
+// The change path is what a new host needs. Every other edge learns that this
+// box homes a host from this file (internal/edge's homeMap mirror, a 30s
+// poll), and until it has, a visitor steered to one of them gets that edge's
+// unknown-host answer. On the timer alone that window was the whole 5min +
+// jitter cycle plus the poll: a titogo sandbox host served here at +51s was a
+// dead end on the uk edge for about a minute more, and could have been for
+// six. Watching the registry fingerprint (the same 2s poll edge.Watch
+// reloads on) closes the writer's half to seconds; what is left is the
+// reader's 30s poll.
+//
+// It stays cheap because edgesFilePass is: local registry reads, and a
+// bucket write only when the file's body (Generation aside) changed. A
+// deploy or a health flip moves the fingerprint without moving the hosts,
+// so it costs a pass and no write.
+//
+// The trigger covers what a new host needs, not every input to the file.
+// Fingerprint hashes a route's row and status (so a route turning ready
+// moves it), aliases' names and wildcard_domains; it does not hash a route's
+// platform, provisioned or kind columns, nor an alias's dns_status. A change
+// to only those still waits for the timer, and so does a newly attached
+// custom domain: its pass runs at once but withholds it until the DNS sweep
+// stamps it "pointing".
+//
+// The timer is kept for what the fingerprint deliberately does not see:
+// route_aliases' dns_status (a cadence-stamped column Fingerprint excludes)
+// decides which aliases are published, and a failed write must be retried
+// even if nothing else changes.
+//
+// Platform-box job, retentionLoop's rules: skipped on fleet boxes, disabled
+// with a log line when there is no deployment bucket. Single-box deployments:
+// the file is published and nothing reads it yet.
func edgesFileLoop(reg *registry.Registry) {
ctx := context.Background()
if shouldSkipPublisherElection(os.Getenv) {
@@ -466,17 +489,127 @@ func edgesFileLoop(reg *registry.Registry) {
// ask two questions that could drift.
HibernationConfigured: strings.TrimSpace(os.Getenv("CARLOS_HIBERNATE_BUCKET")) != "",
}
- log.Printf("agent: edges file loop armed (5min + jitter, %s)", edgesFileKey(box))
+ log.Printf("agent: edges file loop armed (5min + jitter, and %s after a registry change, %s)", edgesFileWatchInterval, edgesFileKey(box))
last := seedEdgesFileGate(ctx, store, box)
- delay := time.Minute
- for {
- time.Sleep(delay)
- delay = 5*time.Minute + rand.N(time.Minute)
- eip := readPublicIP()
- if err := edgesFilePass(ctx, store, reg, ident, eip, &last); err != nil {
- log.Printf("agent: edges file pass: %v", err)
+ sched := newEdgesFileSchedule(time.Now(), func() time.Duration {
+ return 5*time.Minute + rand.N(time.Minute)
+ })
+ tick := time.NewTicker(edgesFileWatchInterval)
+ defer tick.Stop()
+ fpErrs, passErrs := 0, 0
+ // The EIP cannot change under a running box, so it is read once here and
+ // re-read only on the timer's passes, never on every change pass: off
+ // EC2 each read can block for the probe's whole timeout.
+ eip := readPublicIP()
+ for now := range tick.C {
+ fp, err := reg.Fingerprint()
+ if err != nil {
+ // Not fatal: the timer still publishes. Counted and throttled the
+ // way edge.Watch counts its own poll errors.
+ fpErrs++
+ if fpErrs == 1 || fpErrs%300 == 0 {
+ log.Printf("agent: edges file registry poll error (#%d): %v", fpErrs, err)
+ }
+ } else {
+ fpErrs = 0
+ }
+ if !sched.due(now, fp, err == nil) {
+ continue
}
+ if sched.timed {
+ eip = readPublicIP()
+ }
+ err = edgesFilePass(ctx, store, reg, ident, eip, &last)
+ if err != nil {
+ // Throttled: a bucket that refuses every write (an IAM Deny, an
+ // outage) is retried on a backoff, and each retry must not be a
+ // journal line.
+ passErrs++
+ if passErrs == 1 || passErrs%30 == 0 {
+ log.Printf("agent: edges file pass failed (#%d): %v", passErrs, err)
+ }
+ } else {
+ passErrs = 0
+ }
+ sched.done(now, err == nil)
+ }
+}
+
+// edgesFileWatchInterval is how often edgesFileLoop polls the registry
+// fingerprint: edge.Watch's own 2s, so this file follows a route change on
+// the same beat the local edge starts serving it.
+const edgesFileWatchInterval = 2 * time.Second
+
+// edgesFileSchedule decides, tick by tick, whether edgesFileLoop runs a
+// pass. Pulled out of the loop so the decision is testable without a
+// bucket, a registry or a clock.
+//
+// A pass is due when the timer is up, or when the fingerprint moved since
+// the last SUCCESSFUL pass. The first fingerprint seen only baselines: the
+// first timed pass stays a minute after arm, as it always was, rather than
+// publishing in the loop's first two seconds.
+//
+// The baseline advances only when a pass succeeds, so a change whose pass
+// failed is retried rather than lost until the timer — the rule edge.Watch
+// keeps for a failed Reload. Unlike a Reload, a pass is a bucket write, so
+// the retries back off: 2s doubling to the timer's own cadence, reset by the
+// next success. A bucket that refuses every write is therefore asked about
+// as often as it was before the change trigger existed, not every tick.
+type edgesFileSchedule struct {
+ next time.Time // when the timer is next due
+ jitter func() time.Duration // the cadence after each timed pass
+ seen string // the fingerprint on the tick being decided
+ baseline string // the fingerprint the last good pass covered
+ armed bool // a baseline exists
+ timed bool // the pass being run is the timer's
+ retryAt time.Time // no change pass before this, after a failure
+ backoff time.Duration // the wait after the next failure
+}
+
+// Bounds on the change-pass retry backoff.
+const (
+ edgesFileRetryMin = 2 * time.Second
+ edgesFileRetryMax = 5 * time.Minute
+)
+
+func newEdgesFileSchedule(now time.Time, jitter func() time.Duration) *edgesFileSchedule {
+ return &edgesFileSchedule{next: now.Add(time.Minute), jitter: jitter}
+}
+
+// due reports whether to run a pass on this tick. fpOK=false means the
+// fingerprint could not be read, which never triggers a change pass.
+func (s *edgesFileSchedule) due(now time.Time, fp string, fpOK bool) bool {
+ s.timed = !now.Before(s.next)
+ s.seen = ""
+ if fpOK {
+ s.seen = fp
+ if !s.armed {
+ s.baseline, s.armed = fp, true
+ }
+ }
+ changed := fpOK && fp != s.baseline && !now.Before(s.retryAt)
+ return s.timed || changed
+}
+
+// done records the outcome of the pass due just asked for.
+func (s *edgesFileSchedule) done(now time.Time, ok bool) {
+ if s.timed {
+ // The timer moves on whatever happened: a failed timed pass is
+ // retried by the next timed one, exactly as before this schedule.
+ s.next = now.Add(s.jitter())
+ }
+ if ok {
+ if s.seen != "" {
+ s.baseline = s.seen
+ }
+ s.retryAt, s.backoff = time.Time{}, 0
+ return
+ }
+ if s.backoff == 0 {
+ s.backoff = edgesFileRetryMin
}
+ s.retryAt = now.Add(s.backoff)
+ s.backoff = min(2*s.backoff, edgesFileRetryMax)
}
// aliasClaimCache is the claim fold both alias legs share (the fast-converge
diff --git a/internal/host/edgesfile_schedule_test.go b/internal/host/edgesfile_schedule_test.go
new file mode 100644
index 000000000..e67392b71
--- /dev/null
+++ b/internal/host/edgesfile_schedule_test.go
@@ -0,0 +1,152 @@
+package host
+
+import (
+ "testing"
+ "time"
+)
+
+// The edges file schedule (#174): a pass within one watch tick of a registry
+// change, the old timer kept, and a failed change pass retried.
+
+func newTestSchedule(t0 time.Time) *edgesFileSchedule {
+ return newEdgesFileSchedule(t0, func() time.Duration { return 5 * time.Minute })
+}
+
+// TestEdgesFileScheduleFirstFingerprintOnlyBaselines — arming must not
+// publish in the loop's first two seconds; the first timed pass stays a
+// minute after arm, as it was before the change trigger existed.
+func TestEdgesFileScheduleFirstFingerprintOnlyBaselines(t *testing.T) {
+ t0 := time.Unix(1_000_000, 0)
+ s := newTestSchedule(t0)
+ if s.due(t0.Add(2*time.Second), "fp-a", true) {
+ t.Fatal("the first fingerprint triggered a pass; it should only baseline")
+ }
+ if s.due(t0.Add(30*time.Second), "fp-a", true) {
+ t.Fatal("an unchanged fingerprint triggered a pass before the timer")
+ }
+ if !s.due(t0.Add(time.Minute), "fp-a", true) {
+ t.Fatal("the first timed pass did not come due a minute after arm")
+ }
+}
+
+// TestEdgesFileScheduleChangeTriggersAPass is the headline: a route written
+// to the registry is published on the next tick, not up to six minutes
+// later.
+func TestEdgesFileScheduleChangeTriggersAPass(t *testing.T) {
+ t0 := time.Unix(1_000_000, 0)
+ s := newTestSchedule(t0)
+ s.due(t0.Add(2*time.Second), "fp-a", true)
+ now := t0.Add(4 * time.Second)
+ if !s.due(now, "fp-b", true) {
+ t.Fatal("a fingerprint change did not trigger a pass")
+ }
+ s.done(now, true)
+ if s.due(t0.Add(6*time.Second), "fp-b", true) {
+ t.Fatal("the same fingerprint triggered a second pass after a good one")
+ }
+ // A change pass does not move the timer: it is still due at arm + 1min.
+ if !s.due(t0.Add(time.Minute), "fp-b", true) {
+ t.Fatal("a change pass pushed the timer back")
+ }
+}
+
+// TestEdgesFileScheduleFailedChangePassIsRetried — the baseline advances only
+// on a successful pass, so a failed bucket write is retried after the first
+// backoff step (2s) instead of waiting for the timer.
+func TestEdgesFileScheduleFailedChangePassIsRetried(t *testing.T) {
+ t0 := time.Unix(1_000_000, 0)
+ s := newTestSchedule(t0)
+ s.due(t0.Add(2*time.Second), "fp-a", true)
+ s.due(t0.Add(4*time.Second), "fp-b", true)
+ s.done(t0.Add(4*time.Second), false)
+ if !s.due(t0.Add(6*time.Second), "fp-b", true) {
+ t.Fatal("a change whose pass failed was not retried on the next tick")
+ }
+ s.done(t0.Add(6*time.Second), true)
+ if s.due(t0.Add(8*time.Second), "fp-b", true) {
+ t.Fatal("a change was passed again after its retry succeeded")
+ }
+}
+
+// TestEdgesFileScheduleTimerSurvivesAnUnreadableRegistry — a fingerprint
+// error never triggers a change pass, and never stops the timer.
+func TestEdgesFileScheduleTimerSurvivesAnUnreadableRegistry(t *testing.T) {
+ t0 := time.Unix(1_000_000, 0)
+ s := newTestSchedule(t0)
+ s.due(t0.Add(2*time.Second), "fp-a", true)
+ if s.due(t0.Add(4*time.Second), "", false) {
+ t.Fatal("an unreadable fingerprint triggered a pass")
+ }
+ now := t0.Add(time.Minute)
+ if !s.due(now, "", false) {
+ t.Fatal("the timer did not fire while the fingerprint was unreadable")
+ }
+ s.done(now, true)
+ // The timed pass could not vouch for a fingerprint, so the baseline is
+ // still fp-a: a later read of fp-a is not a change, fp-b is.
+ if s.due(now.Add(2*time.Second), "fp-a", true) {
+ t.Fatal("the baseline moved on a pass that saw no fingerprint")
+ }
+ if !s.due(now.Add(4*time.Second), "fp-b", true) {
+ t.Fatal("a change after an unreadable spell was missed")
+ }
+}
+
+// TestEdgesFileScheduleTimerAdvancesOnFailure — a failed timed pass moves the
+// timer on by the cadence, exactly as the old sleep loop did.
+func TestEdgesFileScheduleTimerAdvancesOnFailure(t *testing.T) {
+ t0 := time.Unix(1_000_000, 0)
+ s := newTestSchedule(t0)
+ now := t0.Add(time.Minute)
+ if !s.due(now, "fp-a", true) {
+ t.Fatal("timer not due")
+ }
+ s.done(now, false)
+ if s.due(now.Add(2*time.Minute), "fp-a", true) {
+ t.Fatal("a failed timed pass did not move the timer on")
+ }
+ if !s.due(now.Add(5*time.Minute), "fp-a", true) {
+ t.Fatal("the timer did not come due one cadence after the failed pass")
+ }
+}
+
+// TestEdgesFileScheduleFailedChangePassesBackOff — a bucket refusing every
+// write is retried on a doubling backoff, not every tick, and a success
+// resets it.
+func TestEdgesFileScheduleFailedChangePassesBackOff(t *testing.T) {
+ t0 := time.Unix(1_000_000, 0)
+ s := newTestSchedule(t0)
+ s.due(t0, "fp-a", true)
+ now := t0.Add(2 * time.Second)
+ if !s.due(now, "fp-b", true) {
+ t.Fatal("change not due")
+ }
+ s.done(now, false) // retry no sooner than +2s
+ if s.due(now.Add(time.Second), "fp-b", true) {
+ t.Fatal("a failed change pass was retried inside its backoff")
+ }
+ now = now.Add(2 * time.Second)
+ if !s.due(now, "fp-b", true) {
+ t.Fatal("not retried once the 2s backoff passed")
+ }
+ s.done(now, false) // now +4s
+ if s.due(now.Add(3*time.Second), "fp-b", true) {
+ t.Fatal("the backoff did not double")
+ }
+ now = now.Add(4 * time.Second)
+ if !s.due(now, "fp-b", true) {
+ t.Fatal("not retried once the 4s backoff passed")
+ }
+ s.done(now, true)
+ s.due(now.Add(2*time.Second), "fp-b", true)
+ later := now.Add(4 * time.Second)
+ if !s.due(later, "fp-c", true) {
+ t.Fatal("a success did not reset the backoff")
+ }
+ for i := 0; i < 20; i++ {
+ s.done(later, false)
+ }
+ if s.backoff > edgesFileRetryMax {
+ t.Fatalf("backoff %s grew past its cap %s", s.backoff, edgesFileRetryMax)
+ }
+}