amadan
host+edge: stage-5 gates — seeded edges gate, region field, stuck-state visibility
edgesFileLoop now seeds *last from the bucket at arm (seedEdgesFileGate, mirroring seedSnapshotFingerprint's contract): a crash-looping agent no longer rewrites an unchanged edges file on every restart, and a restart that hits a transient IMDS blank on its very first tick no longer blanks a known-good EIP (plan-2 final review, Important 1, both halves). edgesFile gains an additive "region" field (omitempty, CARLOS_REGION verbatim, read once at loop arm per the domains loop's precedent) for Task 4's steering converger. It participates in the change gate like every other field, so a seeded pre-region body and a fresh body with region differ in bytes and write exactly once. staleHopRefusal and the backhaul ErrorHandler were both silent; a stuck home move can leave a sender retrying for minutes (edges never bump control/serial). staleHopRefusal now logs first-and-every-tenth per (box, host), following homemap.go's own partial-poll cadence; the backhaul ErrorHandler routes through e.logThrottled keyed "backhaul-error "+box. Also fixes edgesfile.go's loop doc, which claimed converge heals a stale home map faster than this loop — there is no such path (edges never bump control/serial); the real heal is this loop's 5min+jitter cycle plus the mirror's 30s poll, the same correction already made in homemap.go's staleHopRefusal doc.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
5 files changed,
+446
−29
internal/edge/edge.go+15 −6internal/edge/homemap.go+42 −1internal/edge/homemap_test.go+83 −0internal/host/edgesfile.go+72 −12internal/host/edgesfile_test.go+234 −10
diff --git a/internal/edge/edge.go b/internal/edge/edge.go| index bb69f7d43..31d8e8b14 100644 |
| --- a/internal/edge/edge.go |
| +++ b/internal/edge/edge.go |
| @@ -832,7 +832,7 @@ func (e *Edge) ServeHTTP(w http.ResponseWriter, r *http.Request) { |
| // accepted hop physically cannot reach it. (e.backhaul opens with |
| // the same refusal anyway, so moving this branch cannot open a |
| // forwarding loop either — see its doc.) |
| - staleHopRefusal(w) |
| + e.homes.Load().staleHopRefusal(w, host) |
| return |
| } |
| // Nothing here serves this host — but another box in the deployment |
| @@ -1807,11 +1807,11 @@ const ( |
| // instead of being forwarded onward. A forwarding loop must take TWO mistakes, |
| // not one. |
| func (e *Edge) backhaul(w http.ResponseWriter, r *http.Request, host string) bool { |
| + m := e.homes.Load() |
| if hopVerified(r.Context()) { |
| - staleHopRefusal(w) |
| + m.staleHopRefusal(w, host) |
| return true |
| } |
| - m := e.homes.Load() |
| eip, box, ok := m.Home(host) |
| if !ok { |
| return false |
| @@ -1832,7 +1832,7 @@ func (e *Edge) backhaul(w http.ResponseWriter, r *http.Request, host string) boo |
| // that may have matched it — because the home edge verifies against its own |
| // literal request host (hopMAC canonicalizes both, mesh.go). |
| auth := signHop(key, host, time.Now()) |
| - backhaulProxy(host, eip, box, auth, e.backhaulTransport(eip)).ServeHTTP(w, r) |
| + backhaulProxy(host, eip, box, auth, e.backhaulTransport(eip), e.logThrottled).ServeHTTP(w, r) |
| return true |
| } |
| @@ -1935,7 +1935,16 @@ func (e *Edge) backhaulTransport(eip string) *http.Transport { |
| // it would be if the request had landed on the home box directly. (The restore |
| // branch cannot fire on THIS side: backhaul only runs when the request did not |
| // arrive as a verified hop.) |
| -func backhaulProxy(host, eip, box, auth string, tr http.RoundTripper) *httputil.ReverseProxy { |
| +// |
| +// logThrottled is e.logThrottled, threaded through rather than closed over |
| +// directly: this is a free function (built fresh per request, unlike the |
| +// cached backhaulTransport) so it takes no *Edge, and a dial failure is a |
| +// STANDING condition for as long as a home box is unreachable — an |
| +// unthrottled log.Printf here would fill the journal with one line per |
| +// failed request for the whole outage. Keyed "backhaul-error "+box (per home |
| +// box, not per host — a wildcard home behind one unreachable box would |
| +// otherwise mint a fresh throttle bucket per invented hostname). |
| +func backhaulProxy(host, eip, box, auth string, tr http.RoundTripper, logThrottled func(key, msg string)) *httputil.ReverseProxy { |
| return &httputil.ReverseProxy{ |
| Rewrite: func(pr *httputil.ProxyRequest) { |
| setForwarded(pr) |
| @@ -1959,7 +1968,7 @@ func backhaulProxy(host, eip, box, auth string, tr http.RoundTripper) *httputil. |
| // against a cert the mirror has not distributed yet, or a home edge |
| // that hung up mid-response are all genuinely bad gateways — and the |
| // cert case heals itself on the next mirror poll. |
| - log.Printf("edge: backhaul %s → %s (%s): %v", host, box, eip, err) |
| + logThrottled("backhaul-error "+box, fmt.Sprintf("backhaul %s → %s (%s): %v", host, box, eip, err)) |
| w.WriteHeader(http.StatusBadGateway) |
| }, |
| } |
diff --git a/internal/edge/homemap.go b/internal/edge/homemap.go| index dec82865a..cea4be36a 100644 |
| --- a/internal/edge/homemap.go |
| +++ b/internal/edge/homemap.go |
| @@ -172,6 +172,12 @@ type homeMap struct { |
| // partialPolls counts polls that could not read (or parse) every object, |
| // for the first-and-every-tenth log cadence. |
| partialPolls int |
| + // staleHopN counts staleHopRefusal occurrences per "box host" key (box is |
| + // always m.self — see staleHopRefusal's doc for why it is still named in |
| + // the key and the log line), for the SAME first-and-every-tenth cadence |
| + // partialPolls drives. Guarded by mu, following logPartialPoll's own |
| + // choice to reuse the one lock rather than mint a second. |
| + staleHopN map[string]int |
| // The mirrored mesh keys, read on the same refresh. Replaced only on a |
| // SUCCESSFUL load: a bucket blip leaves the last good key verifying hops, |
| @@ -559,9 +565,44 @@ func stripForwardedHeaders(r *http.Request) { |
| // line for the ONE case where it could still leak: a peer whose key is valid |
| // but whose map is wrong learns only "not here", never "here is where it |
| // went". |
| -func staleHopRefusal(w http.ResponseWriter) { |
| +// |
| +// Silent until now was leaving a home move that gets stuck mid-way with |
| +// nothing on this box's OWN journal to show it — a sender can retry into |
| +// this refusal for minutes (see above) and the only visibility was on the |
| +// SENDING side. This logs first-and-every-tenth per (box, host), the same |
| +// cadence logPartialPoll (above) uses for its own standing condition; box is |
| +// always m.self (there is exactly one homeMap per process, and it never |
| +// learns the sending edge's identity — the hop MAC covers only host and |
| +// time, mesh.go), named in the key and the line anyway so a centrally |
| +// aggregated journal reads as "box X refused host Y" rather than an |
| +// anonymous host with no owner. |
| +// |
| +// A nil receiver (no deployment store — the no-mesh-at-all case) still |
| +// answers the refusal; it just cannot count or log one, the same "answer |
| +// first, degrade the bookkeeping" shape Home and meshKeys already have. |
| +func (m *homeMap) staleHopRefusal(w http.ResponseWriter, host string) { |
| w.Header().Set("Retry-After", "2") |
| http.Error(w, "not served here", http.StatusServiceUnavailable) |
| + if m == nil { |
| + return |
| + } |
| + m.logStaleHop(host) |
| +} |
| + |
| +// logStaleHop is staleHopRefusal's counting half, split out so the HTTP |
| +// response is written before anything touches mu. |
| +func (m *homeMap) logStaleHop(host string) { |
| + key := m.self + " " + host |
| + m.mu.Lock() |
| + if m.staleHopN == nil { |
| + m.staleHopN = make(map[string]int) |
| + } |
| + m.staleHopN[key]++ |
| + n := m.staleHopN[key] |
| + m.mu.Unlock() |
| + if n == 1 || n%10 == 0 { |
| + log.Printf("edge: stale hop refusal on box %s for %s (#%d) — the sending edge's home map has not caught up yet", m.self, host, n) |
| + } |
| } |
| // setForwarded is what every proxy Rewrite in this package calls in place of |
diff --git a/internal/edge/homemap_test.go b/internal/edge/homemap_test.go| index 687a85927..56b61245b 100644 |
| --- a/internal/edge/homemap_test.go |
| +++ b/internal/edge/homemap_test.go |
| @@ -386,6 +386,42 @@ func TestVerifiedHopForAnUnservedHostRefusesWithoutForwarding(t *testing.T) { |
| } |
| } |
| +// Task 1: the refusal used to be silent. A home move stuck mid-way can leave |
| +// a sender retrying into this refusal for minutes (the loop cadences, |
| +// edgesfile.go/homemap.go's own docs), so it now logs — but first-and- |
| +// every-tenth, logPartialPoll's own cadence, or a stuck move would fill the |
| +// journal with one line per hop. |
| +func TestStaleHopRefusalLogsFirstAndEveryTenth(t *testing.T) { |
| + e, m, key := hopEdge(t, "annie", "app.test") |
| + putJSON(t, m.s, "control/edges/betty.json", |
| + `{"v":1,"box":"betty","eip":"2.2.2.2","hosts":{"moved.test":{"role":"home"}}}`) |
| + refreshHomeMap(t, m) |
| + |
| + buf := captureLog(t) |
| + for i := 0; i < 10; i++ { |
| + resp := hopRequest(t, e, "moved.test", map[string]string{ |
| + hopHeader: "1", |
| + hopAuthHeader: signHop(key, "moved.test", time.Now()), |
| + }) |
| + if resp.StatusCode != http.StatusServiceUnavailable { |
| + t.Fatalf("refusal #%d: status = %d, want 503", i+1, resp.StatusCode) |
| + } |
| + } |
| + got := buf.String() |
| + if n := strings.Count(got, "moved.test"); n != 2 { |
| + t.Fatalf("log mentions the stale host %d time(s), want exactly 2 (occurrence #1 and #10, not one line per refusal): %s", n, got) |
| + } |
| + if !strings.Contains(got, "(#1)") { |
| + t.Fatalf("log output = %q, want the first occurrence logged immediately", got) |
| + } |
| + if !strings.Contains(got, "(#10)") { |
| + t.Fatalf("log output = %q, want the tenth occurrence logged", got) |
| + } |
| + if !strings.Contains(got, "box annie") { |
| + t.Fatalf("log output = %q, want this box's own label named in the line", got) |
| + } |
| +} |
| + |
| // Verification failure is not an error path: it is "you are an ordinary |
| // client", with every forwarding claim you made discarded. |
| func TestUnverifiedHopIsServedAsAnOrdinaryClientWithForwardedHeadersStripped(t *testing.T) { |
| @@ -928,6 +964,53 @@ func TestBackhaulToAnUnreachableHomeIsA502(t *testing.T) { |
| } |
| } |
| +// Task 1: an unreachable home used to log unthrottled, once per failed dial — |
| +// a home box down for any real stretch would fill the journal with one line |
| +// per request. It now goes through e.logThrottled, keyed "backhaul-error |
| +// "+box, so a run of failures against the SAME home box produces one line |
| +// (occurrence #1), not one per failure. |
| +func TestBackhaulErrorHandlerRoutesThroughLogThrottled(t *testing.T) { |
| + e, m, _ := hopEdge(t, "annie", "local.test") |
| + ln, err := net.Listen("tcp", "127.0.0.1:0") |
| + if err != nil { |
| + t.Fatalf("listen: %v", err) |
| + } |
| + eip, port, err := net.SplitHostPort(ln.Addr().String()) |
| + if err != nil { |
| + t.Fatalf("split: %v", err) |
| + } |
| + ln.Close() |
| + old := backhaulPort |
| + backhaulPort = port |
| + t.Cleanup(func() { backhaulPort = old }) |
| + homedOn(t, m, "betty", eip, "example.com") |
| + |
| + buf := captureLog(t) |
| + for i := 0; i < 3; i++ { |
| + resp := hopRequest(t, e, "example.com", nil) |
| + if resp.StatusCode != http.StatusBadGateway { |
| + t.Fatalf("attempt #%d: status = %d, want 502", i+1, resp.StatusCode) |
| + } |
| + } |
| + got := buf.String() |
| + // logThrottled's own format is "edge: <msg> (occurrence #N)" — it prints |
| + // no key of its own, so the key's effect is observed as throttling: three |
| + // failed dials against the SAME "backhaul-error "+box key must produce |
| + // exactly one logged line (#1), not three. |
| + if n := strings.Count(got, "backhaul"); n != 1 { |
| + t.Fatalf("log mentions the backhaul failure %d time(s), want exactly 1 (logThrottled's first-and-every-300th, not one line per failed dial): %s", n, got) |
| + } |
| + if !strings.Contains(got, "occurrence #1") { |
| + t.Fatalf("log output = %q, want the first occurrence logged (logThrottled's own format)", got) |
| + } |
| + if strings.Contains(got, "occurrence #2") || strings.Contains(got, "occurrence #3") { |
| + t.Fatalf("log output = %q, want the second and third failures throttled, not logged again", got) |
| + } |
| + if !strings.Contains(got, "example.com") || !strings.Contains(got, "betty") { |
| + t.Fatalf("log output = %q, want the host and the home box named", got) |
| + } |
| +} |
| + |
| // One transport per home BOX. The pool is what makes a backhaul cost one |
| // handshake rather than one per request; keying it per box rather than per |
| // host is what keeps a catch-all home — where every invented subdomain is a |
diff --git a/internal/host/edgesfile.go b/internal/host/edgesfile.go| index 4fa38a39d..fbe7d5523 100644 |
| --- a/internal/host/edgesfile.go |
| +++ b/internal/host/edgesfile.go |
| @@ -8,6 +8,7 @@ import ( |
| "log" |
| "math/rand/v2" |
| "os" |
| + "strings" |
| "time" |
| "github.com/carlosframework/platform/internal/registry" |
| @@ -31,11 +32,18 @@ type edgeHost struct { |
| // edgesFile is the wire shape at edgesFileKey(box), schema v1 (spec Part 3): |
| // |
| -// {"v":1,"box":"annie-welding","eip":"99.81.104.219","generation":"2026-08-22T09:00:00Z","hosts":{"app.example.com":{"role":"home"}}} |
| +// {"v":1,"box":"annie-welding","eip":"99.81.104.219","region":"eu-west-1","generation":"2026-08-22T09:00:00Z","hosts":{"app.example.com":{"role":"home"}}} |
| +// |
| +// Region is additive (Part 3, Task 4's steering converger needs it for |
| +// latency records): omitempty, value = CARLOS_REGION verbatim, absent |
| +// entirely when the env is unset — a pre-region body written by an older |
| +// build still parses, and this build's own body for an unset-region box is |
| +// byte-identical to what it always published. |
| type edgesFile struct { |
| V int `json:"v"` |
| Box string `json:"box"` |
| EIP string `json:"eip"` |
| + Region string `json:"region,omitempty"` |
| Generation string `json:"generation"` |
| Hosts map[string]edgeHost `json:"hosts"` |
| } |
| @@ -63,7 +71,12 @@ type edgesFile struct { |
| // file's EIP instead. "" only ever reaches the wire for a box that has |
| // NEVER had a discoverable IP — the DR runbook's hosts list still gets |
| // published in that case, just without a dial address. |
| -func edgesFilePass(ctx context.Context, s release.Store, reg *registry.Registry, box, eip string, last *[]byte) error { |
| +// |
| +// region is CARLOS_REGION verbatim (read once at loop arm, the domains |
| +// loop's precedent — see edgesFileLoop) and is part of the change gate like |
| +// every other field except Generation: a region relabel is a real change and |
| +// must write, not be swallowed as a timestamp-only refresh would be. |
| +func edgesFilePass(ctx context.Context, s release.Store, reg *registry.Registry, box, eip, region string, last *[]byte) error { |
| routes, err := reg.List() |
| if err != nil { |
| return fmt.Errorf("edges: list: %w", err) |
| @@ -98,7 +111,7 @@ func edgesFilePass(ctx context.Context, s release.Store, reg *registry.Registry, |
| } |
| } |
| - f := edgesFile{V: 1, Box: box, EIP: eip, Hosts: hosts} |
| + f := edgesFile{V: 1, Box: box, EIP: eip, Region: region, Hosts: hosts} |
| gate, err := json.Marshal(f) |
| if err != nil { |
| return fmt.Errorf("edges: marshal: %w", err) |
| @@ -119,14 +132,57 @@ func edgesFilePass(ctx context.Context, s release.Store, reg *registry.Registry, |
| return nil |
| } |
| +// seedEdgesFileGate reads this box's last-published edges file at loop arm |
| +// and returns the change-gate body (Generation cleared) to prime *last, |
| +// mirroring seedSnapshotFingerprint's contract exactly (snapshot_loop.go): |
| +// without this, a freshly started (or crash-looping) process's in-memory |
| +// *last starts empty, so edgesFilePass's own change gate — bytes.Equal(gate, |
| +// *last) — can never match on the first tick and the process republishes an |
| +// UNCHANGED file on every restart, burning an ETag/ifMatch round trip for |
| +// nothing (plan-2 final review, Important 1, first half). |
| +// |
| +// The same read also recovers the EIP hold for free: edgesFilePass's blank- |
| +// probe fallback reads *last's EIP when this tick's probe is "", and the |
| +// seeded gate still carries the prior file's EIP (only Generation is |
| +// cleared). Without this seed, a box that restarts and hits a transient |
| +// IMDS blip on its very first tick has a nil *last, the hold's `*last != |
| +// nil` guard never fires, and the box would publish "" — blanking a known- |
| +// good EIP for every remote edge's dial address until the next successful |
| +// probe (the same review's Important 1, second half; the restart-blip |
| +// case). One seed read closes both. |
| +// |
| +// Any read or parse failure — no prior file yet, a corrupt object, a store |
| +// that is down — returns nil: the loop then behaves exactly as it always |
| +// has on a box's first-ever tick. This must never fail the loop. |
| +func seedEdgesFileGate(ctx context.Context, s release.Store, box string) []byte { |
| + body, _, err := s.Get(ctx, edgesFileKey(box)) |
| + if err != nil { |
| + return nil |
| + } |
| + var prior edgesFile |
| + if err := json.Unmarshal(body, &prior); err != nil { |
| + return nil |
| + } |
| + prior.Generation = "" |
| + gate, err := json.Marshal(prior) |
| + if err != nil { |
| + return nil |
| + } |
| + return gate |
| +} |
| + |
| // edgesFileLoop publishes this box's edges file every 5 minutes + jitter — |
| -// the same cadence and arming site as registrySnapshotLoop (home moves are |
| -// placement events, not per-request events; the backhaul hop's stale-map |
| -// 503 + converge heals faster than this loop could anyway). Platform-box |
| -// job, retentionLoop's rules: skipped on fleet boxes, disabled with a log |
| -// line when there is no deployment bucket. Single-box deployments: the file |
| -// is published and nothing reads it yet — inert until Task 3's mirror |
| -// ships. |
| +// the same cadence and arming site as registrySnapshotLoop. Home moves ARE |
| +// placement events, not per-request events, but the comment this replaces |
| +// overstated how fast that heals: there is no 2s serial-converge path for |
| +// this file (edges never bump control/serial — homemap.go's staleHopRefusal |
| +// doc makes the identical correction on the reader's side), so what |
| +// actually heals a stale home map is this loop's own 5min+jitter cycle |
| +// followed by each remote edge's 30s mirror poll (internal/edge's |
| +// homeMap) — minutes during a home move, not seconds. Platform-box job, |
| +// retentionLoop's rules: skipped on fleet boxes, disabled with a log line |
| +// when there is no deployment bucket. Single-box deployments: the file is |
| +// published and nothing reads it yet — inert until Task 3's mirror ships. |
| func edgesFileLoop(reg *registry.Registry) { |
| ctx := context.Background() |
| if shouldSkipPublisherElection(os.Getenv) { |
| @@ -139,14 +195,18 @@ func edgesFileLoop(reg *registry.Registry) { |
| return |
| } |
| box := metricsBoxLabel(os.Getenv, readMachineID()) |
| + // CARLOS_REGION, read once here at arm — the domains loop's read-once |
| + // precedent (domainsLoop reads its env once before the loop and threads |
| + // the result through every tick via cfg, never re-reading per pass). |
| + region := strings.TrimSpace(os.Getenv("CARLOS_REGION")) |
| log.Printf("agent: edges file loop armed (5min + jitter, %s)", edgesFileKey(box)) |
| - var last []byte |
| + last := seedEdgesFileGate(ctx, store, box) |
| delay := time.Minute |
| for { |
| time.Sleep(delay) |
| delay = 5*time.Minute + rand.N(time.Minute) |
| eip := readPublicIP() |
| - if err := edgesFilePass(ctx, store, reg, box, eip, &last); err != nil { |
| + if err := edgesFilePass(ctx, store, reg, box, eip, region, &last); err != nil { |
| log.Printf("agent: edges file pass: %v", err) |
| } |
| } |
diff --git a/internal/host/edgesfile_test.go b/internal/host/edgesfile_test.go| index 162db2870..43fd5a19c 100644 |
| --- a/internal/host/edgesfile_test.go |
| +++ b/internal/host/edgesfile_test.go |
| @@ -49,7 +49,7 @@ func TestEdgesFilePublishesHomedHosts(t *testing.T) { |
| s, reg := release.NewDirStore(t.TempDir()), edgesTestRegistry(t) |
| seedRoute(t, reg, "app.example.com") |
| var lastBody []byte |
| - if err := edgesFilePass(context.Background(), s, reg, "annie", "1.2.3.4", &lastBody); err != nil { |
| + if err := edgesFilePass(context.Background(), s, reg, "annie", "1.2.3.4", "", &lastBody); err != nil { |
| t.Fatal(err) |
| } |
| got, _, err := s.Get(context.Background(), "control/edges/annie.json") |
| @@ -80,7 +80,7 @@ func TestEdgesFilePublishesWithEmptyEIP(t *testing.T) { |
| s, reg := release.NewDirStore(t.TempDir()), edgesTestRegistry(t) |
| seedRoute(t, reg, "app.example.com") |
| var lastBody []byte |
| - if err := edgesFilePass(context.Background(), s, reg, "annie", "", &lastBody); err != nil { |
| + if err := edgesFilePass(context.Background(), s, reg, "annie", "", "", &lastBody); err != nil { |
| t.Fatal(err) |
| } |
| got, _, err := s.Get(context.Background(), "control/edges/annie.json") |
| @@ -106,7 +106,7 @@ func TestEdgesFileSkipsWriteWhenUnchanged(t *testing.T) { |
| seedRoute(t, reg, "app.example.com") |
| var lastBody []byte |
| - if err := edgesFilePass(context.Background(), s, reg, "annie", "1.2.3.4", &lastBody); err != nil { |
| + if err := edgesFilePass(context.Background(), s, reg, "annie", "1.2.3.4", "", &lastBody); err != nil { |
| t.Fatal(err) |
| } |
| if s.puts != 1 { |
| @@ -116,7 +116,7 @@ func TestEdgesFileSkipsWriteWhenUnchanged(t *testing.T) { |
| // Second pass, identical registry state: a fresh generation timestamp |
| // would differ from the first pass's, but the change-gate excludes |
| // Generation, so this must not write. |
| - if err := edgesFilePass(context.Background(), s, reg, "annie", "1.2.3.4", &lastBody); err != nil { |
| + if err := edgesFilePass(context.Background(), s, reg, "annie", "1.2.3.4", "", &lastBody); err != nil { |
| t.Fatal(err) |
| } |
| if s.puts != 1 { |
| @@ -125,7 +125,7 @@ func TestEdgesFileSkipsWriteWhenUnchanged(t *testing.T) { |
| // A registry change makes the next pass write again. |
| seedRoute(t, reg, "second.example.com") |
| - if err := edgesFilePass(context.Background(), s, reg, "annie", "1.2.3.4", &lastBody); err != nil { |
| + if err := edgesFilePass(context.Background(), s, reg, "annie", "1.2.3.4", "", &lastBody); err != nil { |
| t.Fatal(err) |
| } |
| if s.puts != 2 { |
| @@ -145,12 +145,12 @@ func TestEdgesFileHoldsLastKnownGoodEIPOnTransientBlank(t *testing.T) { |
| seedRoute(t, reg, "app.example.com") |
| var last []byte |
| - if err := edgesFilePass(context.Background(), s, reg, "annie", "1.2.3.4", &last); err != nil { |
| + if err := edgesFilePass(context.Background(), s, reg, "annie", "1.2.3.4", "", &last); err != nil { |
| t.Fatal(err) |
| } |
| // Simulate a transient IMDS blip: this tick's probe returns "". |
| - if err := edgesFilePass(context.Background(), s, reg, "annie", "", &last); err != nil { |
| + if err := edgesFilePass(context.Background(), s, reg, "annie", "", "", &last); err != nil { |
| t.Fatal(err) |
| } |
| @@ -180,7 +180,7 @@ func TestEdgesFileExcludesNonReadyRows(t *testing.T) { |
| } |
| var last []byte |
| - if err := edgesFilePass(context.Background(), s, reg, "annie", "1.2.3.4", &last); err != nil { |
| + if err := edgesFilePass(context.Background(), s, reg, "annie", "1.2.3.4", "", &last); err != nil { |
| t.Fatal(err) |
| } |
| got, _, err := s.Get(context.Background(), "control/edges/annie.json") |
| @@ -213,7 +213,7 @@ func TestEdgesFileWildcardRowPublishesVerbatim(t *testing.T) { |
| } |
| var last []byte |
| - if err := edgesFilePass(context.Background(), s, reg, "annie", "1.2.3.4", &last); err != nil { |
| + if err := edgesFilePass(context.Background(), s, reg, "annie", "1.2.3.4", "", &last); err != nil { |
| t.Fatal(err) |
| } |
| got, _, err := s.Get(context.Background(), "control/edges/annie.json") |
| @@ -244,7 +244,7 @@ func TestEdgesFileRawWireBodyMatchesSchema(t *testing.T) { |
| seedRoute(t, reg, "app.example.com") |
| var last []byte |
| - if err := edgesFilePass(context.Background(), s, reg, "annie-welding", "99.81.104.219", &last); err != nil { |
| + if err := edgesFilePass(context.Background(), s, reg, "annie-welding", "99.81.104.219", "", &last); err != nil { |
| t.Fatal(err) |
| } |
| got, _, err := s.Get(context.Background(), "control/edges/annie-welding.json") |
| @@ -268,3 +268,227 @@ func TestEdgesFileRawWireBodyMatchesSchema(t *testing.T) { |
| t.Fatalf("raw wire body =\n%s\nwant\n%s", got, want) |
| } |
| } |
| + |
| +// FINDING 5 (task 1, 2026-08-22): region is additive to the pinned schema — |
| +// this extends FINDING 4's raw-body pin rather than replacing it. Two |
| +// bodies, byte for byte: CARLOS_REGION unset must produce the IDENTICAL |
| +// bytes FINDING 4 already pins (no "region" key at all, omitempty), and |
| +// CARLOS_REGION set must insert exactly `"region":"eu-west-1"` between |
| +// "eip" and "generation" — the struct field order — with nothing else |
| +// disturbed. |
| +func TestEdgesFileRawWireBodyRegionField(t *testing.T) { |
| + s, reg := release.NewDirStore(t.TempDir()), edgesTestRegistry(t) |
| + seedRoute(t, reg, "app.example.com") |
| + |
| + genOf := func(t *testing.T, body []byte) string { |
| + t.Helper() |
| + var probe struct { |
| + Generation string `json:"generation"` |
| + } |
| + if err := json.Unmarshal(body, &probe); err != nil { |
| + t.Fatal(err) |
| + } |
| + return probe.Generation |
| + } |
| + |
| + t.Run("unset region is omitted entirely", func(t *testing.T) { |
| + var last []byte |
| + if err := edgesFilePass(context.Background(), s, reg, "annie-welding", "99.81.104.219", "", &last); err != nil { |
| + t.Fatal(err) |
| + } |
| + got, _, err := s.Get(context.Background(), "control/edges/annie-welding.json") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + gen := genOf(t, got) |
| + want := `{"v":1,"box":"annie-welding","eip":"99.81.104.219","generation":"` + |
| + gen + `","hosts":{"app.example.com":{"role":"home"}}}` |
| + if string(got) != want { |
| + t.Fatalf("raw wire body (region unset) =\n%s\nwant\n%s", got, want) |
| + } |
| + }) |
| + |
| + t.Run("set region is inserted between eip and generation", func(t *testing.T) { |
| + s2, reg2 := release.NewDirStore(t.TempDir()), edgesTestRegistry(t) |
| + seedRoute(t, reg2, "app.example.com") |
| + var last []byte |
| + if err := edgesFilePass(context.Background(), s2, reg2, "annie-welding", "99.81.104.219", "eu-west-1", &last); err != nil { |
| + t.Fatal(err) |
| + } |
| + got, _, err := s2.Get(context.Background(), "control/edges/annie-welding.json") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + gen := genOf(t, got) |
| + want := `{"v":1,"box":"annie-welding","eip":"99.81.104.219","region":"eu-west-1","generation":"` + |
| + gen + `","hosts":{"app.example.com":{"role":"home"}}}` |
| + if string(got) != want { |
| + t.Fatalf("raw wire body (region set) =\n%s\nwant\n%s", got, want) |
| + } |
| + }) |
| +} |
| + |
| +// FINDING 6 (task 1, 2026-08-22, the plan-2 final review's Important 1): a |
| +// freshly ARMED loop must seed its change-gate from whatever this box last |
| +// published, or two bugs follow — see seedEdgesFileGate's doc. This test |
| +// pins the seed-then-pass contract directly: a process that primes its |
| +// *last from seedEdgesFileGate, then runs a pass against UNCHANGED registry |
| +// state, must write ZERO times. Before this task, edgesFileLoop always |
| +// started with a bare `var last []byte`, so the very first tick after any |
| +// restart always wrote — this is the regression that closes. |
| +func TestEdgesFileSeedGateSkipsWriteOnUnchangedBucketState(t *testing.T) { |
| + real := release.NewDirStore(t.TempDir()) |
| + reg := edgesTestRegistry(t) |
| + seedRoute(t, reg, "app.example.com") |
| + |
| + // A prior process (or the prior pass of this one) already published. |
| + var priming []byte |
| + if err := edgesFilePass(context.Background(), real, reg, "annie", "1.2.3.4", "", &priming); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + // A fresh process arms: seed the gate from the bucket, as edgesFileLoop |
| + // now does at loop arm. |
| + last := seedEdgesFileGate(context.Background(), real, "annie") |
| + if last == nil { |
| + t.Fatal("seedEdgesFileGate returned nil, want the prior file's gate body") |
| + } |
| + |
| + counting := &putCountingStore{Store: real} |
| + if err := edgesFilePass(context.Background(), counting, reg, "annie", "1.2.3.4", "", &last); err != nil { |
| + t.Fatal(err) |
| + } |
| + if counting.puts != 0 { |
| + t.Fatalf("puts = %d, want 0 — a seeded gate against unchanged state must not write", counting.puts) |
| + } |
| +} |
| + |
| +// The restart-blip case named in the final review: a process restarts, |
| +// arms, and its very FIRST tick hits both a transient IMDS blank AND a real |
| +// registry change (so the pass must write). The write must still carry the |
| +// bucket's held EIP, not the blank probe — seedEdgesFileGate's dual |
| +// contract (gate AND EIP hold) has to survive an ACTUAL write, not just a |
| +// no-op pass, or the hold half of the fix is unverified. |
| +func TestEdgesFileSeedHoldsEIPThroughRestartBlip(t *testing.T) { |
| + s, reg := release.NewDirStore(t.TempDir()), edgesTestRegistry(t) |
| + seedRoute(t, reg, "app.example.com") |
| + |
| + var priming []byte |
| + if err := edgesFilePass(context.Background(), s, reg, "annie", "1.2.3.4", "", &priming); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + last := seedEdgesFileGate(context.Background(), s, "annie") |
| + if last == nil { |
| + t.Fatal("seedEdgesFileGate returned nil, want the prior file's gate body") |
| + } |
| + |
| + // This tick both hits a transient IMDS blip AND sees a real registry |
| + // change, so the pass must write. |
| + seedRoute(t, reg, "second.example.com") |
| + if err := edgesFilePass(context.Background(), s, reg, "annie", "", "", &last); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + got, _, err := s.Get(context.Background(), "control/edges/annie.json") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var f edgesFile |
| + if err := json.Unmarshal(got, &f); err != nil { |
| + t.Fatal(err) |
| + } |
| + if f.EIP != "1.2.3.4" { |
| + t.Fatalf("eip after restart + blank probe = %q, want held %q", f.EIP, "1.2.3.4") |
| + } |
| + if _, ok := f.Hosts["second.example.com"]; !ok { |
| + t.Fatalf("hosts = %+v, want second.example.com present — the write must carry the real registry change too", f.Hosts) |
| + } |
| +} |
| + |
| +// Without the seed, a fresh *last is nil and edgesFilePass's own hold guard |
| +// (`*last != nil`) cannot fire — this pins that this really is the bug the |
| +// two tests above close, not a tautology of the hold logic that would pass |
| +// regardless of seeding. |
| +func TestEdgesFileUnseededLastPublishesBlankEIPOnFirstTickBlip(t *testing.T) { |
| + s, reg := release.NewDirStore(t.TempDir()), edgesTestRegistry(t) |
| + seedRoute(t, reg, "app.example.com") |
| + |
| + var priming []byte |
| + if err := edgesFilePass(context.Background(), s, reg, "annie", "1.2.3.4", "", &priming); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + // The pre-fix shape: a fresh process's *last starts nil (no seed read). |
| + var unseeded []byte |
| + if err := edgesFilePass(context.Background(), s, reg, "annie", "", "", &unseeded); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + got, _, err := s.Get(context.Background(), "control/edges/annie.json") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var f edgesFile |
| + if err := json.Unmarshal(got, &f); err != nil { |
| + t.Fatal(err) |
| + } |
| + if f.EIP != "" { |
| + t.Fatalf("eip = %q, want blank — demonstrating why the seed in the tests above is load-bearing, not incidental", f.EIP) |
| + } |
| +} |
| + |
| +// Self-review: does the seed interact correctly with the change gate when |
| +// the bucket file PREDATES the region field? A box upgraded to this build |
| +// finds a bucket file an older build left with no "region" key at all. |
| +// Seeding *last from it must not treat this build's first pass — the one |
| +// that finally sets a real region — as unchanged: Region participates in |
| +// the change gate exactly like every other field (edgesFilePass's doc), so |
| +// a seeded pre-region gate and a fresh body WITH region differ in bytes and |
| +// must write exactly once, not zero and not on every tick after. |
| +func TestEdgesFileSeedPreRegionBucketWritesOnceWhenRegionArrives(t *testing.T) { |
| + real := release.NewDirStore(t.TempDir()) |
| + reg := edgesTestRegistry(t) |
| + seedRoute(t, reg, "app.example.com") |
| + |
| + // Hand-written exactly as a pre-region build would have left it in the |
| + // bucket: no "region" key present. |
| + preRegion := []byte(`{"v":1,"box":"annie","eip":"1.2.3.4","generation":"2026-08-01T00:00:00Z","hosts":{"app.example.com":{"role":"home"}}}`) |
| + if err := putReplacing(context.Background(), real, edgesFileKey("annie"), preRegion); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + last := seedEdgesFileGate(context.Background(), real, "annie") |
| + if last == nil { |
| + t.Fatal("seedEdgesFileGate returned nil") |
| + } |
| + |
| + // This build now has CARLOS_REGION set: the first pass must write, once. |
| + counting := &putCountingStore{Store: real} |
| + if err := edgesFilePass(context.Background(), counting, reg, "annie", "1.2.3.4", "eu-west-1", &last); err != nil { |
| + t.Fatal(err) |
| + } |
| + if counting.puts != 1 { |
| + t.Fatalf("puts = %d, want 1 — a region arriving on a seeded pre-region body is a real change", counting.puts) |
| + } |
| + got, _, err := real.Get(context.Background(), edgesFileKey("annie")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var f edgesFile |
| + if err := json.Unmarshal(got, &f); err != nil { |
| + t.Fatal(err) |
| + } |
| + if f.Region != "eu-west-1" { |
| + t.Fatalf("region = %q, want eu-west-1", f.Region) |
| + } |
| + |
| + // A second, identical-region pass must not write again — the gate now |
| + // includes region on both sides. |
| + if err := edgesFilePass(context.Background(), counting, reg, "annie", "1.2.3.4", "eu-west-1", &last); err != nil { |
| + t.Fatal(err) |
| + } |
| + if counting.puts != 1 { |
| + t.Fatalf("puts after unchanged second pass = %d, want still 1", counting.puts) |
| + } |
| +} |