amadan
edge: holding entries expire five minutes after their fetch, with today's fallback including wildcards
The witnesses for the fetch clock (plan 3 Task 5, Review Focus 1). A remote exact hold over a retained exact ready (1,0) on another box and on this one, over a local wildcard upstream and over a remote wildcard route answers the page at 4 m 59 s and today's answer at 5 m 0 s and 5 m 1 s, with no refresh in between, through a failed List, every Get failing, the hold's file unreadable or corrupt alone (carried forward with its old fetch while the bystander's fresh file keeps its hold), and renewed by a good poll; no store read happens across the expiry. Home, ExactHome and Owner answer the retained route once the hold expires and stop again when a fresh fetch revives it. KnownHost, deploymentHost, HoldingHosts and certNames drop an expired remote hold's names at exactly 5 m 0 s; a local hold stays Allowed and certified an hour on.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1 file changed,
+277
−0
internal/edge/homemap_test.go+277 −0
diff --git a/internal/edge/homemap_test.go b/internal/edge/homemap_test.go| index de5298eec..75d8c2eea 100644 |
| --- a/internal/edge/homemap_test.go |
| +++ b/internal/edge/homemap_test.go |
| @@ -2088,6 +2088,283 @@ func (fx *freshScene) fetchedAt(box string) time.Time { |
| // at moves the clock to d after rankNow, the first poll's time. |
| func (fx *freshScene) at(d time.Duration) { fx.now = rankNow.Add(d) } |
| +// Review Focus 1: a remote exact hold over each fallback, at 4 m 59 s, 5 m 0 s |
| +// and 5 m 1 s after its file's fetch, with no refresh in between and through |
| +// each way a later poll can go wrong. Before expiry the page; from exactly |
| +// five minutes, today's answer. A bystander hold on a third box shows whose |
| +// clock each poll left standing: a file carried forward keeps its old fetch, |
| +// a file read again gets the poll's. |
| +func TestHoldingEntriesExpireFromTheFetchClock(t *testing.T) { |
| + const host, bystander = "shop.example.com", "cart.example.org" |
| + type fallback struct { |
| + name string |
| + setup func(fx *freshScene, eip string) |
| + want string |
| + } |
| + fallbacks := []fallback{ |
| + {"a retained exact ready (1,0) on another box", func(fx *freshScene, eip string) { |
| + fx.put("betty", eip, readyJSON(host, "bab", "shop", 1, 0), "") |
| + }, homeEdgeBody}, |
| + {"a retained exact ready (1,0) on this box", func(fx *freshScene, _ string) { |
| + fx.route(host, "the local route", "bab", "shop", func(r *registry.Route) { r.Incarnation, r.ServingSeq = 1, 0 }) |
| + }, "the local route"}, |
| + {"a local wildcard upstream", func(fx *freshScene, _ string) { |
| + fx.route("*.example.com", "the local wildcard", "bab", "wild") |
| + }, "the local wildcard"}, |
| + {"a remote wildcard route", func(fx *freshScene, eip string) { |
| + fx.put("betty", eip, `"*.example.com":{"role":"home","account":"bab","app":"wild"}`, "") |
| + }, homeEdgeBody}, |
| + } |
| + type later struct { |
| + name string |
| + // faults runs before the poll at one minute; nil is no poll at all. |
| + faults func(fx *freshScene) |
| + wantErr bool |
| + // annie and carol are the fetch times the poll leaves on the hold's |
| + // file and the bystander's, after rankNow. |
| + annie, dolly time.Duration |
| + } |
| + laters := []later{ |
| + {name: "no refresh in between"}, |
| + {"a failed List", func(fx *freshScene) { fx.faults.set(true, false) }, true, 0, 0}, |
| + {"every Get failing", func(fx *freshScene) { fx.faults.set(false, true) }, true, 0, 0}, |
| + {"the hold's file unreadable alone", func(fx *freshScene) { fx.faults.set(false, false, edgesKey("annie")) }, false, 0, time.Minute}, |
| + {"the hold's file corrupt alone", func(fx *freshScene) { |
| + putJSON(fx.t, fx.bucket, edgesKey("annie"), `{"v":1,"box":"annie",`) |
| + }, false, 0, time.Minute}, |
| + {"a good poll renews every file", func(fx *freshScene) {}, false, time.Minute, time.Minute}, |
| + } |
| + for _, fb := range fallbacks { |
| + for _, lt := range laters { |
| + t.Run(fb.name+"/"+lt.name, func(t *testing.T) { |
| + fx := newFreshScene(t, "carol") |
| + _, eip := fakeHomeEdge(t, nil) |
| + fb.setup(fx, eip) |
| + fx.put("annie", "1.1.1.1", "", holdJSON(host, "mv_a", false, 1, 1)) |
| + fx.put("dolly", "4.4.4.4", "", holdJSON(bystander, "mv_d", false, 1, 1)) |
| + fx.mustPoll() |
| + wantPage(t, fx.serve(host, "/", ordinary), "the hold at its fetch") |
| + if lt.faults != nil { |
| + fx.at(time.Minute) |
| + lt.faults(fx) |
| + if err := fx.poll(); (err != nil) != lt.wantErr { |
| + t.Fatalf("the poll at 1 m: err = %v, want an error %v", err, lt.wantErr) |
| + } |
| + fx.faults.set(false, false) |
| + if got := fx.fetchedAt("annie"); !got.Equal(rankNow.Add(lt.annie)) { |
| + t.Fatalf("the hold's file is fetched at %v, want %v", got.Sub(rankNow), lt.annie) |
| + } |
| + if got := fx.fetchedAt("dolly"); !got.Equal(rankNow.Add(lt.dolly)) { |
| + t.Fatalf("the bystander's file is fetched at %v, want %v", got.Sub(rankNow), lt.dolly) |
| + } |
| + } |
| + reads := fx.faults.reads.Load() |
| + expiry := lt.annie + holdingValidity |
| + for _, d := range []time.Duration{expiry - time.Second, expiry, expiry + time.Second} { |
| + fx.at(d) |
| + what := fmt.Sprintf("%s at %v", host, d) |
| + if d < expiry { |
| + wantPage(t, fx.serve(host, "/", ordinary), what) |
| + if k := fx.e.resolve(host, fx.now).kind; k != decHolding { |
| + t.Fatalf("%s: decision %v, want the hold", what, k) |
| + } |
| + } else { |
| + wantBody(t, fx.serve(host, "/", ordinary), fb.want, what) |
| + } |
| + what = fmt.Sprintf("the bystander at %v", d) |
| + if d < lt.dolly+holdingValidity { |
| + wantPage(t, fx.serve(bystander, "/", ordinary), what) |
| + } else { |
| + wantUnknown(t, fx.serve(bystander, "/", ordinary), what) |
| + } |
| + } |
| + if n := fx.faults.reads.Load(); n != reads { |
| + t.Fatalf("expiry read the store %d time(s): it is applied at ranking time alone", n-reads) |
| + } |
| + }) |
| + } |
| + } |
| +} |
| + |
| +// After expiry the hold no longer suppresses its target's retained |
| +// publication, and every reader of the map (Home, ExactHome, Owner and the |
| +// request path) answers that route; a later good fetch of the still-holding |
| +// file revives the page. |
| +func TestExpiredHoldFallsBackToTheRetainedRoute(t *testing.T) { |
| + const host, alias = "shop.example.com", "shop.example.net" |
| + t.Run("a retained route on another box", func(t *testing.T) { |
| + fx := newFreshScene(t, "carol") |
| + rec, eip := fakeHomeEdge(t, nil) |
| + fx.put("annie", "1.1.1.1", "", holdJSON(host, "mv_a", false, 1, 1)) |
| + fx.put("betty", eip, readyJSON(host, "bab", "shop", 1, 0), "") |
| + fx.mustPoll() |
| + fx.at(time.Minute) |
| + fx.faults.set(true, false) |
| + if fx.poll() == nil { |
| + t.Fatal("test bug: the failed List must fail the poll") |
| + } |
| + fx.faults.set(false, false) |
| + |
| + fx.at(holdingValidity - time.Second) |
| + if _, _, ok := fx.m.Home(host); ok { |
| + t.Fatal("Home offered the suppressed route under a live hold") |
| + } |
| + if _, _, ok := fx.m.ExactHome(host); ok { |
| + t.Fatal("ExactHome offered the suppressed route under a live hold") |
| + } |
| + if _, _, _, _, ok := fx.m.Owner(host); ok { |
| + t.Fatal("Owner answered the suppressed route under a live hold") |
| + } |
| + wantPage(t, fx.serve(host, "/", ordinary), "under a live hold") |
| + if n := rec.get().n; n != 0 { |
| + t.Fatalf("the retained home saw %d request(s) under a live hold", n) |
| + } |
| + |
| + fx.at(holdingValidity) |
| + if e, box, ok := fx.m.Home(host); !ok || box != "betty" || e != eip { |
| + t.Fatalf("Home = %q %q %v at 5 m 0 s, want betty's retained route", e, box, ok) |
| + } |
| + if _, box, ok := fx.m.ExactHome(host); !ok || box != "betty" { |
| + t.Fatalf("ExactHome = %q %v at 5 m 0 s, want betty", box, ok) |
| + } |
| + if acct, app, _, box, ok := fx.m.Owner(host); !ok || acct != "bab" || app != "shop" || box != "betty" { |
| + t.Fatalf("Owner = %q %q %q %v at 5 m 0 s, want bab/shop on betty", acct, app, box, ok) |
| + } |
| + wantBody(t, fx.serve(host, "/", ordinary), homeEdgeBody, "the retained route at 5 m 0 s") |
| + if n := rec.get().n; n != 1 { |
| + t.Fatalf("the retained home saw %d request(s), want 1", n) |
| + } |
| + |
| + // The bucket answers again and annie still holds: a fresh fetch, the |
| + // page again. |
| + fx.at(6 * time.Minute) |
| + fx.mustPoll() |
| + wantPage(t, fx.serve(host, "/", ordinary), "a revived hold") |
| + if _, _, ok := fx.m.Home(host); ok { |
| + t.Fatal("Home offered the route past a revived hold") |
| + } |
| + }) |
| + t.Run("a retained alias publication of the held host's alias", func(t *testing.T) { |
| + // The fake home edge's certificate covers *.example.com: the alias |
| + // is dialled, so it is the name under it. |
| + const host, alias = "shop.example.org", "www.example.com" |
| + fx := newFreshScene(t, "carol") |
| + _, eip := fakeHomeEdge(t, nil) |
| + fx.put("annie", "1.1.1.1", "", holdJSON(host, "mv_a", false, 1, 1, alias)) |
| + fx.put("betty", eip, fmt.Sprintf(`%q:{"role":"home","alias":true,"target_host":%q,"account":"bab","app":"shop","incarnation":1,"serving_seq":0}`, alias, host), "") |
| + fx.mustPoll() |
| + fx.at(holdingValidity - time.Second) |
| + wantPage(t, fx.serve(alias, "/", ordinary), "the alias under a live hold") |
| + fx.at(holdingValidity) |
| + wantBody(t, fx.serve(alias, "/", ordinary), homeEdgeBody, "the alias's retained publication at 5 m 0 s") |
| + if _, box, ok := fx.m.Home(alias); !ok || box != "betty" { |
| + t.Fatalf("Home(alias) = %q %v, want betty", box, ok) |
| + } |
| + }) |
| + t.Run("a retained route on this box, with the hold's file carried forward", func(t *testing.T) { |
| + fx := newFreshScene(t, "carol") |
| + fx.route(host, "the local route", "bab", "shop", func(r *registry.Route) { r.Incarnation, r.ServingSeq = 1, 0 }) |
| + fx.put("annie", "1.1.1.1", "", holdJSON(host, "mv_a", false, 1, 1)) |
| + fx.put("betty", "2.2.2.2", readyJSON("other.example.com", "bab", "other", 0, 0), "") |
| + fx.mustPoll() |
| + fx.at(4 * time.Minute) |
| + fx.faults.set(false, false, edgesKey("annie")) |
| + fx.mustPoll() |
| + fx.faults.set(false, false) |
| + fx.at(holdingValidity - time.Second) |
| + wantPage(t, fx.serve(host, "/", ordinary), "the stale local route under a live remote hold") |
| + fx.at(holdingValidity) |
| + wantBody(t, fx.serve(host, "/", ordinary), "the local route", "the local route at 5 m 0 s, the carried-forward hold expired") |
| + }) |
| +} |
| + |
| +// The certificate gates follow the hold's clock: KnownHost (and so |
| +// deploymentHost), HoldingHosts and certNames drop an expired remote hold's |
| +// names at exactly 5 m 0 s, a failed refresh in between renewing nothing. A |
| +// local hold is this box's own and the remote clock does not touch it. |
| +func TestExpiredHoldNoLongerAdmitsACertificate(t *testing.T) { |
| + const host, alias = "shop.example.com", "shop.example.net" |
| + t.Run("a remote hold's names leave at exactly 5 m 0 s", func(t *testing.T) { |
| + fx := newFreshScene(t, "carol") |
| + fx.put("annie", "1.1.1.1", "", holdJSON(host, "mv_a", false, 1, 1, alias)) |
| + fx.mustPoll() |
| + fx.at(time.Minute) |
| + fx.faults.set(true, false) |
| + if fx.poll() == nil { |
| + t.Fatal("test bug: the failed List must fail the poll") |
| + } |
| + fx.faults.set(false, false) |
| + for _, d := range []time.Duration{holdingValidity - time.Second, holdingValidity, holdingValidity + time.Second} { |
| + fx.at(d) |
| + live := d < holdingValidity |
| + for _, n := range []string{host, alias} { |
| + what := fmt.Sprintf("%s at %v", n, d) |
| + if got := fx.m.KnownHost(n, fx.now); got != live { |
| + t.Fatalf("KnownHost(%s) = %v, want %v", what, got, live) |
| + } |
| + if got := fx.e.deploymentHost(n); got != live { |
| + t.Fatalf("deploymentHost(%s) = %v, want %v", what, got, live) |
| + } |
| + if got := slices.Contains(fx.m.HoldingHosts(fx.now), n); got != live { |
| + t.Fatalf("HoldingHosts has %s: %v, want %v", what, got, live) |
| + } |
| + if got := slices.Contains(fx.e.certNames(), n); got != live { |
| + t.Fatalf("certNames has %s: %v, want %v", what, got, live) |
| + } |
| + if fx.e.Allowed(n) { |
| + t.Fatalf("Allowed(%s): a remote hold is never this box's own", what) |
| + } |
| + } |
| + } |
| + }) |
| + t.Run("a retained route behind it keeps the name a member, not a holding name", func(t *testing.T) { |
| + fx := newFreshScene(t, "carol") |
| + fx.put("annie", "1.1.1.1", "", holdJSON(host, "mv_a", false, 1, 1)) |
| + fx.put("betty", "2.2.2.2", readyJSON(host, "bab", "shop", 1, 0), "") |
| + fx.mustPoll() |
| + fx.at(holdingValidity - time.Second) |
| + if !fx.m.KnownHost(host, fx.now) || !slices.Contains(fx.m.HoldingHosts(fx.now), host) { |
| + t.Fatal("at 4 m 59 s the name is a member and a holding name") |
| + } |
| + fx.at(holdingValidity) |
| + if !fx.m.KnownHost(host, fx.now) { |
| + t.Fatal("at 5 m 0 s the retained route keeps the name a member") |
| + } |
| + if slices.Contains(fx.m.HoldingHosts(fx.now), host) || slices.Contains(fx.e.certNames(), host) { |
| + t.Fatal("at 5 m 0 s the name is no longer a holding name") |
| + } |
| + }) |
| + t.Run("a local hold is Allowed whatever the remote clock says", func(t *testing.T) { |
| + const remote = "cart.example.org" |
| + fx := newFreshScene(t, "annie") |
| + fx.heldRow(host, "bab", "shop", "mv_r", 1, 1) |
| + fx.view(map[string]HoldState{host: moveR}) |
| + fx.put("betty", "2.2.2.2", "", holdJSON(remote, "mv_b", false, 1, 1)) |
| + fx.mustPoll() |
| + fx.faults.set(true, false) |
| + for _, d := range []time.Duration{holdingValidity - time.Second, holdingValidity, time.Hour} { |
| + fx.at(d) |
| + if fx.poll() == nil { |
| + t.Fatal("test bug: the failed List must fail the poll") |
| + } |
| + what := fmt.Sprintf("the local hold at %v", d) |
| + if !fx.e.Allowed(host) || !fx.e.deploymentHost(host) || !slices.Contains(fx.e.certNames(), host) { |
| + t.Fatalf("%s: Allowed %v, deploymentHost %v, certNames %v", what, fx.e.Allowed(host), fx.e.deploymentHost(host), fx.e.certNames()) |
| + } |
| + // At an hour the page is past its estimate and says so; it is |
| + // still the holding page. |
| + if w := fx.serve(host, "/", ordinary); w.Code != http.StatusServiceUnavailable || |
| + w.Header().Get("Retry-After") != holdingRetryAfter || !strings.Contains(w.Body.String(), "carlos-holding-") { |
| + t.Fatalf("%s = %d %q, want the holding page", what, w.Code, w.Body.String()) |
| + } |
| + live := d < holdingValidity |
| + if got := slices.Contains(fx.e.certNames(), remote); got != live { |
| + t.Fatalf("certNames has the remote hold at %v: %v, want %v", d, got, live) |
| + } |
| + } |
| + }) |
| +} |
| + |
| // Task 2 review Minor 2: the wildcard manager's parent set follows a remote |
| // platform hold's clock. The wake compares the LIVE platform holds at the |
| // poll's time, so the expiry itself wakes the manager (the parent leaves on |