amadan
console: a box with only one of its two objects is unknown liveness, so the evidence is incomplete (re-review r2, rulings A and B reversed)
The liveness join walked only the edges files, so a box with a FRESH status snapshot and no edges file never reached the fold. Reconcile does not need the edges file to admit a box, so a keyless live box whose edges publication failed was invisible while its keyed sibling made the region eligible. Ruling (A): a fresh snapshot whose box has no edges file makes the evidence incomplete, whole-request, like a failed read. Ruling (B), reversed on production evidence: on the flagship every box writes both objects and a retired box leaves both behind, so it reads as a STALE snapshot, never an absent one. An edges file whose box has no status object at all is therefore unknown liveness, and incomplete too; the stale snapshot beside its file is the retired box, not fresh, counting for neither side, and the region stays eligible through its fresh keyed sibling (the F1 case, kept). The join key is the snapshot's Label against the edges file's own box field, the pair edgesfile.EdgeIPs joins on. Each new incompleteness is said once. Three regiondefault tests modelled "absent" by deleting the edges object and leaving the snapshot fresh, which is now a live box that has not published; retireBox deletes the file and stales the snapshot, the production shape. Tests, RED against b01bab82: a fresh keyless status-only box beside the keyed box leaves uk ineligible at the fold and answers C28 at the door (explicit --region and the home app unchanged); a keyless edges file with no status object leaves uk ineligible; the same file beside a stale snapshot leaves uk eligible; a stale status-only box is ignored.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
4 files changed,
+172
−11
internal/console/instancesplacement.go+40 −6internal/console/regioncandidates_test.go+108 −2internal/console/regiondefault_test.go+3 −3internal/console/regionfill_test.go+21 −0
CI passed — run details
diff --git a/internal/console/instancesplacement.go b/internal/console/instancesplacement.go| index a3102aefd..0b9510d44 100644 |
| --- a/internal/console/instancesplacement.go |
| +++ b/internal/console/instancesplacement.go |
| @@ -111,18 +111,50 @@ func readLiveEdgesSnapshot(ctx context.Context, s release.Store, now time.Time) |
| return snap, nil |
| } |
| snap.BoxesComplete = boxesComplete |
| - live := map[string]bool{} |
| + // known is every box with a status object at all; live is the fresh |
| + // subset. The join key is the status snapshot's Label against the edges |
| + // file's own box field, the pair edgesfile.EdgeIPs joins on; the object |
| + // key names play no part. |
| + known, live := map[string]bool{}, map[string]bool{} |
| for _, b := range boxes { |
| + known[b.Label] = true |
| if !status.BoxStale(b, now, status.SnapshotPeriod) { |
| live[b.Label] = true |
| } |
| } |
| + published := make(map[string]bool, len(snap.Files)) |
| joined := 0 |
| for i := range snap.Files { |
| + published[snap.Files[i].Box] = true |
| snap.Files[i].Fresh = live[snap.Files[i].Box] |
| if snap.Files[i].Fresh { |
| joined++ |
| } |
| + // Ruling (B) reversed, re-review r2, on production evidence: every |
| + // box writes both objects and a retired box leaves both behind, so |
| + // an ABSENT snapshot is unknown liveness (incomplete evidence, |
| + // whole-request) and a STALE one is the retired box (not fresh, |
| + // counts for neither side, evidence complete). |
| + if !known[snap.Files[i].Box] { |
| + snap.BoxesComplete = false |
| + if _, said := livenessSaid.LoadOrStore("nostatus", true); !said { |
| + log.Printf("console: box %q has an edges file under %s but no status snapshot at all, so no region outside the home is automatically eligible until it reports; said once", snap.Files[i].Box, edgesFilePrefix) |
| + } |
| + } |
| + } |
| + // Ruling (A), re-review r2: a FRESH snapshot whose box has no edges |
| + // file is incomplete evidence, whole-request, like a failed read. |
| + // Reconcile admits a box without its edges file (its publication may |
| + // simply have failed), so a live box the fold cannot see could still |
| + // take the record. A stale snapshot with no file is a retired box and |
| + // is ignored. |
| + for label := range live { |
| + if !published[label] { |
| + snap.BoxesComplete = false |
| + if _, said := livenessSaid.LoadOrStore("unmatched", true); !said { |
| + log.Printf("console: box %q has a fresh status snapshot but no edges file under %s, so no region outside the home is automatically eligible until it publishes one; said once", label, edgesFilePrefix) |
| + } |
| + } |
| } |
| if joined == 0 { |
| if _, said := livenessSaid.LoadOrStore("none", true); !said { |
| @@ -177,11 +209,13 @@ func readEdgesFiles(ctx context.Context, s release.Store) []edgesFilePlacementVi |
| type edgesSnapshot struct { |
| Files []edgesFilePlacementView |
| Complete bool |
| - // BoxesComplete: every listed status snapshot was read, so each file's |
| - // Fresh is a KNOWN answer. Set only by readLiveEdgesSnapshot; a plain |
| - // readEdgesSnapshot carries no liveness evidence at all. A box whose |
| - // snapshot could not be read has unknown liveness, which is not stale |
| - // (re-review r1, hole 2). |
| + // BoxesComplete: every listed status snapshot was read, and every box |
| + // on either side has its counterpart, so each file's Fresh is a KNOWN |
| + // answer. Set only by readLiveEdgesSnapshot; a plain readEdgesSnapshot |
| + // carries no liveness evidence at all. Unknown liveness is not stale |
| + // (re-review r1 hole 2, r2 rulings (A) and (B) reversed): a snapshot |
| + // that could not be read, a fresh snapshot with no edges file, or an |
| + // edges file with no snapshot at all each make the evidence incomplete. |
| BoxesComplete bool |
| } |
diff --git a/internal/console/regioncandidates_test.go b/internal/console/regioncandidates_test.go| index 2e0f3d77f..c7e3aebef 100644 |
| --- a/internal/console/regioncandidates_test.go |
| +++ b/internal/console/regioncandidates_test.go |
| @@ -131,6 +131,96 @@ func TestIncompleteReadsNeverMakeARegionEligible(t *testing.T) { |
| check("corrupt status snapshot", corrupt, false) |
| } |
| +// Unmatched boxes, ruling (A) (re-review r2): a FRESH status snapshot |
| +// whose box has no edges file is incomplete evidence, whole-request, the |
| +// same as a failed read. Reconcile does not need the edges file to admit a |
| +// box, so a keyless live box whose edges publication failed would |
| +// otherwise never reach the fold while its keyed sibling made the region |
| +// eligible. Matched by the status Box.Label against the edges file's own |
| +// box field, the pair edgesfile.EdgeIPs joins on. A STALE snapshot with no |
| +// edges file is a retired box and is ignored. |
| +func TestFreshStatusWithoutAnEdgesFileIsIncompleteEvidence(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + for key, body := range map[string]string{"ie-box": homeBox, "uk-box": ukEligible} { |
| + if _, err := s.Put(ctx, "control/edges/"+key+".json", []byte(body), ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + setBoxSnapshot(t, s, key, time.Now()) |
| + } |
| + check := func(name string, wantUK bool) { |
| + t.Helper() |
| + snap, err := readLiveEdgesSnapshot(ctx, s, time.Now()) |
| + if err != nil { |
| + t.Fatalf("%s: %v", name, err) |
| + } |
| + cands := regionCandidates(snap, "ie", testKeyFP) |
| + uk, _ := findCandidate(cands, "uk") |
| + home, _ := findCandidate(cands, "") |
| + if uk.Eligible != wantUK { |
| + t.Errorf("%s: uk = %+v, want eligible %v", name, uk, wantUK) |
| + } |
| + if !home.Eligible { |
| + t.Errorf("%s: home = %+v, want eligible regardless", name, home) |
| + } |
| + } |
| + check("control: every fresh box has an edges file", true) |
| + // Box B reports status, fresh, and has published no edges file. |
| + setBoxSnapshot(t, s, "uk-box-2", time.Now()) |
| + check("fresh status-only box", false) |
| + // The same box gone dark: a retired box, ignored. |
| + setBoxSnapshot(t, s, "uk-box-2", time.Now().Add(-time.Hour)) |
| + check("stale status-only box", true) |
| +} |
| + |
| +// Unmatched boxes, ruling (B) REVERSED (re-review r2, production |
| +// evidence): on the flagship every box has both objects, and a retired |
| +// box leaves both behind, so it reads as a STALE snapshot, never an absent |
| +// one. An edges file whose box has no status object at all is therefore |
| +// not a retired box but unknown liveness, and it makes the evidence |
| +// incomplete, whole-request, like (A). A stale snapshot beside the file is |
| +// the retired box: not fresh, counts for neither side, and the region |
| +// stays eligible through its fresh keyed sibling (the F1 case). |
| +func TestEdgesFileWithoutAnyStatusSnapshotIsIncompleteEvidence(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + for key, body := range map[string]string{"ie-box": homeBox, "uk-box": ukEligible} { |
| + if _, err := s.Put(ctx, "control/edges/"+key+".json", []byte(body), ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + setBoxSnapshot(t, s, key, time.Now()) |
| + } |
| + if _, err := s.Put(ctx, "control/edges/uk-box-2.json", []byte(ukKeylessSibling), ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + snap, err := readLiveEdgesSnapshot(ctx, s, time.Now()) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if snap.complete() { |
| + t.Fatalf("a keyless edges file with NO status object left the evidence complete (edges %v, boxes %v)", snap.Complete, snap.BoxesComplete) |
| + } |
| + if c, _ := findCandidate(regionCandidates(snap, "ie", testKeyFP), "uk"); c.Eligible { |
| + t.Fatalf("uk = %+v, want ineligible: a box with no status object has unknown liveness", c) |
| + } |
| + if h, _ := findCandidate(regionCandidates(snap, "ie", testKeyFP), ""); !h.Eligible { |
| + t.Fatalf("home = %+v, want eligible regardless", h) |
| + } |
| + // The retired box: the same edges file beside a STALE snapshot. F1's |
| + // case, and it must keep working. |
| + setBoxSnapshot(t, s, "uk-box-2", time.Now().Add(-time.Hour)) |
| + snap, err = readLiveEdgesSnapshot(ctx, s, time.Now()) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if !snap.complete() { |
| + t.Fatalf("a stale snapshot beside its edges file made the evidence incomplete (edges %v, boxes %v); a retired box is complete evidence", snap.Complete, snap.BoxesComplete) |
| + } |
| + if c, _ := findCandidate(regionCandidates(snap, "ie", testKeyFP), "uk"); !c.Eligible { |
| + t.Fatalf("retired keyless sibling: uk = %+v, want eligible through the fresh keyed box", c) |
| + } |
| +} |
| + |
| func mustEtag(t *testing.T, s release.Store, key string) string { |
| t.Helper() |
| _, etag, err := s.Get(context.Background(), key) |
| @@ -203,6 +293,19 @@ func setBoxSnapshot(t *testing.T, s release.Store, label string, at time.Time) { |
| } |
| } |
| +// retireBox is what a retired box looks like in production (ruling (B) |
| +// reversed, re-review r2): its status snapshot lingers and goes STALE, and |
| +// here its edges file is gone too. Deleting the edges object alone would |
| +// leave a fresh snapshot with no file, which reads as a live box that has |
| +// not published (incomplete evidence), not as an absent region. |
| +func retireBox(t *testing.T, fx *consoleFixture, label string) { |
| + t.Helper() |
| + if err := fx.rel.Delete(context.Background(), "control/edges/"+label+".json"); err != nil && !errors.Is(err, os.ErrNotExist) { |
| + t.Fatal(err) |
| + } |
| + setBoxSnapshot(t, fx.rel, label, time.Now().Add(-time.Hour)) |
| +} |
| + |
| type failingList struct{ release.Store } |
| func (failingList) List(context.Context, string) ([]string, error) { |
| @@ -257,6 +360,7 @@ func TestRegionCandidatesNeedAFreshBoxSnapshot(t *testing.T) { |
| } |
| } |
| now := time.Now() |
| + setBoxSnapshot(t, s, "ie-box", now) // the home box reports too; see ruling (B) reversed |
| uk := func(name string) regionCandidate { |
| t.Helper() |
| snap, err := readLiveEdgesSnapshot(ctx, s, now) |
| @@ -266,6 +370,8 @@ func TestRegionCandidatesNeedAFreshBoxSnapshot(t *testing.T) { |
| c, _ := findCandidate(regionCandidates(snap, "ie", testKeyFP), "uk") |
| return c |
| } |
| + // No snapshot at all: unknown liveness (incomplete evidence, ruling (B) |
| + // reversed), so not eligible either way, and nothing live publishes uk. |
| if c := uk("no snapshot"); c.Eligible || c.unavailableReason() != "absent" { |
| t.Fatalf("no snapshot at all: uk = %+v, want absent", c) |
| } |
| @@ -277,8 +383,8 @@ func TestRegionCandidatesNeedAFreshBoxSnapshot(t *testing.T) { |
| if c := uk("fresh"); !c.Eligible { |
| t.Fatalf("fresh snapshot: uk = %+v, want eligible", c) |
| } |
| - // The home candidate stays as it is: no snapshot was ever written for |
| - // ie-box, and it is eligible throughout. |
| + // The home candidate stays as it is: eligible throughout, exempt from |
| + // every eligibility fact. |
| snap, _ := readLiveEdgesSnapshot(ctx, s, now) |
| if h, _ := findCandidate(regionCandidates(snap, "ie", testKeyFP), ""); !h.Eligible || h.AWSRegion != "eu-west-1" { |
| t.Fatalf("home = %+v", h) |
diff --git a/internal/console/regiondefault_test.go b/internal/console/regiondefault_test.go| index a4c35aa67..4e51842ae 100644 |
| --- a/internal/console/regiondefault_test.go |
| +++ b/internal/console/regiondefault_test.go |
| @@ -61,7 +61,7 @@ func TestAppDefaultReasons(t *testing.T) { |
| effective string |
| }{ |
| {"available", func(*testing.T, *consoleFixture) {}, "", bp(true), "uk"}, |
| - {"absent", func(t *testing.T, fx *consoleFixture) { fx.rel.Delete(t.Context(), "control/edges/uk-box.json") }, "absent", bp(false), "uk"}, |
| + {"absent", func(t *testing.T, fx *consoleFixture) { retireBox(t, fx, "uk-box") }, "absent", bp(false), "uk"}, |
| {"not-hosting", func(t *testing.T, fx *consoleFixture) { |
| setEdges(t, fx, map[string]string{"uk-box": edgeJSON("uk-box", "1.2.3.4", "eu-west-2", "uk", edgeOpts{hib: bp(false), shared: bp(true), inbound: bp(true), fp: testKeyFP})}) |
| }, "not-hosting", bp(false), "uk"}, |
| @@ -371,7 +371,7 @@ func TestInstancesTabRegionLines(t *testing.T) { |
| if strings.Contains(body, `id="app-region-line" class="banner error"`) { |
| t.Error("C7 carries the warning class") |
| } |
| - fx.rel.Delete(t.Context(), "control/edges/uk-box.json") |
| + retireBox(t, fx, "uk-box") |
| // No box publishes uk, so its place falls back to the bare code (§14). |
| body = page(fx) |
| if !strings.Contains(body, cantUK) || strings.Contains(body, starts) { |
| @@ -423,7 +423,7 @@ func TestInstancesTabRegionLineForAHomeApp(t *testing.T) { |
| // Only the home is left: nothing to distinguish it from. |
| fx, _, _ = fillFixture(t, "") |
| - fx.rel.Delete(t.Context(), "control/edges/uk-box.json") |
| + retireBox(t, fx, "uk-box") |
| if body := page(fx); strings.Contains(body, `id="app-region-line"`) { |
| t.Error("one eligible region: a region line showed") |
| } |
diff --git a/internal/console/regionfill_test.go b/internal/console/regionfill_test.go| index 02973d467..7f5e095aa 100644 |
| --- a/internal/console/regionfill_test.go |
| +++ b/internal/console/regionfill_test.go |
| @@ -174,6 +174,27 @@ func TestAppDerivedNeedsCompleteEvidence(t *testing.T) { |
| } |
| } |
| +// Unmatched boxes, ruling (A): a fresh status snapshot for a box with no |
| +// edges file is incomplete evidence at the door too, so an app-derived uk |
| +// create is C28, while an explicit --region uk create and a home app's |
| +// create keep their behaviour. |
| +func TestAppDerivedNeedsEveryFreshBoxToPublishEdges(t *testing.T) { |
| + fx, token, u := fillFixture(t, "uk") |
| + setBoxSnapshot(t, fx.rel, "uk-box-2", time.Now()) |
| + status, _, out := create(t, u, token, `{"host":"y.go.tito.io"}`) |
| + if status != http.StatusServiceUnavailable || !strings.Contains(out, msgAppRegionUnconfirmed("uk")) { |
| + t.Errorf("fresh status-only box, app-derived uk = %d %s, want 503 C28", status, out) |
| + } |
| + if rec := mustCreate(t, u, token, `{"host":"x.go.tito.io","region":"uk"}`); rec.Region != "uk" { |
| + t.Errorf("explicit uk = %+v, want created in uk as before", rec) |
| + } |
| + fx, token, u = fillFixture(t, "") |
| + setBoxSnapshot(t, fx.rel, "uk-box-2", time.Now()) |
| + if rec := mustCreate(t, u, token, `{"host":"h.go.tito.io"}`); rec.Region != "" { |
| + t.Errorf("home app = %+v, want created at the home", rec) |
| + } |
| +} |
| + |
| func TestAppDerivedEdgesReadFailureIs503(t *testing.T) { |
| fx, token, u := fillFixture(t, "uk") |
| fx.srv.cfg.Releases = failingList{fx.rel} |