amadan
region HA 2a: an incomplete placement read is not no room; alert and Retry-After
Task 11 fix round 1, per the controller's rulings: - hostops.ExecutorChooser's Choose, Permits and PlaceableBoxes (and so Capacity and SelectExecutor) answer ErrEvidenceIncomplete when the edges/status snapshot they read is incomplete, never ErrNoEligibleBox or an empty set. The check wraps the chooser's own Edges read (judged); the accepted-legacy ExecutorChooser.eligible is unchanged. Every choosing door answers it as the temporarily unavailable 503 (C28 on the app-derived path), with no page. - The no-capacity alert counts as sent only once the send succeeds: a failed send gives its slot back, so the next refusal pages again. - The no-room 503 carries Retry-After: 60 on every choosing door; every other refusal's Retry-After is unchanged. Census: console and hostops envelopes re-pinned; B8 key-site handleAPIDomainAttach re-digested (one header line, no write site changed); no legacy or authority row moved, seal unchanged.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
9 files changed,
+354
−15
internal/console/apidomains.go+1 −0internal/console/appdomains.go+1 −0internal/console/claim_response.go+1 −0internal/console/nocapacity.go+58 −5internal/console/nocapacity_test.go+197 −0internal/hostops/executor.go+38 −5internal/hostops/executor_test.go+53 −2internal/provision/plan9_b8_census_test.go+2 −1internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv+3 −2
diff --git a/internal/console/apidomains.go b/internal/console/apidomains.go| index a779d2790..c97618b7d 100644 |
| --- a/internal/console/apidomains.go |
| +++ b/internal/console/apidomains.go |
| @@ -452,6 +452,7 @@ func (s *Server) handleAPIDomainAttach(w http.ResponseWriter, r *http.Request) { |
| // route the claim names. |
| res, ref := s.attachDomainAs(ctx, acct, app, host, in.Kind, in.Channel, principal, in.Platform, message) |
| if ref != nil { |
| + setNoCapacityRetryAfter(w, ref) |
| jsonError(w, ref.Status, ref.Msg) |
| return |
| } |
diff --git a/internal/console/appdomains.go b/internal/console/appdomains.go| index a9f6427ba..35460b721 100644 |
| --- a/internal/console/appdomains.go |
| +++ b/internal/console/appdomains.go |
| @@ -153,5 +153,6 @@ func (s *Server) renderDoorRefusal(w http.ResponseWriter, r *http.Request, ref * |
| http.NotFound(w, r) |
| return |
| } |
| + setNoCapacityRetryAfter(w, ref) |
| s.renderMessage(w, r, ref.Status, "Refused", ref.Msg) |
| } |
diff --git a/internal/console/claim_response.go b/internal/console/claim_response.go| index 7f61b7669..b48535d94 100644 |
| --- a/internal/console/claim_response.go |
| +++ b/internal/console/claim_response.go |
| @@ -29,6 +29,7 @@ func claimResponseRefusal(err error) *doorRefusal { |
| func writeClaimResponseError(w http.ResponseWriter, err error) { |
| ref := claimResponseRefusal(err) |
| w.Header().Set("Retry-After", "2") |
| + setNoCapacityRetryAfter(w, ref) |
| jsonError(w, ref.Status, ref.Msg) |
| } |
diff --git a/internal/console/nocapacity.go b/internal/console/nocapacity.go| index 0c0071150..e17912324 100644 |
| --- a/internal/console/nocapacity.go |
| +++ b/internal/console/nocapacity.go |
| @@ -5,6 +5,8 @@ import ( |
| "errors" |
| "fmt" |
| "log" |
| + "net/http" |
| + "strings" |
| "sync" |
| "time" |
| @@ -21,11 +23,18 @@ import ( |
| // one extra alert, which H27 accepts rather than persist the limit. A |
| // create door chooses before it registers anything (chooseExecutorFirst), |
| // so a refusal leaves nothing registered, nothing written to the claim and |
| -// nothing for recovery to settle. |
| +// nothing for recovery to settle. Only a complete placement read with no |
| +// placeable box is "no room": an incomplete one (hostops.ErrEvidenceIncomplete) |
| +// is answered as temporarily unavailable and pages nobody. |
| // noCapacityAlertEvery is how often one region's alert may repeat. |
| const noCapacityAlertEvery = time.Hour |
| +// noCapacityRetryAfter is the Retry-After (seconds) a no-room 503 carries: |
| +// RHA-C17 tells the caller to try again in a few minutes. Every other |
| +// claim refusal keeps its own. |
| +const noCapacityRetryAfter = "60" |
| + |
| func noCapacityRefusal(region string) string { |
| return fmt.Sprintf("No box in %s has room right now. Try again in a few minutes.", region) |
| } |
| @@ -50,7 +59,11 @@ func (e *noCapacityError) Is(target error) bool { return target == hostops.ErrNo |
| var pageNoCapacity = alerting.Page |
| // noCapacityAlerts remembers when each region's alert was last sent, in |
| -// this console process. The zero value is ready. |
| +// this console process. An alert counts as sent only once its send |
| +// succeeds: due reserves the region's slot so concurrent refusals do not |
| +// page twice while one send is in flight, and failed gives the slot back, |
| +// so the next refusal tries again rather than the region going quiet for |
| +// an hour. The zero value is ready. |
| type noCapacityAlerts struct { |
| mu sync.Mutex |
| last map[string]time.Time |
| @@ -71,18 +84,55 @@ func (a *noCapacityAlerts) due(region string, now time.Time) bool { |
| return true |
| } |
| +// failed gives back the slot due reserved at for region, because that send |
| +// failed. A later reservation is left alone. |
| +func (a *noCapacityAlerts) failed(region string, at time.Time) { |
| + a.mu.Lock() |
| + defer a.mu.Unlock() |
| + if last, ok := a.last[region]; ok && last.Equal(at) { |
| + delete(a.last, region) |
| + } |
| +} |
| + |
| +// isNoCapacityRefusal reports whether ref is the no-room refusal |
| +// (claimResponseRefusal's answer to a noCapacityError), which the attach |
| +// doors carry as a doorRefusal. |
| +func isNoCapacityRefusal(ref *doorRefusal) bool { |
| + if ref == nil || ref.Status != http.StatusServiceUnavailable { |
| + return false |
| + } |
| + before, after, _ := strings.Cut(noCapacityRefusal("\x00"), "\x00") |
| + region, ok := strings.CutPrefix(ref.Msg, before) |
| + if !ok { |
| + return false |
| + } |
| + region, ok = strings.CutSuffix(region, after) |
| + return ok && region != "" |
| +} |
| + |
| +// setNoCapacityRetryAfter stamps noCapacityRetryAfter on w when ref is the |
| +// no-room refusal, and leaves every other refusal's headers as its door |
| +// writes them. |
| +func setNoCapacityRetryAfter(w http.ResponseWriter, ref *doorRefusal) { |
| + if isNoCapacityRefusal(ref) { |
| + w.Header().Set("Retry-After", noCapacityRetryAfter) |
| + } |
| +} |
| + |
| // noRoom is the refusal for region (record vocabulary, "" for the home): |
| // it logs, sends the operator alert when one is due, and returns the |
| // error the door answers with. |
| func (s *Server) noRoom(ctx context.Context, region string) error { |
| name := s.homeRegionOut(region) |
| log.Printf("console: refused: no eligible box in %s", name) |
| - if s.noCapacity.due(name, time.Now()) { |
| + if now := time.Now(); s.noCapacity.due(name, now) { |
| subject, body := noCapacityAlert(name) |
| go func() { |
| ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) |
| defer cancel() |
| if err := pageNoCapacity(ctx, alerting.LevelWarning, subject, body); err != nil { |
| + // Not sent: the next refusal in name tries again. |
| + s.noCapacity.failed(name, now) |
| log.Printf("console: no-capacity alert for %s: %v", name, err) |
| } |
| }() |
| @@ -110,7 +160,9 @@ func (s *Server) placementChooser(pool string) *hostops.ExecutorChooser { |
| // role), is the no-room refusal, never a 400. It only refuses: it chooses |
| // nothing and changes nothing the door goes on to do. It answers nil with |
| // region HA off, for a region the roster does not place (the door's own |
| -// checks answer that), and when a box has room. |
| +// checks answer that), and when a box has room. An incomplete read is |
| +// hostops.ErrEvidenceIncomplete, which the door answers as temporarily |
| +// unavailable, with no alert. |
| func (s *Server) roomIn(ctx context.Context, region, pool string) error { |
| x := s.placementChooser(pool) |
| if x == nil { |
| @@ -150,7 +202,8 @@ type chosenOn struct { |
| // registered, and carries the choice to the create |
| // (hostops.WithChosenExecutor). It takes no lease and writes nothing: the |
| // choice is SelectExecutor's read of the roster and the edges files. No |
| -// eligible box is noRoom's refusal. A choice ctx already carries for host |
| +// eligible box on a complete read is noRoom's refusal; an incomplete read |
| +// (hostops.ErrEvidenceIncomplete) is returned as it is. A choice ctx already carries for host |
| // (chooseCreateFirst's) is kept only when before is the claim observation |
| // it was decided on (census ruling region-ha-2a, item 5): otherwise the |
| // create is refused with errChoiceStale, and the old choice is never used |
diff --git a/internal/console/nocapacity_test.go b/internal/console/nocapacity_test.go| index ad3a49385..9facac57d 100644 |
| --- a/internal/console/nocapacity_test.go |
| +++ b/internal/console/nocapacity_test.go |
| @@ -6,6 +6,7 @@ import ( |
| "errors" |
| "fmt" |
| "net/http" |
| + "net/http/httptest" |
| "strings" |
| "testing" |
| "time" |
| @@ -551,3 +552,199 @@ func TestPlacementChooserIsScoped(t *testing.T) { |
| t.Fatal("region HA off: a chooser or a refusal") |
| } |
| } |
| + |
| +// Region HA 2a Task 11 fix round 1 (ruling: an incomplete placement read |
| +// is not "no room"). A malformed edges file beside a placeable box makes |
| +// the read incomplete: every choosing door answers main's temporarily |
| +// unavailable 503 (on the app-derived path C28's could-not-confirm), never |
| +// RHA-C17, pages nobody, and writes nothing. |
| +func TestAnIncompletePlacementReadIsNotNoRoom(t *testing.T) { |
| + malformed := func(t *testing.T, cx *claimCase) { |
| + t.Helper() |
| + if _, err := cx.rel.Put(context.Background(), "control/edges/box-x.json", []byte("{"), ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + unavailable := func(t *testing.T, r claimReply, msg string) { |
| + t.Helper() |
| + if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, msg) || strings.Contains(r.Body, noCapacityRefusal("ie")) { |
| + t.Fatalf("answer %d %s, want 503 %q and not the no-room refusal", r.Status, r.Body, msg) |
| + } |
| + } |
| + t.Run("instance create, an explicit region", func(t *testing.T) { |
| + pages := pagesTo(t) |
| + cx := haClaimCase(t, "") |
| + cx.instanceWorld(t) |
| + publishHABox(t, cx, "box-ie-tito", false, nil) |
| + malformed(t, cx) |
| + before := registrations(t, cx, "bab", "titogo") |
| + mark := cx.rec.mark() |
| + status, hdr, body := rawReq(t, http.MethodPost, cx.ts.URL+"/api/cli/apps/bab/titogo/instances", cx.bearer, `{"host":"acme.go.tito.io","region":"ie"}`) |
| + unavailable(t, claimReply{Status: status, Body: body}, claimResponseUnavailable) |
| + if got := hdr.Get("Retry-After"); got != "2" { |
| + t.Fatalf("Retry-After %q, want main's 2", got) |
| + } |
| + wroteNothing(t, cx, mark) |
| + refusedBeforeRegistering(t, cx, "acme.go.tito.io", before) |
| + expectPages(t, pages) |
| + }) |
| + t.Run("standalone sidecar create", func(t *testing.T) { |
| + pages := pagesTo(t) |
| + cx := haClaimCase(t, "") |
| + cx.sidecarWorld(t) |
| + publishHABox(t, cx, "box-ie-tito", false, nil) |
| + malformed(t, cx) |
| + before := registrations(t, cx, "bab", "titogo") |
| + mark := cx.rec.mark() |
| + unavailable(t, cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/sidecars", `{"name":"bot","region":"ie"}`), claimResponseUnavailable) |
| + wroteNothing(t, cx, mark) |
| + refusedBeforeRegistering(t, cx, provision.SidecarHost("bot", "titogo", "bab"), before) |
| + expectPages(t, pages) |
| + }) |
| + t.Run("domain attach, a first route", func(t *testing.T) { |
| + pages := pagesTo(t) |
| + cx := haClaimCase(t, "ie") |
| + shipVersion(t, cx.consoleFixture, "titogo", "v1") |
| + seedChannel(t, cx.consoleFixture, "titogo", "edge", "v1") |
| + publishHABox(t, cx, "box-ie-default", false, nil) |
| + malformed(t, cx) |
| + before := registrations(t, cx, "bab", "titogo") |
| + mark := cx.rec.mark() |
| + unavailable(t, cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/domains", `{"host":"first.titogo-claim.example","kind":"static"}`), claimResponseUnavailable) |
| + wroteNothing(t, cx, mark) |
| + refusedBeforeRegistering(t, cx, "first.titogo-claim.example", before) |
| + expectPages(t, pages) |
| + }) |
| + t.Run("instance create, the app's region", func(t *testing.T) { |
| + pages := pagesTo(t) |
| + cx := haClaimCase(t, "") |
| + appDerivedWorld(t, cx) |
| + publishHABox(t, cx, "box-ie-default", false, nil) |
| + malformed(t, cx) |
| + before := registrations(t, cx, "bab", "titogo") |
| + unavailable(t, cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", `{"host":"acme.go.tito.io"}`), msgAppRegionUnconfirmed("ie")) |
| + refusedBeforeRegistering(t, cx, "acme.go.tito.io", before) |
| + expectPages(t, pages) |
| + }) |
| +} |
| + |
| +// Region HA 2a Task 11 fix round 1 (ruling): the no-capacity alert is |
| +// recorded as sent only once a send succeeds, so a failed send is retried |
| +// on the next refusal rather than silenced for an hour; a sent alert |
| +// still silences the region for the hour. |
| +func TestNoCapacityAlertIsRecordedOnlyWhenSent(t *testing.T) { |
| + var a noCapacityAlerts |
| + t0 := time.Date(2026, 10, 16, 12, 0, 0, 0, time.UTC) |
| + if !a.due("uk", t0) { |
| + t.Fatal("the first alert is not due") |
| + } |
| + if a.due("uk", t0.Add(time.Second)) { |
| + t.Fatal("a second alert while the first is being sent") |
| + } |
| + a.failed("uk", t0) |
| + if !a.due("uk", t0.Add(2*time.Second)) { |
| + t.Fatal("a failed send silenced the region") |
| + } |
| + a.failed("uk", t0) // an older send's failure does not free a newer one |
| + if a.due("uk", t0.Add(3*time.Second)) { |
| + t.Fatal("a stale failure freed the region") |
| + } |
| + |
| + sent := make(chan string, 16) |
| + calls := 0 |
| + old := pageNoCapacity |
| + pageNoCapacity = func(_ context.Context, level, subject, _ string) error { |
| + calls++ |
| + sent <- subject |
| + if calls == 1 { |
| + return errors.New("the mail relay is down") |
| + } |
| + return nil |
| + } |
| + t.Cleanup(func() { pageNoCapacity = old }) |
| + s := &Server{} |
| + ctx := context.Background() |
| + want, _ := noCapacityAlert("ie") |
| + if err := s.noRoom(ctx, "ie"); !errors.Is(err, hostops.ErrNoEligibleBox) { |
| + t.Fatalf("noRoom = %v", err) |
| + } |
| + if got := <-sent; got != want { |
| + t.Fatalf("page %q", got) |
| + } |
| + // The failed send is given back once it returns: a later refusal pages. |
| + deadline := time.Now().Add(5 * time.Second) |
| + for second := false; !second; { |
| + if time.Now().After(deadline) { |
| + t.Fatal("a failed alert was never sent again") |
| + } |
| + _ = s.noRoom(ctx, "ie") |
| + select { |
| + case got := <-sent: |
| + if got != want { |
| + t.Fatalf("page %q", got) |
| + } |
| + second = true |
| + case <-time.After(20 * time.Millisecond): |
| + } |
| + } |
| + // That one was sent: the region is quiet for the hour. |
| + for range 3 { |
| + _ = s.noRoom(ctx, "ie") |
| + } |
| + select { |
| + case got := <-sent: |
| + t.Fatalf("an extra page after a sent alert: %q", got) |
| + case <-time.After(200 * time.Millisecond): |
| + } |
| +} |
| + |
| +// Region HA 2a Task 11 fix round 1 (ruling): the no-room 503 carries |
| +// Retry-After: 60, matching RHA-C17's "try again in a few minutes", on |
| +// every choosing door; every other claim refusal keeps its own. |
| +func TestNoRoomCarriesRetryAfterAMinute(t *testing.T) { |
| + pagesTo(t) |
| + for _, d := range []struct { |
| + name, home, path, body string |
| + world func(*testing.T, *claimCase) |
| + }{ |
| + {"instance create", "", "/api/cli/apps/bab/titogo/instances", `{"host":"acme.go.tito.io","region":"ie"}`, func(t *testing.T, cx *claimCase) { cx.instanceWorld(t) }}, |
| + {"standalone sidecar create", "", "/api/cli/apps/bab/titogo/sidecars", `{"name":"bot","region":"ie"}`, func(t *testing.T, cx *claimCase) { cx.sidecarWorld(t) }}, |
| + {"domain attach, a first route", "ie", "/api/cli/apps/bab/titogo/domains", `{"host":"first.titogo-claim.example","kind":"static"}`, func(t *testing.T, cx *claimCase) { |
| + shipVersion(t, cx.consoleFixture, "titogo", "v1") |
| + seedChannel(t, cx.consoleFixture, "titogo", "edge", "v1") |
| + }}, |
| + } { |
| + cx := haClaimCase(t, d.home) |
| + d.world(t, cx) |
| + publishHABox(t, cx, "box-ie-tito", true, nil) |
| + publishHABox(t, cx, "box-ie-default", true, nil) |
| + status, hdr, body := rawReq(t, http.MethodPost, cx.ts.URL+d.path, cx.bearer, d.body) |
| + if status != http.StatusServiceUnavailable || !strings.Contains(body, noCapacityRefusal("ie")) || hdr.Get("Retry-After") != "60" { |
| + t.Fatalf("%s: %d Retry-After %q %s, want the no-room 503 with Retry-After 60", d.name, status, hdr.Get("Retry-After"), body) |
| + } |
| + } |
| + cx := haClaimCase(t, "") |
| + // The browser's attach door renders the same refusal. |
| + req := httptest.NewRequest(http.MethodPost, "/a/bab/apps/titogo/domains/attach", nil) |
| + rec := httptest.NewRecorder() |
| + cx.srv.renderDoorRefusal(rec, req, claimResponseRefusal(&noCapacityError{Region: "ie"})) |
| + if rec.Code != http.StatusServiceUnavailable || rec.Header().Get("Retry-After") != "60" { |
| + t.Fatalf("browser refusal %d Retry-After %q", rec.Code, rec.Header().Get("Retry-After")) |
| + } |
| + // Other refusals are as they were. |
| + rec = httptest.NewRecorder() |
| + writeClaimResponseError(rec, hostclaim.ErrConflict) |
| + if rec.Header().Get("Retry-After") != "2" { |
| + t.Fatalf("busy refusal Retry-After %q, want 2", rec.Header().Get("Retry-After")) |
| + } |
| + rec = httptest.NewRecorder() |
| + writeClaimResponseError(rec, hostops.ErrEvidenceIncomplete) |
| + if rec.Code != http.StatusServiceUnavailable || rec.Header().Get("Retry-After") != "2" { |
| + t.Fatalf("unavailable refusal %d Retry-After %q, want 503 and 2", rec.Code, rec.Header().Get("Retry-After")) |
| + } |
| + rec = httptest.NewRecorder() |
| + cx.srv.renderDoorRefusal(rec, req, &doorRefusal{http.StatusServiceUnavailable, claimResponseUnavailable}) |
| + if rec.Header().Get("Retry-After") != "" { |
| + t.Fatalf("browser unavailable refusal Retry-After %q, want none (main's)", rec.Header().Get("Retry-After")) |
| + } |
| +} |
diff --git a/internal/hostops/executor.go b/internal/hostops/executor.go| index 8291e53b0..6541cbed2 100644 |
| --- a/internal/hostops/executor.go |
| +++ b/internal/hostops/executor.go |
| @@ -16,13 +16,21 @@ import ( |
| // eligible (region HA §5.2). The console maps it to today's refusal. |
| var ErrNoEligibleBox = errors.New("hostops: region has no box that can take this") |
| +// ErrEvidenceIncomplete is a placement read that could not judge every |
| +// box: an edges file unread or malformed, the status listing unread, or a |
| +// fresh box with no edges file (boxchoice.Snapshot.Complete false). It is |
| +// never ErrNoEligibleBox: a door answers it as temporarily unavailable, |
| +// and nothing pages "no room" for it (region HA 2a, Task 11 ruling). |
| +var ErrEvidenceIncomplete = errors.New("hostops: the placement read was incomplete, so no box could be judged") |
| + |
| // ExecutorChooser is create's executor selection: claim P5-D2's choose() |
| // over the complete eligible set (region HA spec 1 §5.2, H12, L8). It is |
| // made once, before the create CAS, and recorded in the create intent. |
| type ExecutorChooser struct { |
| Directory hostactors.Directory |
| // Edges is one read of every box's edges file with the caller's |
| - // liveness verdict per box. An incomplete snapshot selects nothing. |
| + // liveness verdict per box. An incomplete snapshot selects nothing: |
| + // every question answers ErrEvidenceIncomplete (judged). |
| Edges func(context.Context) (boxchoice.Snapshot, error) |
| // Loads counts claimed hosts per box and per "acct/app", from the |
| // claim-recovery listing (claim §5.1 step 5). A stale count only |
| @@ -51,7 +59,7 @@ const sharedPool = "default" |
| // it, is never chosen. Among the rest, region HA spec 2's policy (§7) |
| // decides: placeable boxes only (placeable), then boxchoice.ChooseRoomiest. |
| func (x ExecutorChooser) Choose(ctx context.Context, obs hostclaim.Observed, r Record) (hostclaim.ExecutorSelection, error) { |
| - o, cands, err := x.eligible(ctx, obs, r) |
| + o, cands, err := x.judged(ctx, obs, r) |
| if err != nil { |
| return hostclaim.ExecutorSelection{}, err |
| } |
| @@ -75,7 +83,7 @@ func (x ExecutorChooser) Choose(ctx context.Context, obs hostclaim.Observed, r R |
| // does a box spec 2's policy does not place on (placeable): a legacy or |
| // platform role, too little memory or disk, or stale headroom. |
| func (x ExecutorChooser) Permits(ctx context.Context, obs hostclaim.Observed, r Record, box string) error { |
| - o, cands, err := x.eligible(ctx, obs, r) |
| + o, cands, err := x.judged(ctx, obs, r) |
| if err != nil { |
| return err |
| } |
| @@ -90,10 +98,11 @@ func (x ExecutorChooser) Permits(ctx context.Context, obs hostclaim.Observed, r |
| // PlaceableBoxes is every box in placement that could take a new |
| // instance in the chooser's namespace and pool now, sorted: Choose's set, |
| // before the choice. The console's move door takes a move's destination |
| -// from it (region HA spec 2 §7, H27). |
| +// from it (region HA spec 2 §7, H27). An incomplete read is |
| +// ErrEvidenceIncomplete, never an empty set. |
| func (x ExecutorChooser) PlaceableBoxes(ctx context.Context, placement string) ([]string, error) { |
| r := Record{Holder: hostclaim.NewHolder{Region: placement}} |
| - o, cands, err := x.eligible(ctx, hostclaim.Observed{Absent: true}, r) |
| + o, cands, err := x.judged(ctx, hostclaim.Observed{Absent: true}, r) |
| if err != nil { |
| return nil, err |
| } |
| @@ -112,6 +121,30 @@ func (x ExecutorChooser) Capacity(ctx context.Context, placement string) (int, e |
| return len(boxes), err |
| } |
| +// judged is eligible's read, refused with ErrEvidenceIncomplete when the |
| +// edges snapshot it read was incomplete: eligible's empty set then means |
| +// "could not tell", never "no room". eligible itself (an accepted-legacy |
| +// census unit) is unchanged; the snapshot's completeness is taken from |
| +// the same Edges read it makes, through a wrapper on this copy of x. |
| +func (x ExecutorChooser) judged(ctx context.Context, obs hostclaim.Observed, r Record) (hostactors.Observed, []boxchoice.Candidate, error) { |
| + complete := false |
| + if edges := x.Edges; edges != nil { |
| + x.Edges = func(ctx context.Context) (boxchoice.Snapshot, error) { |
| + snap, err := edges(ctx) |
| + complete = err == nil && snap.Complete |
| + return snap, err |
| + } |
| + } |
| + o, cands, err := x.eligible(ctx, obs, r) |
| + if err != nil { |
| + return o, cands, err |
| + } |
| + if !complete { |
| + return hostactors.Observed{}, nil, ErrEvidenceIncomplete |
| + } |
| + return o, cands, nil |
| +} |
| + |
| // placeable is region HA spec 2's policy over the eligible set (§7): |
| // each box's roster role, then boxchoice.Placeable's thresholds at |
| // x.Now(). Identity was Eligible's; nothing here reads a provider. |
diff --git a/internal/hostops/executor_test.go b/internal/hostops/executor_test.go| index 8c172ee6e..890e93d37 100644 |
| --- a/internal/hostops/executor_test.go |
| +++ b/internal/hostops/executor_test.go |
| @@ -246,10 +246,11 @@ func TestExecutorChooserNeverNamesAFencedLostOrUnknownBox(t *testing.T) { |
| } |
| }) |
| t.Run("an incomplete snapshot", func(t *testing.T) { |
| + // Region HA 2a Task 11 ruling: a partial read is not "no room". |
| x, _, snap, _ := chooserFixture(t, "box-a") |
| snap.Complete = false |
| - if _, err := x.Choose(ctx, absent, chooserRecord()); !errors.Is(err, ErrNoEligibleBox) { |
| - t.Fatalf("chose from a partial read: %v", err) |
| + if _, err := x.Choose(ctx, absent, chooserRecord()); !errors.Is(err, ErrEvidenceIncomplete) || errors.Is(err, ErrNoEligibleBox) { |
| + t.Fatalf("chose from a partial read: %v, want ErrEvidenceIncomplete", err) |
| } |
| }) |
| t.Run("an unreadable fence", func(t *testing.T) { |
| @@ -519,3 +520,53 @@ func TestSelectExecutorHonoursTheDoorsChoice(t *testing.T) { |
| t.Fatalf("region HA off selected: %v %v", ok, err) |
| } |
| } |
| + |
| +// Region HA 2a Task 11 ruling: an incomplete placement read (an unreadable |
| +// or malformed edges file, an unread status listing, a fresh box with no |
| +// edges file: boxchoice.Snapshot.Complete false) is ErrEvidenceIncomplete |
| +// from every chooser question, never ErrNoEligibleBox or an empty set, so |
| +// a door never answers or pages a partial read as "no room". The same |
| +// placeable box is chosen, permitted and counted once the read completes. |
| +func TestExecutorChooserAnIncompleteReadIsNotNoRoom(t *testing.T) { |
| + ctx := context.Background() |
| + x, _, snap, _ := chooserFixture(t, "box-a") |
| + absent := hostclaim.Observed{Host: "shop.bab.oncarlos.com", Absent: true} |
| + snap.Complete = false |
| + incomplete := func(what string, err error) { |
| + t.Helper() |
| + if !errors.Is(err, ErrEvidenceIncomplete) || errors.Is(err, ErrNoEligibleBox) { |
| + t.Fatalf("%s on a partial read = %v, want ErrEvidenceIncomplete", what, err) |
| + } |
| + } |
| + _, err := x.Choose(ctx, absent, chooserRecord()) |
| + incomplete("Choose", err) |
| + incomplete("Permits", x.Permits(ctx, absent, chooserRecord(), "box-a")) |
| + boxes, err := x.PlaceableBoxes(ctx, "ie") |
| + incomplete("PlaceableBoxes", err) |
| + if boxes != nil { |
| + t.Fatalf("PlaceableBoxes named %v from a partial read", boxes) |
| + } |
| + _, err = x.Capacity(ctx, "ie") |
| + incomplete("Capacity", err) |
| + _, _, err = (ProductionAuthority{Namespace: "platform", Executors: &x}).SelectExecutor(ctx, absent, chooserRecord()) |
| + incomplete("SelectExecutor", err) |
| + |
| + snap.Complete = true |
| + if sel, err := x.Choose(ctx, absent, chooserRecord()); err != nil || sel.Box != "box-a" { |
| + t.Fatalf("Choose on a complete read: %+v %v", sel, err) |
| + } |
| + if err := x.Permits(ctx, absent, chooserRecord(), "box-a"); err != nil { |
| + t.Fatalf("Permits on a complete read: %v", err) |
| + } |
| + if n, err := x.Capacity(ctx, "ie"); err != nil || n != 1 { |
| + t.Fatalf("Capacity on a complete read: %d %v", n, err) |
| + } |
| + // A complete read with no placeable box is still no room. |
| + snap.Boxes[0].Fresh = false |
| + if _, err := x.Choose(ctx, absent, chooserRecord()); !errors.Is(err, ErrNoEligibleBox) { |
| + t.Fatalf("Choose with every box stale: %v, want ErrNoEligibleBox", err) |
| + } |
| + if n, err := x.Capacity(ctx, "ie"); err != nil || n != 0 { |
| + t.Fatalf("Capacity with every box stale: %d %v", n, err) |
| + } |
| +} |
diff --git a/internal/provision/plan9_b8_census_test.go b/internal/provision/plan9_b8_census_test.go| index b9de09793..9f6ca38a5 100644 |
| --- a/internal/provision/plan9_b8_census_test.go |
| +++ b/internal/provision/plan9_b8_census_test.go |
| @@ -662,7 +662,8 @@ var plan9B8CensusInventory = []censusEntry{ |
| {"internal/boxvolumes/volumes.go", "Write", "nonrecord", "sel:Put", "70ec487d1180d4abe18031d07f49ea98ff7d7fb222403c1ed9304c00429e7825"}, |
| {"internal/console/account.go", "WriteAccountManifest", "nonrecord", "sel:Put", "f6f1ec0f2aa488ab51850ccf47753e37736d65b94708a1cf25ee0af1152ce251"}, |
| {"internal/console/apidbclaim.go", "Server.dbClaim", "key-site", "key:RecordKey", "ba56b1742d07a47e9ccb17ff1f7a4def9722c2ef0ed51eecf6fd8550d292fa31"}, |
| - {"internal/console/apidomains.go", "Server.handleAPIDomainAttach", "key-site", "lit:instances", "f235eb06f14a7a2d74b69214076329b7d506eb5d49532ba74aa78b61e90fc5e2"}, |
| + // Re-reviewed for region HA 2a Task 11 fix round 1: one header line before the attach core's refusal is written (setNoCapacityRetryAfter: Retry-After 60 on the no-room 503 only). The instances key site and every write are unchanged. |
| + {"internal/console/apidomains.go", "Server.handleAPIDomainAttach", "key-site", "lit:instances", "df556c7622bbc7e0fa1d01b7c0246b58f53131fbc09b26f8a166b1b7ab594f42"}, |
| {"internal/console/apidoorkeys.go", "Server.publishDoorKeysLocked", "foreign-recordkey", "key:RecordKey,sel:Put", "0fc498cd4af50570a0b7ed3ff71994213cab40040f6701a303bdf85d94f958da"}, |
| {"internal/console/apiinstancemove.go", "Server.moveTransitionSaying", "record-writer", "key:SetRecordMove", "d1f7162eecdc234607c708a83e9633cdf77cc6931bac6754fb51042dd8409377"}, |
| // Re-reviewed at the merge of carlos-run into seeded-instances (2026-10-13): main's door with the seed rule (refuseSeedMove) after its gate and before the transition; the transition is unchanged and refuses the same seed again under its CAS. |
diff --git a/internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv b/internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv| index 1df9d4843..1299a57bd 100644 |
| --- a/internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv |
| +++ b/internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv |
| @@ -10,17 +10,18 @@ |
| # Task 10: the operator's box and group role doors (internal/console apiboxroles.go, registered with host claims on in registerCLIRoutes) and the CLI (system_roles.go). Census ruling region-ha-2a (platform-1e, approved with conditions) covers one accepted-legacy unit, Server.boxRoster: the role doors' read of the signed roster (Directory.Read), operator-only (it is reached only past apiSystemRoller), read-only, the same roster every reader reads, and no role it reads feeds root's claim authority. Its basis is main's for a console root input (J4). The seal moves. |
| # Task 10 (system.go): the box and group verbs add `box` and `group` to systemVerb's verb list and two cases to runSystem's dispatch, calling runSystemBoxVerb and runSystemGroupVerb (system_roles.go). The two seed-overlay authority declarations are re-digested, class, task, basis and (empty) kinds unchanged: no root read is added, and the verbs reach the console over the operator's CLI session. |
| # Task 11: H27 on the create doors (internal/console nocapacity.go; hostops chosen.go): beginClaimCreate chooses the executor before it registers anything (chooseExecutorFirst) and carries the choice to the create (hostops.WithChosenExecutor, honoured by ProductionAuthority.SelectExecutor), claimResponseRefusal answers no room with a 503 on instance create, domain attach's first route (attachClaimFirstRoute, unchanged) and standalone sidecar create, and handleAPIInstanceCreate judges an app-derived region by the chooser's eligible set. handleAPIInstanceCreate chooses before its first write, the environment binding (chooseCreateFirst), and beginClaimCreate keeps that choice; with region HA on, an explicit region with no box that has room is the no-room refusal before the armed-placement gate (roomIn). Census ruling region-ha-2a covers two accepted-legacy units: Server.beginClaimCreate re-frozen (one call added before RegisterStart, its one root-read site, the claim read file:Read@470, unchanged), and Server.chooseCreateFirst, new: the same claim read, made before the binding, feeding only the placement choice; it returns its choice with the claim observation it decided on, and beginClaimCreate keeps the choice only if its own read is that observation (ruling item 5). The seal moves. |
| +# Task 11 fix round 1: an incomplete placement read is not "no room" (controller ruling). hostops.ExecutorChooser's Choose, Permits and PlaceableBoxes ask judged, which wraps the chooser's own Edges read and answers ErrEvidenceIncomplete when that snapshot is incomplete; ExecutorChooser.eligible (accepted-legacy, merge ruling 2) is unchanged. The no-capacity alert is recorded only once sent (noCapacityAlerts.failed gives a failed send's slot back), and the no-room 503 carries Retry-After: 60 (setNoCapacityRetryAfter in writeClaimResponseError, renderDoorRefusal and handleAPIDomainAttach's refusal write). Envelope-only: no root-read unit changed or was added, and the legacy seal stands. |
| internal/activator/activator.go type Service authority B,C,D,F,J4 rev3=PROPOSED-seed; existing activator lifecycle and due-wake state plus owned seed exclusion and incarnation-keyed diagnostics; normal refusal and preparation ordering remain authoritative be4ca3a3ea2bd768e7e52c1056513d4fef1c1702d2e04656c540cdcbb858865c |
| @coverage internal/lbhealth ad1211a01a4eb3d02cd8ed95cbff0f8b17b3976ce9c0279ebd2cae2f20c07124 |
| @coverage internal/hostactors 6736698380a247764ada687f2476f89382fc32a1183a50c06afe75967d716eab |
| @coverage internal/boxchoice 1bfd561bcb48b51e60e63f99e086fd44d3b1be2160e28b3ed0253a14bbb9ff66 |
| @coverage internal/edgesfile f62c3973bb48fa36f561d26f496c0ba9a14fe882fe5843867e9f7a9eaf0abbdf |
| @coverage internal/host 4ad9bdcd8b94516a6ccdad3bc900b166833faf1f0334ac83478be5b04de47069 |
| -@coverage internal/hostops 6e494cc9e53bf4fa4433fad798ecd3493c972f65083b0b2153b253f0bd50dcba |
| +@coverage internal/hostops 2a6dc31a50858f27e9b96ce89f98d8ec184a0d524ba77ecb64931cf054186788 |
| @coverage internal/edge 20177c9e7dc961937677bbf2763c73004d7c95f68512591800f6fe841c36152f |
| internal/host/edgesfile.go edgesFileWriteClaims authority F,J1 root-route;kept=F;scope=protected-decision-only registry:Account@10835,registry:Alias@5297,registry:Alias@5332,registry:Alias@5754,registry:Alias@5911,registry:Alias@5981,registry:Alias@6013,registry:Alias@6482,registry:Alias@8772,registry:App@10847,registry:DNSStatus@5429,registry:DNSStatus@5920,registry:Host@10480,registry:Host@2878,registry:Host@5389,registry:Host@6234,registry:Host@6567,registry:Host@8819,registry:Host@8866,registry:Host@8916,registry:Host@8951,registry:Host@9000,registry:Platform@2921,registry:Provisioned@2935,registry:Replica@2959,registry:Status@877,whole:Route@3212,whole:Route@323,whole:Route@3505,whole:Route@351,whole:RouteAlias@5282,whole:RouteAlias@8725,whole:RoutePath@3520,whole:WildcardDomain@8905 8c7cdffba66886b92f9a2ac663a86dbdd5f80699afd2e6afb4571da916c727ad |
| @coverage internal/activator fd9cf14712f55895541bb8bfbf8137cfd446298bcaa8634b0aba7b8344c29ec3 |
| -@coverage internal/console b3f337a5ba7806d28fe6c4d92882accba8572f94d15b008d793f57c26f1bee34 |
| +@coverage internal/console 97ee054fd00a99d3d97cb0b7a1d3292d03711063e1d709257b9544e5730cf368 |
| @coverage . 0e6e4542758248000de1882a01f13e3449c880b4ae23e8b2ff0d6dce076b614c |
| @legacy b02b534b6617f57381ba12e9d9a2bf1bd63f737dc68005de926ff18b7db93dd3 internal/console/apiboxroles.go Server.boxRoster legacy accepted-legacy accepted-legacy;rev3=J4;basis=root-input;census-ruling-region-ha-2a file:Read@484 9be9925de14dbbd7d86123bf9faf1535c7b116575cadcb8ba983a665af0513f4 |
| internal/console/apiboxroles.go Server.boxRoster legacy accepted-legacy accepted-legacy;rev3=J4;basis=root-input;census-ruling-region-ha-2a file:Read@484 9be9925de14dbbd7d86123bf9faf1535c7b116575cadcb8ba983a665af0513f4 |