amadan

carlos / platform Public

region HA 2a: create doors choose first and refuse when no box has room

H27: with region HA on, beginClaimCreate chooses the executor before it
registers anything (chooseExecutorFirst) and carries the choice to the
create (hostops.WithChosenExecutor, honoured by
ProductionAuthority.SelectExecutor). The instance create door chooses
before the environment binding (chooseCreateFirst) and carries the claim
observation it decided on; beginClaimCreate refuses with errChoiceStale
(hostclaim.ErrConflict) when its own read differs. No room is a 503
(RHA-C17) on instance create, domain attach's first route and standalone
sidecar create, paged at most once per region an hour (RHA-C18, C19).
An app-derived region is judged by the scoped chooser's eligible set, and
an explicit region with no room is refused before the armed-placement
gate (roomIn). Region HA off: nothing here runs.

Census: beginClaimCreate re-frozen and chooseCreateFirst added as
accepted-legacy (census ruling region-ha-2a); seal 57efed07; B8 digest
of handleAPIInstanceCreate re-pinned, no write site added or changed.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude 6065b5e32f4845fdecf1e14c916ab45242e706af parent c7484fd
14 files changed, +910 −16
  • approved_region_ha_copy_test.go +1 −1
  • internal/console/apiinstances.go +47 −11
  • internal/console/claim_create.go +6 −0
  • internal/console/claim_response.go +5 −0
  • internal/console/nocapacity.go +211 −0
  • internal/console/nocapacity_test.go +553 −0
  • internal/console/server.go +4 −0
  • internal/hostops/backfill.go +6 −0
  • internal/hostops/chosen.go +37 −0
  • internal/hostops/executor_test.go +28 −0
  • internal/provision/plan14_root_reads_test.go +1 −1
  • internal/provision/plan9_b8_census_test.go +2 −1
  • internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv +7 −2
  • region_ha_boundary_test.go +2 −0
diff --git a/approved_region_ha_copy_test.go b/approved_region_ha_copy_test.go
index 75bb34b2c..e5f00c146 100644
--- a/approved_region_ha_copy_test.go
+++ b/approved_region_ha_copy_test.go
@@ -38,7 +38,7 @@ var regionHACopyIDs = func() []string {
// regionHACopyBuilt is every id whose text the code or docs carry. The
// task that writes a row's text adds its id here in the same commit, once
// Paul has approved the row.
-var regionHACopyBuilt = []string{"RHA-C1", "RHA-C2", "RHA-C3", "RHA-C4", "RHA-C5", "RHA-C6", "RHA-C7", "RHA-C8", "RHA-C9", "RHA-C10", "RHA-C11", "RHA-C12", "RHA-C13", "RHA-C14", "RHA-C15", "RHA-C16", "RHA-C20", "RHA-C21", "RHA-C22", "RHA-C23", "RHA-C24", "RHA-C25"}
+var regionHACopyBuilt = []string{"RHA-C1", "RHA-C2", "RHA-C3", "RHA-C4", "RHA-C5", "RHA-C6", "RHA-C7", "RHA-C8", "RHA-C9", "RHA-C10", "RHA-C11", "RHA-C12", "RHA-C13", "RHA-C14", "RHA-C15", "RHA-C16", "RHA-C17", "RHA-C18", "RHA-C19", "RHA-C20", "RHA-C21", "RHA-C22", "RHA-C23", "RHA-C24", "RHA-C25"}
func mustParseRegionHARecord(t *testing.T) []copyRow {
t.Helper()
diff --git a/internal/console/apiinstances.go b/internal/console/apiinstances.go
index 00eee4776..1aed8a1d5 100644
--- a/internal/console/apiinstances.go
+++ b/internal/console/apiinstances.go
@@ -1259,6 +1259,25 @@ func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request)
if c, ok := findCandidate(cands, region); ok {
reason = c.unavailableReason()
}
+ // Region HA (spec 2 §7, H27): the region is judged by the eligible
+ // set the create door's chooser will choose from, never by every
+ // sibling's own answer, and an empty set is the door's no-room
+ // refusal, alert included.
+ if x := s.placementChooser(""); x != nil {
+ switch boxes, err := x.PlaceableBoxes(ctx, s.cfg.claimPlacementRegion(region)); {
+ case errors.Is(err, hostops.ErrContract):
+ reason = "absent"
+ case err != nil:
+ log.Printf("console: api instances create %s/%s: region %s placeable boxes: %v", acct.Sqid, app, region, err)
+ jsonError(w, http.StatusServiceUnavailable, msgAppRegionUnconfirmed(s.homeRegionOut(region)))
+ return
+ case len(boxes) == 0:
+ writeClaimResponseError(w, s.noRoom(ctx, region))
+ return
+ default:
+ reason = ""
+ }
+ }
// Each list names only what an explicit request for this same
// instance would pass (codex plan-R4 2): instanceChoices.
if reason != "" {
@@ -1331,6 +1350,12 @@ func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request)
// provision.Record's own (DerivesPaths), asked of a probe record rather
// than restated here, so it cannot drift from what reconcile will mint.
if placed == "" && region != "" {
+ // Region HA (spec 2 §7, H27): a region with no box that has room is
+ // the no-room refusal, before this gate answers it as a 400.
+ if err := s.roomIn(ctx, region, m.Pool); err != nil {
+ writeClaimResponseError(w, err)
+ return
+ }
probe := provision.Record{Kind: strings.TrimSpace(req.Kind), Addrs: req.Addrs, Role: probeRec.Role}
caps, published := placements(ctx, s.cfg.Releases)
// The replica placement rides the same gate as the record's own
@@ -1568,25 +1593,36 @@ func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request)
cancel()
ctx, cancel = context.WithTimeout(r.Context(), instanceStoreTimeout)
defer cancel()
+ buildClaim := func(now time.Time, acquisition string) (hostops.Record, error) {
+ rec.CreatedAt = time.Time{}
+ rec.Acquisition = acquisition
+ region := s.cfg.claimPlacementRegion(rec.Region)
+ prepared, err := hostops.PrepareInstance(rec, now, region)
+ if err == nil {
+ rec.CreatedAt = now
+ }
+ return prepared, err
+ }
+ // Region HA (spec 2 §7, H27): the box is chosen before the door's first
+ // write, the environment binding below, so a create refused for want of
+ // room changes nothing. beginClaimCreate keeps this choice.
+ createContext := r.Context()
+ if s.cfg.HostClaim {
+ var chooseErr error
+ if createContext, chooseErr = s.chooseCreateFirst(createContext, host, buildClaim); chooseErr != nil {
+ writeClaimResponseError(w, chooseErr)
+ return
+ }
+ }
// An environment named without a store walk of its own takes the store
// from a seed that made it (final review F3, seedenvbind.go).
if !m.PrivateStorage && !s.bindEnvironmentCreate(ctx, w, rec) {
return
}
var claimRequest *claimCreateRequest
- createContext := r.Context()
if s.cfg.HostClaim {
var createErr error
- claimRequest, createErr = s.beginClaimCreate(r.Context(), acct, app, host, principal, func(now time.Time, acquisition string) (hostops.Record, error) {
- rec.CreatedAt = time.Time{}
- rec.Acquisition = acquisition
- region := s.cfg.claimPlacementRegion(rec.Region)
- prepared, err := hostops.PrepareInstance(rec, now, region)
- if err == nil {
- rec.CreatedAt = now
- }
- return prepared, err
- })
+ claimRequest, createErr = s.beginClaimCreate(createContext, acct, app, host, principal, buildClaim)
if createErr != nil {
writeClaimResponseError(w, createErr)
return
diff --git a/internal/console/claim_create.go b/internal/console/claim_create.go
index 01da7e451..03359f772 100644
--- a/internal/console/claim_create.go
+++ b/internal/console/claim_create.go
@@ -39,6 +39,12 @@ func (s *Server) beginClaimCreate(ctx context.Context, acct *Account, app, host,
if err != nil {
return nil, err
}
+ // Region HA (spec 2 §7, H27): choose before registering anything. A
+ // door's earlier choice is kept only if this read is the claim it was
+ // decided on (census ruling region-ha-2a, item 5).
+ if ctx, err = s.chooseExecutorFirst(ctx, host, before, build); err != nil {
+ return nil, err
+ }
appID, err := s.store.ClaimAppID(acct.ID, app)
if err != nil {
return nil, err
diff --git a/internal/console/claim_response.go b/internal/console/claim_response.go
index a5f507aa9..7f61b7669 100644
--- a/internal/console/claim_response.go
+++ b/internal/console/claim_response.go
@@ -15,6 +15,11 @@ const claimResponseUnavailable = "This operation is temporarily unavailable. Try
func claimResponseRefusal(err error) *doorRefusal {
log.Printf("console: claim request refused: %v", err)
+ // No box has room (region HA spec 2 §7, H27): every create door's 503.
+ var full *noCapacityError
+ if errors.As(err, &full) {
+ return &doorRefusal{http.StatusServiceUnavailable, noCapacityRefusal(full.Region)}
+ }
if errors.Is(err, hostops.ErrPending) || errors.Is(err, hostclaim.ErrConflict) || errors.Is(err, hostclaim.ErrWrongState) || errors.Is(err, release.ErrCASConflict) {
return &doorRefusal{http.StatusConflict, claimRetireBusyMsg}
}
diff --git a/internal/console/nocapacity.go b/internal/console/nocapacity.go
new file mode 100644
index 000000000..0c0071150
--- /dev/null
+++ b/internal/console/nocapacity.go
@@ -0,0 +1,211 @@
+package console
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log"
+ "sync"
+ "time"
+
+ "github.com/carlosframework/platform/internal/alerting"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/hostops"
+)
+
+// No box has room (region HA spec 2 §7, ruling H27). Every door that
+// chooses a box (instance create, domain attach's first route, standalone
+// sidecar create, and the move door's destination) refuses with a 503 when
+// the placement policy finds no eligible box, and the operator hears of it
+// at most once per region an hour per console process: a restart may send
+// one extra alert, which H27 accepts rather than persist the limit. A
+// create door chooses before it registers anything (chooseExecutorFirst),
+// so a refusal leaves nothing registered, nothing written to the claim and
+// nothing for recovery to settle.
+
+// noCapacityAlertEvery is how often one region's alert may repeat.
+const noCapacityAlertEvery = time.Hour
+
+func noCapacityRefusal(region string) string {
+ return fmt.Sprintf("No box in %s has room right now. Try again in a few minutes.", region)
+}
+
+func noCapacityAlert(region string) (subject, body string) {
+ return fmt.Sprintf("No box in %s has room", region),
+ fmt.Sprintf("A request in %s was refused because no box there can take it. Every box is full, stale or excluded by its role. Add a box, or free memory or disk on one, and check the eligible-capacity alarm for %s.", region, region)
+}
+
+// noCapacityError is a request refused because no box in Region (its
+// display name) can take it. It is hostops.ErrNoEligibleBox, and
+// claimResponseRefusal answers it with noCapacityRefusal.
+type noCapacityError struct{ Region string }
+
+func (e *noCapacityError) Error() string {
+ return "no eligible box in " + e.Region + ": " + hostops.ErrNoEligibleBox.Error()
+}
+
+func (e *noCapacityError) Is(target error) bool { return target == hostops.ErrNoEligibleBox }
+
+// pageNoCapacity sends the alert; tests replace it.
+var pageNoCapacity = alerting.Page
+
+// noCapacityAlerts remembers when each region's alert was last sent, in
+// this console process. The zero value is ready.
+type noCapacityAlerts struct {
+ mu sync.Mutex
+ last map[string]time.Time
+}
+
+// due reports whether region's alert should be sent at now, and if so
+// records it as sent.
+func (a *noCapacityAlerts) due(region string, now time.Time) bool {
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ if at, ok := a.last[region]; ok && now.Sub(at) < noCapacityAlertEvery {
+ return false
+ }
+ if a.last == nil {
+ a.last = map[string]time.Time{}
+ }
+ a.last[region] = now
+ return true
+}
+
+// noRoom is the refusal for region (record vocabulary, "" for the home):
+// it logs, sends the operator alert when one is due, and returns the
+// error the door answers with.
+func (s *Server) noRoom(ctx context.Context, region string) error {
+ name := s.homeRegionOut(region)
+ log.Printf("console: refused: no eligible box in %s", name)
+ if s.noCapacity.due(name, time.Now()) {
+ subject, body := noCapacityAlert(name)
+ go func() {
+ ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
+ defer cancel()
+ if err := pageNoCapacity(ctx, alerting.LevelWarning, subject, body); err != nil {
+ log.Printf("console: no-capacity alert for %s: %v", name, err)
+ }
+ }()
+ }
+ return &noCapacityError{Region: name}
+}
+
+// placementChooser is the create door's chooser scoped as
+// ProductionAuthority.SelectExecutor scopes it, for pool ("" the shared
+// pool), or nil with region HA off or no claim runtime.
+func (s *Server) placementChooser(pool string) *hostops.ExecutorChooser {
+ a := s.cfg.ClaimAuthority
+ if !s.cfg.RegionHA || a == nil || a.Executors == nil {
+ return nil
+ }
+ x := *a.Executors
+ x.Namespace, x.Pool = a.Namespace, pool
+ return &x
+}
+
+// roomIn is H27's question about region (record vocabulary) for pool, asked
+// with region HA on before a door's own placement checks (Astra 2a round 2
+// finding 1): a region the roster places boxes in, whose placeable set is
+// empty (no edges file, no public address, stale, full or excluded by
+// role), is the no-room refusal, never a 400. It only refuses: it chooses
+// nothing and changes nothing the door goes on to do. It answers nil with
+// region HA off, for a region the roster does not place (the door's own
+// checks answer that), and when a box has room.
+func (s *Server) roomIn(ctx context.Context, region, pool string) error {
+ x := s.placementChooser(pool)
+ if x == nil {
+ return nil
+ }
+ boxes, err := x.PlaceableBoxes(ctx, s.cfg.claimPlacementRegion(region))
+ switch {
+ case errors.Is(err, hostops.ErrContract):
+ return nil
+ case err != nil:
+ return err
+ case len(boxes) == 0:
+ return s.noRoom(ctx, region)
+ }
+ return nil
+}
+
+// errChoiceStale is an early choice whose claim observation is no longer
+// the claim: a write landed between the choice and beginClaimCreate's own
+// read. It wraps hostclaim.ErrConflict, so errors.Is(errChoiceStale,
+// hostclaim.ErrConflict) holds on the sentinel itself and the door answers
+// main's retryable busy refusal; nothing was registered or written.
+var errChoiceStale = fmt.Errorf("%w: the host's claim changed after its executor was chosen", hostclaim.ErrConflict)
+
+// chosenOnKey carries the claim observation an early choice was decided
+// on, beside the choice itself (hostops.WithChosenExecutor).
+type chosenOnKey struct{}
+
+type chosenOn struct {
+ host, etag string
+ absent bool
+}
+
+// chooseExecutorFirst is beginClaimCreate's first step with region HA on,
+// given beginClaimCreate's own read of the claim (before). It chooses the
+// executor for the record build would make, before anything is
+// registered, and carries the choice to the create
+// (hostops.WithChosenExecutor). It takes no lease and writes nothing: the
+// choice is SelectExecutor's read of the roster and the edges files. No
+// eligible box is noRoom's refusal. A choice ctx already carries for host
+// (chooseCreateFirst's) is kept only when before is the claim observation
+// it was decided on (census ruling region-ha-2a, item 5): otherwise the
+// create is refused with errChoiceStale, and the old choice is never used
+// and nothing is chosen again. With region HA off it returns ctx
+// unchanged, and the create is main's.
+func (s *Server) chooseExecutorFirst(ctx context.Context, host string, before hostclaim.Observed, build func(time.Time, string) (hostops.Record, error)) (context.Context, error) {
+ a := s.cfg.ClaimAuthority
+ if !s.cfg.RegionHA || a == nil || a.Executors == nil {
+ return ctx, nil
+ }
+ if _, ok := hostops.ChosenExecutor(ctx, host); ok {
+ on, ok := ctx.Value(chosenOnKey{}).(chosenOn)
+ if !ok || on.host != host || on.etag != before.ETag || on.absent != before.Absent {
+ return ctx, fmt.Errorf("%w: %s", errChoiceStale, host)
+ }
+ return ctx, nil
+ }
+ probe, err := build(time.Now().UTC(), "")
+ if err != nil {
+ return ctx, err
+ }
+ sel, chosen, err := a.SelectExecutor(ctx, before, probe)
+ if errors.Is(err, hostops.ErrNoEligibleBox) {
+ return ctx, s.noRoom(ctx, probe.Holder.RecordRegion)
+ }
+ if err != nil || !chosen {
+ return ctx, err
+ }
+ return hostops.WithChosenExecutor(ctx, host, sel), nil
+}
+
+// chooseCreateFirst is the instance create door's choice, made before the
+// door's first write (Astra 2a round 2 finding 2): the environment binding
+// (bindEnvironmentCreate) can move a seed's store to the new instance, so
+// a create refused for want of room must be refused before it. It reads
+// host's claim and chooses as chooseExecutorFirst does, and returns the
+// choice together with the claim observation it decided on (its ETag, or
+// its absence). beginClaimCreate keeps the choice only if its own read is
+// that same observation. With region HA off it returns ctx unchanged and
+// reads nothing.
+func (s *Server) chooseCreateFirst(ctx context.Context, host string, build func(time.Time, string) (hostops.Record, error)) (context.Context, error) {
+ a := s.cfg.ClaimAuthority
+ if !s.cfg.RegionHA || a == nil || a.Executors == nil {
+ return ctx, nil
+ }
+ before, err := hostclaim.Read(ctx, s.cfg.Releases, host)
+ if err != nil {
+ return ctx, err
+ }
+ chosen, err := s.chooseExecutorFirst(ctx, host, before, build)
+ if err != nil {
+ return ctx, err
+ }
+ if _, ok := hostops.ChosenExecutor(chosen, host); !ok {
+ return chosen, nil
+ }
+ return context.WithValue(chosen, chosenOnKey{}, chosenOn{host: host, etag: before.ETag, absent: before.Absent}), nil
+}
diff --git a/internal/console/nocapacity_test.go b/internal/console/nocapacity_test.go
new file mode 100644
index 000000000..ad3a49385
--- /dev/null
+++ b/internal/console/nocapacity_test.go
@@ -0,0 +1,553 @@
+package console
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "net/http"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/boxchoice"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/hostops"
+ "github.com/carlosframework/platform/internal/objstore"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/status"
+)
+
+func TestNoCapacityAlertsRepeatAfterAnHour(t *testing.T) {
+ var a noCapacityAlerts
+ t0 := time.Date(2026, 10, 16, 12, 0, 0, 0, time.UTC)
+ if !a.due("uk", t0) || a.due("uk", t0.Add(59*time.Minute)) || !a.due("ie", t0) || !a.due("uk", t0.Add(time.Hour)) {
+ t.Fatal("once per region an hour")
+ }
+}
+
+func TestClaimRefusalAnswersNoRoom(t *testing.T) {
+ ref := claimResponseRefusal(fmt.Errorf("begin: %w", &noCapacityError{Region: "uk"}))
+ if ref.Status != http.StatusServiceUnavailable || ref.Msg != noCapacityRefusal("uk") {
+ t.Fatalf("refusal %+v", ref)
+ }
+ var err error = &noCapacityError{Region: "uk"}
+ if !errors.Is(err, hostops.ErrNoEligibleBox) {
+ t.Fatal("a no-room refusal is not ErrNoEligibleBox")
+ }
+}
+
+// pagesTo replaces the alert sender for the test and returns what it sent.
+func pagesTo(t *testing.T) chan string {
+ t.Helper()
+ pages := make(chan string, 16)
+ old := pageNoCapacity
+ pageNoCapacity = func(_ context.Context, level, subject, _ string) error {
+ pages <- level + " " + subject
+ return nil
+ }
+ t.Cleanup(func() { pageNoCapacity = old })
+ return pages
+}
+
+func expectPages(t *testing.T, pages chan string, want ...string) {
+ t.Helper()
+ for _, w := range want {
+ select {
+ case got := <-pages:
+ if got != w {
+ t.Fatalf("page %q, want %q", got, w)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatalf("no page %q", w)
+ }
+ }
+ select {
+ case got := <-pages:
+ t.Fatalf("an extra page: %q", got)
+ case <-time.After(100 * time.Millisecond):
+ }
+}
+
+func noRoomSubject(region string) string {
+ subject, _ := noCapacityAlert(region)
+ return "warning " + subject
+}
+
+// haClaimCase is a host-claim console with region HA on: its create doors
+// choose through the region HA chooser over the edges files the test
+// publishes (publishHABox). home names the console's home region ("" for
+// none): a first route is placed in the home.
+func haClaimCase(t *testing.T, home string) *claimCase {
+ t.Helper()
+ cx := newClaimCaseWith(t, func(c *Config) { c.HomeRegion = home })
+ cx.srv.cfg.RegionHA = true
+ a := cx.srv.cfg.ClaimAuthority
+ a.Executors = cx.srv.executorChooser(a.Directory)
+ return cx
+}
+
+// publishHABox publishes (or republishes) a roster box's edges file and
+// status snapshot: placeable in ie unless mut changes it, and stale when
+// stale is set.
+func publishHABox(t *testing.T, cx *claimCase, box string, stale bool, mut func(map[string]any)) {
+ t.Helper()
+ ctx := context.Background()
+ m := map[string]any{
+ "v": 1, "box": box, "box_id": box, "eip": "203.0.113.7", "region": "eu-west-1", "placement": "ie",
+ "hibernation_configured": true, "serves_shared_pool": true, "accepts_inbound": true,
+ "secrets_key_fp": cx.srv.expectedSecretsFP(), "capabilities": []string{"region-ha-1"},
+ "headroom": map[string]any{"mem_free_mib": 4000, "mem_total_mib": 8000, "disk_free_mib": 50000, "disk_total_mib": 100000, "awake": 0},
+ "generation": time.Now().UTC().Format(time.RFC3339), "hosts": map[string]any{},
+ }
+ if mut != nil {
+ mut(m)
+ }
+ body, err := json.Marshal(m)
+ if err != nil {
+ t.Fatal(err)
+ }
+ key := "control/edges/" + box + ".json"
+ _, tag, err := cx.rel.Get(ctx, key)
+ if err != nil {
+ tag = ""
+ }
+ if _, err := cx.rel.Put(ctx, key, body, tag); err != nil {
+ t.Fatal(err)
+ }
+ at := time.Now()
+ if stale {
+ at = at.Add(-time.Hour)
+ }
+ if err := status.WriteBox(ctx, cx.rel, status.Box{Label: box, GeneratedAt: at}); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// registrations counts account/app's registered operations.
+func registrations(t *testing.T, cx *claimCase, account, app string) int {
+ t.Helper()
+ a, err := hostops.ReadCanonicalAdmission(context.Background(), cx.rel, account, app)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return len(a.Value.Registrations)
+}
+
+// refusedBeforeRegistering checks what H27 promises of a refused create:
+// nothing registered, no claim, and nothing recovery writes afterwards.
+func refusedBeforeRegistering(t *testing.T, cx *claimCase, host string, registered int) {
+ t.Helper()
+ ctx := context.Background()
+ if n := registrations(t, cx, "bab", "titogo"); n != registered {
+ t.Fatalf("registrations %d after the refusal, want %d: the door registered before choosing", n, registered)
+ }
+ if obs, err := hostclaim.Read(ctx, cx.rel, host); err != nil || !obs.Absent {
+ t.Fatalf("claim after the refusal: %+v %v", obs, err)
+ }
+ if err := cx.srv.claimRecoveryPass(ctx); err != nil {
+ t.Fatalf("recovery: %v", err)
+ }
+ if obs, err := hostclaim.Read(ctx, cx.rel, host); err != nil || !obs.Absent {
+ t.Fatalf("claim after recovery: %+v %v", obs, err)
+ }
+ if n := registrations(t, cx, "bab", "titogo"); n != registered {
+ t.Fatalf("registrations %d after recovery, want %d", n, registered)
+ }
+}
+
+// wroteNothing checks that nothing reached the bucket or a backend since
+// mark: no lease, no registration, no record and no store write.
+func wroteNothing(t *testing.T, cx *claimCase, mark int) {
+ t.Helper()
+ if events := cx.rec.since(mark); len(events) != 0 {
+ t.Fatalf("a refused request wrote %v", events)
+ }
+}
+
+// Region HA spec 2 §7 (H27): every create door that chooses a box refuses
+// with a 503 when none is eligible, before it registers anything, and the
+// operator hears of it once per region an hour. Every box here is stale.
+func TestCreateDoorsRefuseWithNoRoomBeforeRegistering(t *testing.T) {
+ t.Run("instance create", func(t *testing.T) {
+ pages := pagesTo(t)
+ cx := haClaimCase(t, "")
+ cx.instanceWorld(t)
+ publishHABox(t, cx, "box-ie-tito", true, nil)
+ before := registrations(t, cx, "bab", "titogo")
+ mark := cx.rec.mark()
+ for range 3 { // a retry loop: refused every time, paged once
+ r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", `{"host":"acme.go.tito.io","region":"ie"}`)
+ if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, noCapacityRefusal("ie")) {
+ t.Fatalf("create = %d %s", r.Status, r.Body)
+ }
+ }
+ wroteNothing(t, cx, mark)
+ refusedBeforeRegistering(t, cx, "acme.go.tito.io", before)
+ expectPages(t, pages, noRoomSubject("ie"))
+ })
+ t.Run("standalone sidecar create", func(t *testing.T) {
+ pages := pagesTo(t)
+ cx := haClaimCase(t, "")
+ cx.sidecarWorld(t)
+ before := registrations(t, cx, "bab", "titogo")
+ mark := cx.rec.mark()
+ r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/sidecars", `{"name":"bot","region":"ie"}`)
+ if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, noCapacityRefusal("ie")) {
+ t.Fatalf("sidecar create = %d %s", r.Status, r.Body)
+ }
+ wroteNothing(t, cx, mark)
+ refusedBeforeRegistering(t, cx, provision.SidecarHost("bot", "titogo", "bab"), before)
+ expectPages(t, pages, noRoomSubject("ie"))
+ })
+ t.Run("domain attach, a first route", func(t *testing.T) {
+ pages := pagesTo(t)
+ cx := haClaimCase(t, "ie")
+ shipVersion(t, cx.consoleFixture, "titogo", "v1")
+ seedChannel(t, cx.consoleFixture, "titogo", "edge", "v1")
+ before := registrations(t, cx, "bab", "titogo")
+ mark := cx.rec.mark()
+ r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/domains", `{"host":"first.titogo-claim.example","kind":"static"}`)
+ if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, noCapacityRefusal("ie")) {
+ t.Fatalf("attach = %d %s", r.Status, r.Body)
+ }
+ wroteNothing(t, cx, mark)
+ refusedBeforeRegistering(t, cx, "first.titogo-claim.example", before)
+ expectPages(t, pages, noRoomSubject("ie"))
+ })
+ t.Run("browser domain attach, a first route", func(t *testing.T) {
+ pages := pagesTo(t)
+ cx := haClaimCase(t, "ie")
+ shipVersion(t, cx.consoleFixture, "titogo", "v1")
+ seedChannel(t, cx.consoleFixture, "titogo", "edge", "v1")
+ before := registrations(t, cx, "bab", "titogo")
+ mark := cx.rec.mark()
+ r := cx.formReply(t, "/a/bab/apps/titogo/domains/attach", "host", "first.titogo-claim.example", "kind", "static")
+ if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, noCapacityRefusal("ie")) {
+ t.Fatalf("browser attach = %d %s", r.Status, r.Body)
+ }
+ wroteNothing(t, cx, mark)
+ refusedBeforeRegistering(t, cx, "first.titogo-claim.example", before)
+ expectPages(t, pages, noRoomSubject("ie"))
+ })
+}
+
+// appDerivedWorld is titogo with no pool and no region allow-list, whose
+// app region is ie: a create that names no region takes the app's.
+func appDerivedWorld(t *testing.T, cx *claimCase) {
+ t.Helper()
+ m := titoManifest()
+ m.Regions = nil
+ m.Pool = ""
+ enableInstances(t, cx.consoleFixture, "titogo", m)
+ cx.srv.appRegionOf = func(int64, string) (string, string, error) { return "ie", "app", nil }
+}
+
+// Region HA spec 2 §7 (H27, Astra 2a round 1 finding 3): with region HA
+// on, an app-derived region is judged by the chooser's eligible set. Every
+// box stale is the no-room refusal, alert included, never the region
+// refusal's 400.
+func TestAppRegionWithEveryBoxStaleIsNoRoom(t *testing.T) {
+ pages := pagesTo(t)
+ cx := haClaimCase(t, "")
+ appDerivedWorld(t, cx)
+ publishHABox(t, cx, "box-ie-default", true, nil)
+ before := registrations(t, cx, "bab", "titogo")
+ r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", `{"host":"acme.go.tito.io"}`)
+ if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, noCapacityRefusal("ie")) {
+ t.Fatalf("create = %d %s", r.Status, r.Body)
+ }
+ refusedBeforeRegistering(t, cx, "acme.go.tito.io", before)
+ expectPages(t, pages, noRoomSubject("ie"))
+}
+
+// ... and a misconfigured sibling (a box in ie holding another secrets
+// key, outside the roster) no longer refuses a region whose eligible set
+// holds another box: the create goes ahead on that box. With region HA off
+// the sibling still refuses, as on main.
+func TestAppRegionWithAMisconfiguredSiblingStillCreates(t *testing.T) {
+ cx := haClaimCase(t, "")
+ appDerivedWorld(t, cx)
+ publishHABox(t, cx, "box-ie-default", false, nil)
+ publishHABox(t, cx, "box-ie-stray", false, func(m map[string]any) { m["secrets_key_fp"] = "sha256:another-key" })
+ off := cx.srv.placementChooser("")
+ cx.srv.cfg.RegionHA = false
+ r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", `{"host":"other.go.tito.io"}`)
+ if r.Status != http.StatusBadRequest {
+ t.Fatalf("region HA off: create = %d %s, want main's region refusal", r.Status, r.Body)
+ }
+ cx.srv.cfg.RegionHA = true
+ if off == nil {
+ t.Fatal("no chooser with region HA on")
+ }
+ r = cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", `{"host":"acme.go.tito.io"}`)
+ if r.Status != http.StatusCreated && r.Status != http.StatusAccepted {
+ t.Fatalf("create = %d %s", r.Status, r.Body)
+ }
+ obs, err := hostclaim.Read(context.Background(), cx.rel, "acme.go.tito.io")
+ if err != nil || obs.Absent {
+ t.Fatalf("claim: %+v %v", obs, err)
+ }
+ if exec, chosen := obs.Claim.Op.Exec, obs.Claim.Op.Intent["selected_executor"]; exec != "box-ie-default" && chosen != "box-ie-default" {
+ t.Fatalf("executor %q / %q, want box-ie-default", exec, chosen)
+ }
+}
+
+// putSeedStore gives titogo's canary environment a store a seed made, and
+// returns the store record's bytes.
+func putSeedStore(t *testing.T, cx *claimCase) (key string, body []byte) {
+ t.Helper()
+ ctx := context.Background()
+ at := time.Now().UTC()
+ if _, err := objstore.PutIn(ctx, cx.rel, "bab", "titogo", "canary", &objstore.Record{
+ Declared: &objstore.Declared{EnvPrefix: "S3", At: at, By: "bob@keymail.dev", Seed: newSeedID(t)},
+ Grant: &objstore.GrantBlock{Backend: "mem", Bucket: "carlos-bab-titogo-canary", Source: objstore.GrantSourceAccountApproval, At: at, By: "operator@keymail.dev"},
+ }, ""); err != nil {
+ t.Fatal(err)
+ }
+ key = objstore.KeyIn("bab", "titogo", "canary")
+ body, _, err := cx.rel.Get(ctx, key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return key, body
+}
+
+// Astra 2a round 2 finding 2: the instance create door chooses before its
+// first write. A create naming an environment whose store a seed made would
+// take that store (bindEnvironmentCreate). Refused for want of room, it
+// leaves the store's bytes as they were and writes nothing at all: when
+// every box is stale (the region check refuses), and when the region's last
+// placeable box goes between the region check and the choice (the early
+// choice refuses, before the binding). With room, the same create takes the
+// store, so the binding is on the path.
+func TestRefusedCreateLeavesTheEnvironmentsStoreAlone(t *testing.T) {
+ const body = `{"host":"acme.go.tito.io","region":"ie","environment":"canary"}`
+ refused := func(t *testing.T, cx *claimCase, pages chan string) {
+ t.Helper()
+ key, stored := putSeedStore(t, cx)
+ before := registrations(t, cx, "bab", "titogo")
+ mark := cx.rec.mark()
+ r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", body)
+ if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, noCapacityRefusal("ie")) {
+ t.Fatalf("create = %d %s", r.Status, r.Body)
+ }
+ wroteNothing(t, cx, mark)
+ if now, _, err := cx.rel.Get(context.Background(), key); err != nil || string(now) != string(stored) {
+ t.Fatalf("the store after the refusal:\n%s\nwant\n%s (%v)", now, stored, err)
+ }
+ refusedBeforeRegistering(t, cx, "acme.go.tito.io", before)
+ expectPages(t, pages, noRoomSubject("ie"))
+ }
+ t.Run("every box stale", func(t *testing.T) {
+ pages := pagesTo(t)
+ cx := haClaimCase(t, "")
+ cx.instanceWorld(t)
+ publishHABox(t, cx, "box-ie-tito", true, nil)
+ refused(t, cx, pages)
+
+ publishHABox(t, cx, "box-ie-tito", false, nil)
+ r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", body)
+ if r.Status != http.StatusCreated && r.Status != http.StatusAccepted {
+ t.Fatalf("create with room = %d %s", r.Status, r.Body)
+ }
+ store, _, err := objstore.GetIn(context.Background(), cx.rel, "bab", "titogo", "canary")
+ if err != nil || store == nil || store.Declared == nil || store.Declared.Seed != "" {
+ t.Fatalf("the store after a create with room: %+v %v, want the seed's mark cleared", store, err)
+ }
+ })
+ t.Run("the last box goes before the choice", func(t *testing.T) {
+ pages := pagesTo(t)
+ cx := haClaimCase(t, "")
+ cx.instanceWorld(t)
+ publishHABox(t, cx, "box-ie-tito", false, nil)
+ x := cx.srv.cfg.ClaimAuthority.Executors
+ edges, reads := x.Edges, 0
+ x.Edges = func(ctx context.Context) (boxchoice.Snapshot, error) {
+ if reads++; reads > 1 {
+ return boxchoice.Snapshot{Complete: true}, nil
+ }
+ return edges(ctx)
+ }
+ refused(t, cx, pages)
+ })
+}
+
+// Astra 2a round 2 finding 1, at the create door: with region HA on, a
+// region whose boxes publish no public address (or no edges file) is a
+// region with no room, not the armed-placement gate's 400. With region HA
+// off the gate answers, as on main.
+func TestCreateRegionWithNoReachableBoxIsNoRoom(t *testing.T) {
+ const body = `{"host":"acme.go.tito.io","region":"ie"}`
+ pages := pagesTo(t)
+ cx := haClaimCase(t, "")
+ cx.instanceWorld(t)
+ publishHABox(t, cx, "box-ie-tito", false, func(m map[string]any) { m["eip"] = "" })
+ cx.srv.cfg.RegionHA = false
+ if r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", body); r.Status != http.StatusBadRequest {
+ t.Fatalf("region HA off: create = %d %s, want main's 400", r.Status, r.Body)
+ }
+ cx.srv.cfg.RegionHA = true
+ before := registrations(t, cx, "bab", "titogo")
+ mark := cx.rec.mark()
+ r := cx.apiReply(t, http.MethodPost, "/api/cli/apps/bab/titogo/instances", body)
+ if r.Status != http.StatusServiceUnavailable || !strings.Contains(r.Body, noCapacityRefusal("ie")) {
+ t.Fatalf("create = %d %s", r.Status, r.Body)
+ }
+ wroteNothing(t, cx, mark)
+ refusedBeforeRegistering(t, cx, "acme.go.tito.io", before)
+ expectPages(t, pages, noRoomSubject("ie"))
+}
+
+// Census ruling region-ha-2a rev 2, item 3: the early choice takes no lease
+// and writes nothing, and its executor is the one the authority chooses on
+// its own (the create's choice before the early choice existed). A choice
+// the context already carries is kept: nothing is chosen again.
+func TestChooseFirstWritesNothingAndChoosesOnce(t *testing.T) {
+ cx := haClaimCase(t, "")
+ appDerivedWorld(t, cx)
+ publishHABox(t, cx, "box-ie-default", false, nil)
+ ctx := context.Background()
+ host := "acme.go.tito.io"
+ build := func(now time.Time, acquisition string) (hostops.Record, error) {
+ rec := provision.Record{Schema: 1, Host: host, App: "titogo", Account: "bab", Channel: "stable", Kind: "static", CreatedBy: "bob@keymail.dev", Acquisition: acquisition}
+ return hostops.PrepareInstance(rec, now, cx.srv.cfg.claimPlacementRegion("ie"))
+ }
+ probe, err := build(time.Now().UTC(), "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ before, err := hostclaim.Read(ctx, cx.rel, host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ want, chosen, err := cx.srv.cfg.ClaimAuthority.SelectExecutor(ctx, before, probe)
+ if err != nil || !chosen || want.Box != "box-ie-default" {
+ t.Fatalf("the authority's own choice: %+v %v %v", want, chosen, err)
+ }
+
+ mark := cx.rec.mark()
+ got, err := cx.srv.chooseCreateFirst(ctx, host, build)
+ if err != nil {
+ t.Fatal(err)
+ }
+ wroteNothing(t, cx, mark)
+ if sel, ok := hostops.ChosenExecutor(got, host); !ok || sel != want {
+ t.Fatalf("early choice %+v (%v), want the authority's %+v", sel, ok, want)
+ }
+
+ again, err := cx.srv.chooseExecutorFirst(got, host, before, func(time.Time, string) (hostops.Record, error) {
+ t.Fatal("chose again")
+ return hostops.Record{}, nil
+ })
+ if err != nil || again != got {
+ t.Fatalf("a carried choice was not kept: %v", err)
+ }
+ if sel, chosen, err := cx.srv.cfg.ClaimAuthority.SelectExecutor(got, before, probe); err != nil || !chosen || sel != want {
+ t.Fatalf("the create's selection %+v %v %v, want the door's %+v", sel, chosen, err, want)
+ }
+}
+
+// Census ruling region-ha-2a item 5 (amadan #236): the early choice
+// carries the claim observation it was decided on, and beginClaimCreate
+// keeps it only if its own read of the claim is that observation. A claim
+// write landing between the choice and begin refuses the create with
+// main's retryable busy answer: the old choice is never used, nothing is
+// chosen again, and nothing is registered or written.
+func TestAStaleEarlyChoiceIsNeverUsed(t *testing.T) {
+ cx := haClaimCase(t, "")
+ cx.instanceWorld(t)
+ publishHABox(t, cx, "box-ie-tito", false, nil)
+ ctx := context.Background()
+ const host = "acme.go.tito.io"
+ acct, err := cx.st.AccountBySqid("bab")
+ if err != nil || acct == nil {
+ t.Fatalf("account: %+v %v", acct, err)
+ }
+ build := func(now time.Time, acquisition string) (hostops.Record, error) {
+ rec := provision.Record{Schema: 1, Host: host, App: "titogo", Account: "bab", Channel: "stable", Kind: "static", Pool: "tito", Region: "ie",
+ CreatedBy: "bob@keymail.dev", Acquisition: acquisition}
+ return hostops.PrepareInstance(rec, now, cx.srv.cfg.claimPlacementRegion("ie"))
+ }
+ chosen, err := cx.srv.chooseCreateFirst(ctx, host, build)
+ if _, ok := hostops.ChosenExecutor(chosen, host); err != nil || !ok {
+ t.Fatalf("early choice: %v (chosen %v)", err, ok)
+ }
+
+ // A claim write lands between the choice and begin.
+ created := time.Now().UTC().Truncate(time.Second)
+ inc, err := hostclaim.Incarnation("bab", "titogo", host, created)
+ if err != nil {
+ t.Fatal(err)
+ }
+ writeServingClaim(t, cx.rel, host, inc, created, "box-ie-tito", "ie")
+ written, _, err := cx.rel.Get(ctx, "control/hosts/"+host+".json")
+ if err != nil {
+ t.Fatal(err)
+ }
+ before := registrations(t, cx, "bab", "titogo")
+ mark := cx.rec.mark()
+ // The build runs only to choose again or, past RegisterStart, to write
+ // the record: either is the old choice's create going on.
+ req, err := cx.srv.beginClaimCreate(chosen, acct, "titogo", host, "bob@keymail.dev", func(time.Time, string) (hostops.Record, error) {
+ t.Fatal("begin went on after the claim changed")
+ return hostops.Record{}, nil
+ })
+ if req != nil {
+ req.Close()
+ }
+ if !errors.Is(err, errChoiceStale) || !errors.Is(err, hostclaim.ErrConflict) {
+ t.Fatalf("begin on a changed claim = %v, want the stale-choice refusal", err)
+ }
+ if ref := claimResponseRefusal(err); ref.Status != http.StatusConflict || ref.Msg != claimRetireBusyMsg {
+ t.Fatalf("refusal %+v, want main's retryable busy answer", ref)
+ }
+ wroteNothing(t, cx, mark)
+ if n := registrations(t, cx, "bab", "titogo"); n != before {
+ t.Fatalf("registrations %d, want %d", n, before)
+ }
+ if now, _, err := cx.rel.Get(ctx, "control/hosts/"+host+".json"); err != nil || string(now) != string(written) {
+ t.Fatalf("the claim changed under the refusal: %v", err)
+ }
+}
+
+// Region HA 2a, Task 6's carry (controller ruling): PlaceableBoxes and
+// Capacity choose nothing without a namespace, so every caller uses
+// placementChooser, scoped as ProductionAuthority.SelectExecutor scopes the
+// create. With a placeable box in each pool, the scoped chooser counts it,
+// in the shared pool and in a named one; the unscoped chooser the authority
+// holds counts nothing, which is why no door asks it.
+func TestPlacementChooserIsScoped(t *testing.T) {
+ ctx := context.Background()
+ cx := haClaimCase(t, "")
+ publishHABox(t, cx, "box-ie-default", false, nil)
+ publishHABox(t, cx, "box-ie-tito", false, nil)
+ ie := cx.srv.cfg.claimPlacementRegion("ie")
+ for pool, want := range map[string]string{"": "box-ie-default", "tito": "box-ie-tito"} {
+ x := cx.srv.placementChooser(pool)
+ if x == nil {
+ t.Fatalf("pool %q: no chooser with region HA on", pool)
+ }
+ if x.Namespace == "" || x.Namespace != cx.srv.cfg.ClaimAuthority.Namespace {
+ t.Fatalf("pool %q: chooser namespace %q, want the authority's %q", pool, x.Namespace, cx.srv.cfg.ClaimAuthority.Namespace)
+ }
+ n, err := x.Capacity(ctx, ie)
+ if err != nil || n == 0 {
+ t.Fatalf("pool %q: capacity %d %v, want a placeable box", pool, n, err)
+ }
+ boxes, err := x.PlaceableBoxes(ctx, ie)
+ if err != nil || len(boxes) != 1 || boxes[0] != want {
+ t.Fatalf("pool %q: placeable %v %v, want [%s]", pool, boxes, err, want)
+ }
+ if err := cx.srv.roomIn(ctx, "ie", pool); err != nil {
+ t.Fatalf("pool %q: roomIn with a placeable box: %v", pool, err)
+ }
+ }
+ if n, err := cx.srv.cfg.ClaimAuthority.Executors.Capacity(ctx, ie); err != nil || n != 0 {
+ t.Fatalf("the unscoped chooser counted %d (%v): the scope is not what makes the count", n, err)
+ }
+ cx.srv.cfg.RegionHA = false
+ if cx.srv.placementChooser("") != nil || cx.srv.roomIn(ctx, "ie", "") != nil {
+ t.Fatal("region HA off: a chooser or a refusal")
+ }
+}
diff --git a/internal/console/server.go b/internal/console/server.go
index 5f89d5076..09a3a3aa1 100644
--- a/internal/console/server.go
+++ b/internal/console/server.go
@@ -837,6 +837,10 @@ type Config struct {
type Server struct {
claimLossPageMu sync.Mutex
claimLossPageDone chan struct{}
+
+ // noCapacity limits the no-capacity alert to once per region an hour
+ // in this process (region HA spec 2 §7, H27, nocapacity.go).
+ noCapacity noCapacityAlerts
// Enabled claim runtime (ConfigureClaimRuntime). Unset flag-off.
claimRuntimeConfigured bool
claimOwner hostops.RegistrationOwner
diff --git a/internal/hostops/backfill.go b/internal/hostops/backfill.go
index 9f13f21c2..7cd117de0 100644
--- a/internal/hostops/backfill.go
+++ b/internal/hostops/backfill.go
@@ -107,10 +107,16 @@ type ProductionAuthority struct {
// before the create CAS, when a chooser is configured; otherwise no
// selection, and the create is today's. The chooser chooses within claim
// Plan 5's scope: this authority's namespace and the record's pool.
+//
+// A door that already chose for this host (WithChosenExecutor) is held to
+// that choice, so the create names the box the door checked (H27).
func (a ProductionAuthority) SelectExecutor(ctx context.Context, obs hostclaim.Observed, r Record) (hostclaim.ExecutorSelection, bool, error) {
if a.Executors == nil {
return hostclaim.ExecutorSelection{}, false, nil
}
+ if sel, ok := ChosenExecutor(ctx, r.Holder.Host); ok {
+ return sel, true, nil
+ }
var shape struct {
Pool string `json:"pool"`
}
diff --git a/internal/hostops/chosen.go b/internal/hostops/chosen.go
new file mode 100644
index 000000000..a01b400d0
--- /dev/null
+++ b/internal/hostops/chosen.go
@@ -0,0 +1,37 @@
+package hostops
+
+import (
+ "context"
+
+ "github.com/carlosframework/platform/internal/hostclaim"
+)
+
+// A door that chooses its executor before it registers anything (region HA
+// spec 2 §7, H27) carries the choice to the create through the context, so
+// the create names exactly the box the door checked, and a refusal leaves
+// nothing registered, nothing written to the claim and nothing for
+// recovery to settle. selectExecutor still checks the choice against
+// claim Plan 5's scope (namespace, region, pool, active, unfenced).
+
+type chosenExecutorKey struct{}
+
+type chosenExecutor struct {
+ host string
+ sel hostclaim.ExecutorSelection
+}
+
+// WithChosenExecutor records sel as host's executor for the create that
+// ctx carries.
+func WithChosenExecutor(ctx context.Context, host string, sel hostclaim.ExecutorSelection) context.Context {
+ return context.WithValue(ctx, chosenExecutorKey{}, chosenExecutor{host: host, sel: sel})
+}
+
+// ChosenExecutor is the executor ctx carries for host, if any: a door that
+// already chose finds its choice here and does not choose again.
+func ChosenExecutor(ctx context.Context, host string) (hostclaim.ExecutorSelection, bool) {
+ c, ok := ctx.Value(chosenExecutorKey{}).(chosenExecutor)
+ if !ok || c.host == "" || c.host != host || c.sel.Box == "" || c.sel.Enrollment == "" {
+ return hostclaim.ExecutorSelection{}, false
+ }
+ return c.sel, true
+}
diff --git a/internal/hostops/executor_test.go b/internal/hostops/executor_test.go
index 6c8a58822..8c172ee6e 100644
--- a/internal/hostops/executor_test.go
+++ b/internal/hostops/executor_test.go
@@ -491,3 +491,31 @@ func TestExecutorChooserPlacesAHomeRecordUnderTheHomeBinding(t *testing.T) {
t.Fatalf("unbound: %v", err)
}
}
+
+// Region HA spec 2 §7 (H27): a door that chose before registering holds
+// the create to its choice, for that host only; Plan 5's scope still
+// binds it (TestProductionCreateSelectsThroughTheChooser).
+func TestSelectExecutorHonoursTheDoorsChoice(t *testing.T) {
+ ctx := context.Background()
+ x, _, snap, _ := chooserFixture(t, "box-a", "box-b")
+ a := ProductionAuthority{Namespace: "platform", Executors: &x}
+ r := chooserRecord()
+ absent := hostclaim.Observed{Host: r.Holder.Host, Absent: true}
+ chosen := hostclaim.ExecutorSelection{Box: "box-b", Enrollment: strings.Repeat("b", 32)}
+ // Every box is now ineligible, yet the door's earlier choice stands.
+ for i := range snap.Boxes {
+ snap.Boxes[i].Draining = true
+ }
+ sel, ok, err := a.SelectExecutor(WithChosenExecutor(ctx, r.Holder.Host, chosen), absent, r)
+ if err != nil || !ok || sel != chosen {
+ t.Fatalf("with the door's choice: %+v %v %v", sel, ok, err)
+ }
+ // Another host's choice is not this create's.
+ if _, _, err := a.SelectExecutor(WithChosenExecutor(ctx, "other.bab.oncarlos.com", chosen), absent, r); !errors.Is(err, ErrNoEligibleBox) {
+ t.Fatalf("another host's choice was used: %v", err)
+ }
+ // No chooser (region HA off): no selection, whatever the context says.
+ if _, ok, err := (ProductionAuthority{}).SelectExecutor(WithChosenExecutor(ctx, r.Holder.Host, chosen), absent, r); ok || err != nil {
+ t.Fatalf("region HA off selected: %v %v", ok, err)
+ }
+}
diff --git a/internal/provision/plan14_root_reads_test.go b/internal/provision/plan14_root_reads_test.go
index cad6a66a5..a012e618b 100644
--- a/internal/provision/plan14_root_reads_test.go
+++ b/internal/provision/plan14_root_reads_test.go
@@ -797,7 +797,7 @@ func TestPlan14RootReadCensus(t *testing.T) {
if err = loadRootReadInventory(filepath.Join(root, "internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv"), &inv, true); err != nil {
t.Fatal(err)
}
- seal = "4f1164be6f79275a71801f5309a77b6e773fe2b4343820106ce8e3ab8e6c2a2a"
+ seal = "57efed074effbf75c0179626dcec5026330081958e88e3b62c3f9c55a5df1944"
errs := append(compareRootReads(got, inv), checkRootReadLegacy(inv, seal)...)
errs = append(errs, checkRootReadDeclarations(got, inv)...)
if len(errs) > 0 {
diff --git a/internal/provision/plan9_b8_census_test.go b/internal/provision/plan9_b8_census_test.go
index daff9cc12..b9de09793 100644
--- a/internal/provision/plan9_b8_census_test.go
+++ b/internal/provision/plan9_b8_census_test.go
@@ -674,7 +674,8 @@ var plan9B8CensusInventory = []censusEntry{
// Region HA stage B: the edges read moved whole into internal/edgesfile (EvidenceComplete); the record is still created through CreateRecord.
// Re-reviewed 2026-10-12 (merge of main's home-region work): main's body (claimPlacementRegion) with stage B's one-line EvidenceComplete call; no new write.
// Re-reviewed at the merge of carlos-run into seeded-instances (2026-10-13): main's door with the two seed hand-offs: once every ordinary check has passed, a create that names `from` or `import` leaves for createFrom or createImport, which write their own record (beginFrom, beginImport, below) and refuse a claim console first (seedRegionReady); the ordinary create asks bindEnvironmentCreate before its write. The door's own CreateRecord and its claim create are as main wrote them.
- {"internal/console/apiinstances.go", "Server.handleAPIInstanceCreate", "record-writer", "key:CreateRecord", "6763eab45774b9e624a9b820ce1585ef3910355bc3ce589b82298e1d12469a2e"},
+ // Re-reviewed for region HA 2a Task 11 (H27): the door asks the region HA chooser (app-derived region, roomIn) and chooses before the environment binding (chooseCreateFirst); the claim create closure is lifted into buildClaim unchanged. No write site is added or changed: CreateRecord and the claim create are as before.
+ {"internal/console/apiinstances.go", "Server.handleAPIInstanceCreate", "record-writer", "key:CreateRecord", "55b802e0cdbae1a970cc8c8a8d1f79b137f82b60cd2ce76af8a2f408f10380d5"},
// Re-reviewed at the merge of carlos-run into seeded-instances (2026-10-13): main's door under the environment-binding lock (lockEnvironmentUpdate); the UpdateRecord call is unchanged, and UpdateRecord refuses any change to the record's seed.
{"internal/console/apiinstances.go", "Server.handleAPIInstanceUpdate", "record-writer", "key:UpdateRecord", "43a8bead675a4aa33777c6c68431882d55d79a9c1d27e1a54c8f103dda087de0"},
{"internal/console/apiscan.go", "Server.writeScanRecord", "nonrecord", "sel:Put", "c3c092f4b7be9bf6bf09d0fe1e2c9fe52892ee97e57f6c7ab964b85fe36fb5c6"},
diff --git a/internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv b/internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv
index b5ee6ae87..1df9d4843 100644
--- a/internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv
+++ b/internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv
@@ -9,19 +9,24 @@
# Task 9: the load balancer health answer (internal/edge lbhealth.go, answered in httpFallbackHandler; the home map and the certificate mirror latch their first loads, Edge.Watch beats, the activator's sweeper beats through Service.LastSweep, and ClaimWiring.startRegional marks the box hosting). The only root-read unit that moved is the activator's authority row type Service, which gains one atomic.Int64 field (lastSweep) written after each finished sweep and read by LastSweep: no read, write or decision of the seed rows' authority changes, so its row is re-pinned to the new digest with its class, owner and basis unchanged. The answer reads only in-memory state. Envelope-only, and the legacy seal stands.
# Task 10: the operator's box and group role doors (internal/console apiboxroles.go, registered with host claims on in registerCLIRoutes) and the CLI (system_roles.go). Census ruling region-ha-2a (platform-1e, approved with conditions) covers one accepted-legacy unit, Server.boxRoster: the role doors' read of the signed roster (Directory.Read), operator-only (it is reached only past apiSystemRoller), read-only, the same roster every reader reads, and no role it reads feeds root's claim authority. Its basis is main's for a console root input (J4). The seal moves.
# Task 10 (system.go): the box and group verbs add `box` and `group` to systemVerb's verb list and two cases to runSystem's dispatch, calling runSystemBoxVerb and runSystemGroupVerb (system_roles.go). The two seed-overlay authority declarations are re-digested, class, task, basis and (empty) kinds unchanged: no root read is added, and the verbs reach the console over the operator's CLI session.
+# Task 11: H27 on the create doors (internal/console nocapacity.go; hostops chosen.go): beginClaimCreate chooses the executor before it registers anything (chooseExecutorFirst) and carries the choice to the create (hostops.WithChosenExecutor, honoured by ProductionAuthority.SelectExecutor), claimResponseRefusal answers no room with a 503 on instance create, domain attach's first route (attachClaimFirstRoute, unchanged) and standalone sidecar create, and handleAPIInstanceCreate judges an app-derived region by the chooser's eligible set. handleAPIInstanceCreate chooses before its first write, the environment binding (chooseCreateFirst), and beginClaimCreate keeps that choice; with region HA on, an explicit region with no box that has room is the no-room refusal before the armed-placement gate (roomIn). Census ruling region-ha-2a covers two accepted-legacy units: Server.beginClaimCreate re-frozen (one call added before RegisterStart, its one root-read site, the claim read file:Read@470, unchanged), and Server.chooseCreateFirst, new: the same claim read, made before the binding, feeding only the placement choice; it returns its choice with the claim observation it decided on, and beginClaimCreate keeps the choice only if its own read is that observation (ruling item 5). The seal moves.
internal/activator/activator.go type Service authority B,C,D,F,J4 rev3=PROPOSED-seed; existing activator lifecycle and due-wake state plus owned seed exclusion and incarnation-keyed diagnostics; normal refusal and preparation ordering remain authoritative be4ca3a3ea2bd768e7e52c1056513d4fef1c1702d2e04656c540cdcbb858865c
@coverage internal/lbhealth ad1211a01a4eb3d02cd8ed95cbff0f8b17b3976ce9c0279ebd2cae2f20c07124
@coverage internal/hostactors 6736698380a247764ada687f2476f89382fc32a1183a50c06afe75967d716eab
@coverage internal/boxchoice 1bfd561bcb48b51e60e63f99e086fd44d3b1be2160e28b3ed0253a14bbb9ff66
@coverage internal/edgesfile f62c3973bb48fa36f561d26f496c0ba9a14fe882fe5843867e9f7a9eaf0abbdf
@coverage internal/host 4ad9bdcd8b94516a6ccdad3bc900b166833faf1f0334ac83478be5b04de47069
-@coverage internal/hostops 3dc23aaff7bdae0341a8c9a6fbac076a2f57ca1e4676d83543433ff9ad7d1c79
+@coverage internal/hostops 6e494cc9e53bf4fa4433fad798ecd3493c972f65083b0b2153b253f0bd50dcba
@coverage internal/edge 20177c9e7dc961937677bbf2763c73004d7c95f68512591800f6fe841c36152f
internal/host/edgesfile.go edgesFileWriteClaims authority F,J1 root-route;kept=F;scope=protected-decision-only registry:Account@10835,registry:Alias@5297,registry:Alias@5332,registry:Alias@5754,registry:Alias@5911,registry:Alias@5981,registry:Alias@6013,registry:Alias@6482,registry:Alias@8772,registry:App@10847,registry:DNSStatus@5429,registry:DNSStatus@5920,registry:Host@10480,registry:Host@2878,registry:Host@5389,registry:Host@6234,registry:Host@6567,registry:Host@8819,registry:Host@8866,registry:Host@8916,registry:Host@8951,registry:Host@9000,registry:Platform@2921,registry:Provisioned@2935,registry:Replica@2959,registry:Status@877,whole:Route@3212,whole:Route@323,whole:Route@3505,whole:Route@351,whole:RouteAlias@5282,whole:RouteAlias@8725,whole:RoutePath@3520,whole:WildcardDomain@8905 8c7cdffba66886b92f9a2ac663a86dbdd5f80699afd2e6afb4571da916c727ad
@coverage internal/activator fd9cf14712f55895541bb8bfbf8137cfd446298bcaa8634b0aba7b8344c29ec3
-@coverage internal/console 1205d53b2e2a2b69a15bfcee5952d4345daf1f133d077b87c9c2d37a118a6dc6
+@coverage internal/console b3f337a5ba7806d28fe6c4d92882accba8572f94d15b008d793f57c26f1bee34
@coverage . 0e6e4542758248000de1882a01f13e3449c880b4ae23e8b2ff0d6dce076b614c
@legacy b02b534b6617f57381ba12e9d9a2bf1bd63f737dc68005de926ff18b7db93dd3 internal/console/apiboxroles.go Server.boxRoster legacy accepted-legacy accepted-legacy;rev3=J4;basis=root-input;census-ruling-region-ha-2a file:Read@484 9be9925de14dbbd7d86123bf9faf1535c7b116575cadcb8ba983a665af0513f4
internal/console/apiboxroles.go Server.boxRoster legacy accepted-legacy accepted-legacy;rev3=J4;basis=root-input;census-ruling-region-ha-2a file:Read@484 9be9925de14dbbd7d86123bf9faf1535c7b116575cadcb8ba983a665af0513f4
system.go runSystem authority B,C,D,F,J4 rev3=PROPOSED-seed; command arguments, claim flags, service identity and root delegation capability before constructing agent-mode resources; one-shot skips Configure, Start, duties and listeners; close resources only after private runtime joins 529ab54816bd15161c4f57e2fcb5899581ee7098a465cd3d53b070bde8ddf415
system.go systemVerb authority B,C,D,F,J4 rev3=PROPOSED-seed; command arguments, claim flags, service identity and root delegation capability before constructing agent-mode resources; one-shot skips Configure, Start, duties and listeners; close resources only after private runtime joins 7067c6024d46c6da98f6db1276aff895b41c0d00939e101ccf70cff8b691fd27
+@legacy 546fc7978037e417ac9df597a95f21add19b025e2d569f8fef7ed4695f0b8bfd internal/console/claim_create.go Server.beginClaimCreate legacy accepted-legacy accepted-legacy;rev3=J4;basis=root-input;census-ruling-region-ha-2a file:Read@470 ff9b246aa3dcfe180ea1901ecd9fc1b3b18561dc309e08c6a922870d89179705
+internal/console/claim_create.go Server.beginClaimCreate legacy accepted-legacy accepted-legacy;rev3=J4;basis=root-input;census-ruling-region-ha-2a file:Read@470 ff9b246aa3dcfe180ea1901ecd9fc1b3b18561dc309e08c6a922870d89179705
+@legacy 5aa99a1b92eb73601585a25265c5285bcfd0c6b6ae7f402248eb1a299bc79831 internal/console/nocapacity.go Server.chooseCreateFirst legacy accepted-legacy accepted-legacy;rev3=J4;basis=root-input;census-ruling-region-ha-2a file:Read@271 7cc51e7812cc59f39ea23d5285c621d912557f4890d6e76d566f6182b0fe3ffd
+internal/console/nocapacity.go Server.chooseCreateFirst legacy accepted-legacy accepted-legacy;rev3=J4;basis=root-input;census-ruling-region-ha-2a file:Read@271 7cc51e7812cc59f39ea23d5285c621d912557f4890d6e76d566f6182b0fe3ffd
diff --git a/region_ha_boundary_test.go b/region_ha_boundary_test.go
index 67f9946de..c2668a896 100644
--- a/region_ha_boundary_test.go
+++ b/region_ha_boundary_test.go
@@ -27,6 +27,8 @@ var regionHACoreFiles = []string{
"internal/edge/lbhealth.go",
"internal/console/apiboxroles.go",
"system_roles.go",
+ "internal/console/nocapacity.go",
+ "internal/hostops/chosen.go",
}
// regionHAAdapterFiles are spec 2's AWS adapters. Each is named for AWS so