carlos / platform Public

hostops: the create chooser chooses within claim Plan 5's scope, this console's namespace and the record's pool, and reads only those boxes' fences

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Paul Campbell pushed by claude 2 weeks ago 53f26ad616fefcea4518a39cf5c0654fbc988965 parent d1ce3f7

4 files changed, +140 −15

internal/hostops/backfill.go +11 −2
internal/hostops/executor.go +30 −1
internal/hostops/executor_test.go +93 −7
internal/hostops/operations.go +6 −5

diff --git a/internal/hostops/backfill.go b/internal/hostops/backfill.go

index 3cb7989ae..38df9f225 100644

--- a/internal/hostops/backfill.go

+++ b/internal/hostops/backfill.go

@@ -100,12 +100,21 @@ type ProductionAuthority struct {

// SelectExecutor is Authority's create-time selection: choose() once,

// before the create CAS, when a chooser is configured; otherwise no

-// selection, and the create is today's.

+// selection, and the create is today's. The chooser chooses within claim

+// Plan 5's scope: this authority's namespace and the record's pool.

func (a ProductionAuthority) SelectExecutor(ctx context.Context, obs hostclaim.Observed, r Record) (hostclaim.ExecutorSelection, bool, error) {

if a.Executors == nil {

return hostclaim.ExecutorSelection{}, false, nil

}

- sel, err := a.Executors.Choose(ctx, obs, r)

+ var shape struct {

+ Pool string `json:"pool"`

+ }

+ if err := json.Unmarshal(r.Body, &shape); err != nil {

+ return hostclaim.ExecutorSelection{}, false, err

+ }

+ x := *a.Executors

+ x.Namespace, x.Pool = a.Namespace, shape.Pool

+ sel, err := x.Choose(ctx, obs, r)

return sel, err == nil, err

}

diff --git a/internal/hostops/executor.go b/internal/hostops/executor.go

index 84e22285e..d948393a1 100644

--- a/internal/hostops/executor.go

+++ b/internal/hostops/executor.go

@@ -30,8 +30,19 @@ type ExecutorChooser struct {

SecretsKeyFP string

HomeName string

Now func() time.Time

+ // Namespace and Pool are the scope the chooser chooses within, claim

+ // Plan 5's create scope: only roster boxes in Namespace serving Pool

+ // are candidates, and only their fences are read. Pool "" is the

+ // shared pool (the roster's "default"). An empty Namespace matches no

+ // roster box, so it chooses nothing. ProductionAuthority.SelectExecutor

+ // sets both from its own namespace and the record's pool.

+ Namespace, Pool string

}

+// sharedPool is the roster's name for the shared "" pool (claim Plan 5's

+// selectExecutor reads a record's empty pool as it).

+const sharedPool = "default"

// Choose returns the box, and its roster enrollment, that a new host in

// r's placement is created on. Identity, placement and AWS region come from

// the roster; a box with a valid termination fence, or not active, is never

@@ -86,9 +97,15 @@ func (x ExecutorChooser) eligible(ctx context.Context, obs hostclaim.Observed, r

if !ok || region.AWSRegion == "" {

return hostactors.Observed{}, nil, ErrContract

}

+ pool := x.Pool

+ if pool == "" {

+ pool = sharedPool

+ }

roster := map[string]boxchoice.RosterBox{}

for ref, b := range o.Roster.Boxes {

- if b.State != "active" || b.Region != placement {

+ // Scope first, so a fence is read only for a box this create could

+ // name: another namespace's unreadable fence never blocks it.

+ if b.State != "active" || b.Region != placement || b.Namespace != x.Namespace || b.Pool != pool {

continue

}

fenced, err := x.Directory.TerminationFence(ctx, o, ref, x.Now())

@@ -103,6 +120,18 @@ func (x ExecutorChooser) eligible(ctx context.Context, obs hostclaim.Observed, r

if err != nil {

return hostactors.Observed{}, nil, err

}

+ if pool != sharedPool {

+ // serves_shared_pool is a box's answer for the shared pool only. A

+ // named pool's membership is the roster's, already applied above,

+ // so the shared-pool answer is not asked of its boxes.

+ yes := true

+ boxes := make([]boxchoice.Box, len(snap.Boxes))

+ for i, b := range snap.Boxes {

+ b.ServesSharedPool = &yes

+ boxes[i] = b

+ }

+ snap.Boxes = boxes

+ }

// Never a box on the claim's lost list, nor one that still owes a

// park-local chore for this host (spec 1 §2: its old installation is

// not yet retired).

diff --git a/internal/hostops/executor_test.go b/internal/hostops/executor_test.go

index ad5243aa8..fc61916d4 100644

--- a/internal/hostops/executor_test.go

+++ b/internal/hostops/executor_test.go

@@ -20,6 +20,13 @@ import (

)

func chooserFixture(t *testing.T, boxes ...string) (ExecutorChooser, release.Store, *boxchoice.Snapshot, map[string]boxchoice.Load) {

+ t.Helper()

+ return chooserFixtureWith(t, nil, boxes...)

+// chooserFixtureWith is chooserFixture with edit applied to the roster

+// before it is signed (another namespace or pool for some box).

+func chooserFixtureWith(t *testing.T, edit func(*hostactors.Roster), boxes ...string) (ExecutorChooser, release.Store, *boxchoice.Snapshot, map[string]boxchoice.Load) {

t.Helper()

s := release.NewDirStore(t.TempDir())

pub, key, err := ed25519.GenerateKey(rand.Reader)

@@ -37,6 +44,9 @@ func chooserFixture(t *testing.T, boxes ...string) (ExecutorChooser, release.Sto

snap.Boxes = append(snap.Boxes, boxchoice.Box{BoxID: ref, EIP: "203.0.113.1", Region: "eu-west-1", Placement: "ie", Fresh: true,

HibernationConfigured: &yes, ServesSharedPool: &yes, AcceptsInbound: &yes, SecretsKeyFP: "fp", Capabilities: []string{boxchoice.Capability}})

}

+ if edit != nil {

+ edit(&roster)

+ }

wire, err := release.SignRootEnvelope(release.RootSigner(key), roster)

if err != nil {

t.Fatal(err)

@@ -45,7 +55,7 @@ func chooserFixture(t *testing.T, boxes ...string) (ExecutorChooser, release.Sto

t.Fatal(err)

}

loads := map[string]boxchoice.Load{}

- x := ExecutorChooser{Directory: hostactors.Directory{Store: s, PublicKey: pub, Bucket: "bucket"}, SecretsKeyFP: "fp",

+ x := ExecutorChooser{Directory: hostactors.Directory{Store: s, PublicKey: pub, Bucket: "bucket"}, SecretsKeyFP: "fp", Namespace: "platform",

Edges: func(context.Context) (boxchoice.Snapshot, error) { return *snap, nil },

Loads: func(context.Context) (map[string]boxchoice.Load, error) { return loads, nil },

Now: func() time.Time { return time.Date(2026, 10, 7, 9, 0, 0, 0, time.UTC) }}

@@ -53,7 +63,8 @@ func chooserFixture(t *testing.T, boxes ...string) (ExecutorChooser, release.Sto

}

func chooserRecord() Record {

- return Record{Holder: hostclaim.NewHolder{Family: "instance", Account: "bab", App: "jam", Host: "shop.bab.oncarlos.com", Region: "ie"}}

+ return Record{Holder: hostclaim.NewHolder{Family: "instance", Account: "bab", App: "jam", Host: "shop.bab.oncarlos.com", Region: "ie"},

+ Body: []byte(`{"host":"shop.bab.oncarlos.com","account":"bab","app":"jam","region":"ie"}`)}

}

func TestExecutorChooserSpreadsThenBalancesThenHashes(t *testing.T) {

@@ -255,12 +266,12 @@ func TestProductionAuthoritySelectsOnlyWithAChooser(t *testing.T) {

x, _, snap, _ := chooserFixture(t, "box-a", "box-b")

snap.Boxes[0].Draining = true

absent := hostclaim.Observed{Host: "shop.bab.oncarlos.com", Absent: true}

- sel, selected, err = (ProductionAuthority{Executors: &x}).SelectExecutor(context.Background(), absent, chooserRecord())

+ sel, selected, err = (ProductionAuthority{Executors: &x, Namespace: "platform"}).SelectExecutor(context.Background(), absent, chooserRecord())

if err != nil || !selected || sel.Box != "box-b" || sel.Enrollment != strings.Repeat("b", 32) {

t.Fatalf("with a chooser: %+v %v %v", sel, selected, err)

}

snap.Boxes[1].Draining = true

- if sel, selected, err = (ProductionAuthority{Executors: &x}).SelectExecutor(context.Background(), absent, chooserRecord()); !errors.Is(err, ErrNoEligibleBox) || selected {

+ if sel, selected, err = (ProductionAuthority{Executors: &x, Namespace: "platform"}).SelectExecutor(context.Background(), absent, chooserRecord()); !errors.Is(err, ErrNoEligibleBox) || selected {

t.Fatalf("no eligible box: %+v %v %v", sel, selected, err)

}

@@ -274,7 +285,6 @@ func TestProductionCreateSelectsThroughTheChooser(t *testing.T) {

ctx := context.Background()

x, s, snap, _ := chooserFixture(t, "box-a", "box-b")

r := chooserRecord()

- r.Body = []byte(`{"host":"shop.bab.oncarlos.com","account":"bab","app":"jam","region":"ie"}`)

absent := hostclaim.Observed{Host: r.Holder.Host, Absent: true}

a := ProductionAuthority{Store: s, Directory: x.Directory, Namespace: "platform", Now: x.Now}

off, _, pool, err := a.selectExecutor(ctx, absent, r)

@@ -295,8 +305,8 @@ func TestProductionCreateSelectsThroughTheChooser(t *testing.T) {

if err != nil || box != other || enrollment != strings.Repeat(map[string]string{"box-a": "a", "box-b": "b"}[other], 32) || pool != "default" {

t.Fatalf("with a chooser: %s %s %s %v, want %s", box, enrollment, pool, err, other)

}

- // Another console namespace: the chooser still names a box, but not

- // one Plan 5's scope holds.

+ // Another console namespace: the chooser chooses within it, and it

+ // holds no box.

a.Namespace = "elsewhere"

if box, _, _, err := a.selectExecutor(ctx, absent, r); !errors.Is(err, ErrNoEligibleBox) {

t.Fatalf("a box outside the namespace: %s %v", box, err)

@@ -308,3 +318,79 @@ func TestProductionCreateSelectsThroughTheChooser(t *testing.T) {

t.Fatalf("no eligible box: %s %v", box, err)

}

+// Fix round 1 (review Important 1): the chooser chooses within claim Plan

+// 5's scope, so a region that also holds another namespace's or another

+// pool's boxes places on the in-scope eligible box, never refusing because

+// an out-of-scope box would have won.

+func TestProductionCreateWithAChooserPlacesWithinPlan5sScope(t *testing.T) {

+ ctx := context.Background()

+ absent := hostclaim.Observed{Host: "shop.bab.oncarlos.com", Absent: true}

+ // The console's chooser carries no scope of its own: the authority

+ // supplies its namespace and the record's pool.

+ authority := func(x *ExecutorChooser, s release.Store) ProductionAuthority {

+ x.Namespace, x.Pool = "", ""

+ return ProductionAuthority{Store: s, Directory: x.Directory, Namespace: "platform", Now: x.Now, Executors: x}

+ }

+ t.Run("another namespace's box would win on load", func(t *testing.T) {

+ x, s, _, loads := chooserFixtureWith(t, func(r *hostactors.Roster) {

+ b := r.Boxes["box-b"]

+ b.Namespace = "other"

+ r.Boxes["box-b"] = b

+ }, "box-a", "box-b")

+ loads["box-a"] = boxchoice.Load{Hosts: 5, Apps: map[string]int{"bab/jam": 1}}

+ box, enrollment, pool, err := authority(&x, s).selectExecutor(ctx, absent, chooserRecord())

+ if err != nil || box != "box-a" || enrollment != strings.Repeat("a", 32) || pool != "default" {

+ t.Fatalf("got %s %s %s %v, want the in-scope box-a", box, enrollment, pool, err)

+ }

+ })

+ t.Run("a named pool", func(t *testing.T) {

+ // box-b serves only pool acme: its edges file does not serve the

+ // shared pool. box-a is the shared pool's, and idle.

+ x, s, snap, loads := chooserFixtureWith(t, func(r *hostactors.Roster) {

+ b := r.Boxes["box-b"]

+ b.Pool = "acme"

+ r.Boxes["box-b"] = b

+ }, "box-a", "box-b")

+ no := false

+ snap.Boxes[1].ServesSharedPool = &no

+ loads["box-b"] = boxchoice.Load{Hosts: 5}

+ named := chooserRecord()

+ named.Body = []byte(`{"host":"shop.bab.oncarlos.com","account":"bab","app":"jam","region":"ie","pool":"acme"}`)

+ box, enrollment, pool, err := authority(&x, s).selectExecutor(ctx, absent, named)

+ if err != nil || box != "box-b" || enrollment != strings.Repeat("b", 32) || pool != "acme" {

+ t.Fatalf("named pool: got %s %s %s %v, want box-b in acme", box, enrollment, pool, err)

+ }

+ // The shared pool still places on its own box, whatever the load.

+ loads["box-a"] = boxchoice.Load{Hosts: 50}

+ if box, _, pool, err := authority(&x, s).selectExecutor(ctx, absent, chooserRecord()); err != nil || box != "box-a" || pool != "default" {

+ t.Fatalf("shared pool: got %s %s %v, want box-a", box, pool, err)

+ }

+ // The named pool's box must still be eligible on its own evidence.

+ snap.Boxes[1].Draining = true

+ if box, _, _, err := authority(&x, s).selectExecutor(ctx, absent, named); !errors.Is(err, ErrNoEligibleBox) {

+ t.Fatalf("a draining named-pool box: %s %v", box, err)

+ }

+ })

+ t.Run("an unreadable fence on another namespace's box", func(t *testing.T) {

+ x, s, _, _ := chooserFixtureWith(t, func(r *hostactors.Roster) {

+ b := r.Boxes["box-b"]

+ b.Namespace = "other"

+ r.Boxes["box-b"] = b

+ }, "box-a", "box-b")

+ if _, err := s.Put(ctx, "control/box-fences/"+strings.Repeat("b", 32)+".json", []byte("{"), ""); err != nil {

+ t.Fatal(err)

+ }

+ if box, _, _, err := authority(&x, s).selectExecutor(ctx, absent, chooserRecord()); err != nil || box != "box-a" {

+ t.Fatalf("got %s %v, want box-a: an out-of-scope fence never blocks", box, err)

+ }

+ // In scope, the same unreadable fence still refuses (absence is

+ // never evidence).

+ if _, err := s.Put(ctx, "control/box-fences/"+strings.Repeat("a", 32)+".json", []byte("{"), ""); err != nil {

+ t.Fatal(err)

+ }

+ if _, _, _, err := authority(&x, s).selectExecutor(ctx, absent, chooserRecord()); err == nil || errors.Is(err, ErrNoEligibleBox) {

+ t.Fatalf("an unreadable in-scope fence must refuse: %v", err)

+ }

+ })

diff --git a/internal/hostops/operations.go b/internal/hostops/operations.go

index e7ea1c824..445b7fd9b 100644

--- a/internal/hostops/operations.go

+++ b/internal/hostops/operations.go

@@ -15,11 +15,12 @@ import (

// Production construction waits for the decisions named in this plan.

type Authority interface {

// SelectExecutor chooses the new host's executor once, before the

- // create CAS (claim P5-D2, region HA §5.2). selected false means this

- // deployment does not choose at create (CARLOS_REGION_HA off): the

- // create proceeds exactly as today, and Executor names the box at

- // completion. Production is ExecutorChooser.Choose behind that switch,

- // consulted inside StartCreate's own selection.

+ // create CAS (claim P5-D2, region HA §5.2). For a non-production

+ // authority, selected false means no choice at create: Attempt runs

+ // today's BeginCreate and Executor names the box at completion.

+ // Production is ExecutorChooser.Choose behind CARLOS_REGION_HA,

+ // consulted inside StartCreate's own selection; with it off, claim

+ // Plan 5's selectExecutor chooses as before.

SelectExecutor(context.Context, hostclaim.Observed, Record) (sel hostclaim.ExecutorSelection, selected bool, err error)

Create(context.Context, hostclaim.Observed, Record, string) error

Executor(context.Context, hostclaim.Observed, Record) (string, error)