amadan

carlos / platform Public

region HA 2a: the placement policy over the eligible set

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude e1ca43c15a6974cf5476e565274fae777cba181c parent a2789b7
7 files changed, +279 −14
  • internal/boxchoice/boxchoice.go +20 −9
  • internal/boxchoice/policy.go +106 −0
  • internal/boxchoice/policy_test.go +111 −0
  • internal/edgesfile/snapshot.go +13 −5
  • internal/edgesfile/snapshot_headroom_test.go +25 −0
  • internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv +3 −0
  • region_ha_boundary_test.go +1 −0
diff --git a/internal/boxchoice/boxchoice.go b/internal/boxchoice/boxchoice.go
index 10a7eccf9..00eab928f 100644
--- a/internal/boxchoice/boxchoice.go
+++ b/internal/boxchoice/boxchoice.go
@@ -21,11 +21,15 @@ import (
// region HA executor.
const Capability = "region-ha-1"
-// Headroom is what a box reports it has free.
+// Headroom is what a box reports it has free, and (region HA spec 2 §7)
+// the totals the free figures are fractions of. A total of 0 is a writer
+// that predates it, or a reading that failed.
type Headroom struct {
- MemFreeMiB int64 `json:"mem_free_mib"`
- DiskFreeMiB int64 `json:"disk_free_mib"`
- Awake int `json:"awake"`
+ MemFreeMiB int64 `json:"mem_free_mib"`
+ DiskFreeMiB int64 `json:"disk_free_mib"`
+ MemTotalMiB int64 `json:"mem_total_mib"`
+ DiskTotalMiB int64 `json:"disk_total_mib"`
+ Awake int `json:"awake"`
}
// Box is one box's edges file (control/edges/<box>.json, written by
@@ -48,6 +52,9 @@ type Box struct {
Capabilities []string `json:"capabilities"`
Draining bool `json:"draining"`
Headroom *Headroom `json:"headroom"`
+ // Generation is when the box last wrote the file (RFC 3339). Spec 2's
+ // policy trusts headroom only from a recent file (Placeable).
+ Generation string `json:"generation"`
// Fresh is the caller's verdict that the box's own status snapshot is
// still fresh. An edges file outlives its box, so the file alone is
// not liveness.
@@ -76,10 +83,13 @@ type RosterBox struct {
// Candidate is one box that may be named as a host's executor (region HA
// spec 1 §2, "eligible box").
type Candidate struct {
- BoxID string
- Region string
- MemFreeMiB, DiskFreeMiB int64
- Awake int
+ BoxID string
+ Region string
+ MemFreeMiB, DiskFreeMiB int64
+ MemTotalMiB, DiskTotalMiB int64
+ Awake int
+ // Generation is the box's edges file's (Box.Generation).
+ Generation string
}
// Load is what claims already place on a box.
@@ -132,9 +142,10 @@ func Eligible(snap Snapshot, roster map[string]RosterBox, placement, awsRegion,
case !slices.Contains(f.Capabilities, Capability):
case f.Draining, lost[f.BoxID]:
default:
- c := Candidate{BoxID: f.BoxID, Region: rb.Cloud.Region}
+ c := Candidate{BoxID: f.BoxID, Region: rb.Cloud.Region, Generation: f.Generation}
if f.Headroom != nil {
c.MemFreeMiB, c.DiskFreeMiB, c.Awake = f.Headroom.MemFreeMiB, f.Headroom.DiskFreeMiB, f.Headroom.Awake
+ c.MemTotalMiB, c.DiskTotalMiB = f.Headroom.MemTotalMiB, f.Headroom.DiskTotalMiB
}
out = append(out, c)
}
diff --git a/internal/boxchoice/policy.go b/internal/boxchoice/policy.go
new file mode 100644
index 000000000..7f704b7f1
--- /dev/null
+++ b/internal/boxchoice/policy.go
@@ -0,0 +1,106 @@
+package boxchoice
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "sort"
+ "time"
+)
+
+// Region HA spec 2's placement policy (§7). Its rules run over candidates
+// Eligible has already filtered for identity, placement and configuration
+// (§25): a provider-neutral box identity (#299) changes only Eligible,
+// never these.
+
+// The thresholds of §7.
+const (
+ // MinMemPercent: a box is placeable only while its free memory is at
+ // least this share of its total, the memory alarm's own line.
+ MinMemPercent = 15
+ // MinDiskPercent: and its data volume's free space at least this
+ // share of the volume.
+ MinDiskPercent = 10
+ // MaxEdgesAge: and its edges file, which carries the headroom, is no
+ // older than this. Files republish every 5 minutes plus up to a
+ // minute of jitter, so a healthy box always has one within two
+ // intervals and a margin (A1-8).
+ MaxEdgesAge = 15 * time.Minute
+ // maxEdgesSkew is how far in the future a file's generation may be
+ // before it is not believed: box clocks are synchronised, not equal.
+ maxEdgesSkew = time.Minute
+)
+
+// ExcludeRoles drops every candidate whose roster role is not the default
+// one (roles is box id -> role, "" for the default). Legacy and platform
+// boxes, and any role this build does not know, take no new instances.
+// The order of cands is kept.
+func ExcludeRoles(cands []Candidate, roles map[string]string) []Candidate {
+ var out []Candidate
+ for _, c := range cands {
+ if roles[c.BoxID] == "" {
+ out = append(out, c)
+ }
+ }
+ return out
+}
+
+// headroomFresh reports whether a candidate's edges file is recent enough
+// at now for its headroom to be trusted.
+func headroomFresh(generation string, now time.Time) bool {
+ at, err := time.Parse(time.RFC3339, generation)
+ if err != nil {
+ return false
+ }
+ age := now.Sub(at)
+ return age <= MaxEdgesAge && age >= -maxEdgesSkew
+}
+
+// Placeable is §7's eligibility on top of Eligible's: the default role,
+// free memory at least MinMemPercent of the total, free disk at least
+// MinDiskPercent of the volume, and an edges file no older than
+// MaxEdgesAge at now. A box that publishes no totals is not placeable: its
+// headroom cannot be judged. The order of cands is kept.
+func Placeable(cands []Candidate, roles map[string]string, now time.Time) []Candidate {
+ var out []Candidate
+ for _, c := range ExcludeRoles(cands, roles) {
+ switch {
+ case c.MemTotalMiB <= 0 || c.MemFreeMiB*100 < c.MemTotalMiB*MinMemPercent:
+ case c.DiskTotalMiB <= 0 || c.DiskFreeMiB*100 < c.DiskTotalMiB*MinDiskPercent:
+ case !headroomFresh(c.Generation, now):
+ default:
+ out = append(out, c)
+ }
+ }
+ return out
+}
+
+// ChooseRoomiest is §7's choice among placeable candidates: prefer a box
+// running no instance of app (soft spread: it never refuses), then the
+// most free memory, then the greatest hexadecimal SHA-256 of host + "\x00"
+// + box id (claim P5-D2), then the smallest box id. The answer does not
+// depend on the order of cands. The bool is false only when cands is
+// empty.
+func ChooseRoomiest(host string, cands []Candidate, running map[string]Load, app string) (Candidate, bool) {
+ if len(cands) == 0 {
+ return Candidate{}, false
+ }
+ tie := func(ref string) string {
+ sum := sha256.Sum256([]byte(host + "\x00" + ref))
+ return hex.EncodeToString(sum[:])
+ }
+ sorted := append([]Candidate(nil), cands...)
+ sort.SliceStable(sorted, func(i, j int) bool {
+ a, b := sorted[i], sorted[j]
+ aRuns, bRuns := running[a.BoxID].Apps[app] > 0, running[b.BoxID].Apps[app] > 0
+ switch {
+ case aRuns != bRuns:
+ return !aRuns
+ case a.MemFreeMiB != b.MemFreeMiB:
+ return a.MemFreeMiB > b.MemFreeMiB
+ case tie(a.BoxID) != tie(b.BoxID):
+ return tie(a.BoxID) > tie(b.BoxID)
+ }
+ return a.BoxID < b.BoxID
+ })
+ return sorted[0], true
+}
diff --git a/internal/boxchoice/policy_test.go b/internal/boxchoice/policy_test.go
new file mode 100644
index 000000000..e33bc4faf
--- /dev/null
+++ b/internal/boxchoice/policy_test.go
@@ -0,0 +1,111 @@
+package boxchoice
+
+import (
+ "reflect"
+ "testing"
+ "time"
+)
+
+var policyNow = time.Date(2026, 10, 16, 12, 0, 0, 0, time.UTC)
+
+// roomy is a candidate every §7 rule passes at policyNow.
+func roomy(id string, memFree int64) Candidate {
+ return Candidate{BoxID: id, Region: "eu-west-2", MemFreeMiB: memFree, MemTotalMiB: 8000, DiskFreeMiB: 50000, DiskTotalMiB: 100000,
+ Generation: policyNow.Add(-5 * time.Minute).Format(time.RFC3339)}
+}
+
+func TestPlaceableAppliesEachRule(t *testing.T) {
+ gen := func(d time.Duration) string { return policyNow.Add(d).Format(time.RFC3339) }
+ cases := []struct {
+ name string
+ mut func(*Candidate)
+ role string
+ want bool
+ }{
+ {"every rule holds", nil, "", true},
+ {"memory exactly 15%", func(c *Candidate) { c.MemFreeMiB = 1200 }, "", true},
+ {"memory under 15%", func(c *Candidate) { c.MemFreeMiB = 1199 }, "", false},
+ {"no memory total", func(c *Candidate) { c.MemTotalMiB = 0 }, "", false},
+ {"disk exactly 10%", func(c *Candidate) { c.DiskFreeMiB = 10000 }, "", true},
+ {"disk under 10%", func(c *Candidate) { c.DiskFreeMiB = 9999 }, "", false},
+ {"no disk total", func(c *Candidate) { c.DiskTotalMiB = 0 }, "", false},
+ {"edges file exactly 15 minutes old", func(c *Candidate) { c.Generation = gen(-15 * time.Minute) }, "", true},
+ {"edges file older than 15 minutes", func(c *Candidate) { c.Generation = gen(-15*time.Minute - time.Second) }, "", false},
+ {"edges file a minute in the future", func(c *Candidate) { c.Generation = gen(time.Minute) }, "", true},
+ {"edges file further in the future", func(c *Candidate) { c.Generation = gen(time.Minute + time.Second) }, "", false},
+ {"no generation", func(c *Candidate) { c.Generation = "" }, "", false},
+ {"unreadable generation", func(c *Candidate) { c.Generation = "yesterday" }, "", false},
+ {"legacy role", nil, "legacy", false},
+ {"platform role", nil, "platform", false},
+ {"a role this build does not know", nil, "draining", false},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ c := roomy("i-0aaa", 4000)
+ if tc.mut != nil {
+ tc.mut(&c)
+ }
+ got := Placeable([]Candidate{c}, map[string]string{"i-0aaa": tc.role}, policyNow)
+ if (len(got) == 1) != tc.want {
+ t.Fatalf("placeable = %v, want %v", ids(got), tc.want)
+ }
+ })
+ }
+}
+
+func TestExcludeRolesKeepsOrderAndDefaults(t *testing.T) {
+ cands := []Candidate{roomy("c", 1), roomy("a", 1), roomy("b", 1)}
+ got := ExcludeRoles(cands, map[string]string{"a": "legacy", "b": ""})
+ if !reflect.DeepEqual(ids(got), []string{"c", "b"}) {
+ t.Fatalf("got %v", ids(got))
+ }
+ if got := ExcludeRoles(cands, nil); len(got) != 3 {
+ t.Fatalf("no roles excluded %v", ids(got))
+ }
+}
+
+func TestChooseRoomiestSpreadsThenMostMemoryThenHash(t *testing.T) {
+ running := map[string]Load{"a": {Hosts: 1, Apps: map[string]int{"bab/jam": 1}}}
+ // a has the most memory but already runs the app: spread wins.
+ got, ok := ChooseRoomiest("h.example", []Candidate{roomy("a", 7000), roomy("b", 2000), roomy("c", 3000)}, running, "bab/jam")
+ if !ok || got.BoxID != "c" {
+ t.Fatalf("spread then memory: got %q", got.BoxID)
+ }
+ // Spread never refuses: every box runs the app, so memory decides.
+ all := map[string]Load{"a": running["a"], "b": running["a"]}
+ got, _ = ChooseRoomiest("h.example", []Candidate{roomy("a", 1000), roomy("b", 2000)}, all, "bab/jam")
+ if got.BoxID != "b" {
+ t.Fatalf("all run the app: got %q", got.BoxID)
+ }
+ // Fewer hosts is no longer a rule: memory beats it.
+ busy := map[string]Load{"a": {Hosts: 50}, "b": {Hosts: 1}}
+ got, _ = ChooseRoomiest("h.example", []Candidate{roomy("a", 5000), roomy("b", 4000)}, busy, "bab/jam")
+ if got.BoxID != "a" {
+ t.Fatalf("memory over host count: got %q", got.BoxID)
+ }
+ // Equal memory: the P5-D2 hash, whatever the order.
+ one, _ := ChooseRoomiest("h.example", []Candidate{roomy("a", 1000), roomy("b", 1000)}, nil, "bab/jam")
+ two, _ := ChooseRoomiest("h.example", []Candidate{roomy("b", 1000), roomy("a", 1000)}, nil, "bab/jam")
+ if one.BoxID != two.BoxID {
+ t.Fatalf("order changed the answer: %q %q", one.BoxID, two.BoxID)
+ }
+ want, _ := Choose("h.example", []Candidate{roomy("a", 1000), roomy("b", 1000)}, nil, "bab/jam")
+ if one.BoxID != want.BoxID {
+ t.Fatalf("hash tie-break differs from spec 1's: %q, want %q", one.BoxID, want.BoxID)
+ }
+ if _, ok := ChooseRoomiest("h.example", nil, nil, "bab/jam"); ok {
+ t.Fatal("chose from nothing")
+ }
+}
+
+func TestEligibleCarriesTotalsAndGeneration(t *testing.T) {
+ body := haBox("i-0aaa", func(m map[string]any) {
+ m["headroom"] = map[string]any{"mem_free_mib": 1000, "disk_free_mib": 2000, "mem_total_mib": 8000, "disk_total_mib": 40000, "awake": 1}
+ m["generation"] = "2026-10-16T11:55:00Z"
+ })
+ got := Eligible(views(t, body), rosterFor("i-0aaa"), "uk", "eu-west-2", "ie", testKeyFP, nil)
+ want := Candidate{BoxID: "i-0aaa", Region: "eu-west-2", MemFreeMiB: 1000, DiskFreeMiB: 2000, MemTotalMiB: 8000, DiskTotalMiB: 40000, Awake: 1, Generation: "2026-10-16T11:55:00Z"}
+ if len(got) != 1 || got[0] != want {
+ t.Fatalf("got %+v, want %+v", got, want)
+ }
+}
diff --git a/internal/edgesfile/snapshot.go b/internal/edgesfile/snapshot.go
index 10308ccbb..79be51eff 100644
--- a/internal/edgesfile/snapshot.go
+++ b/internal/edgesfile/snapshot.go
@@ -58,11 +58,18 @@ type PlacementView struct {
Capabilities []string `json:"capabilities"`
Draining bool `json:"draining"`
Headroom *struct {
- MemFreeMiB int64 `json:"mem_free_mib"`
- DiskFreeMiB int64 `json:"disk_free_mib"`
- Awake int `json:"awake"`
+ MemFreeMiB int64 `json:"mem_free_mib"`
+ DiskFreeMiB int64 `json:"disk_free_mib"`
+ MemTotalMiB int64 `json:"mem_total_mib"`
+ DiskTotalMiB int64 `json:"disk_total_mib"`
+ Awake int `json:"awake"`
} `json:"headroom"`
Hosts map[string]json.RawMessage `json:"hosts"`
+
+ // Generation is when the box last wrote the file (RFC 3339), which
+ // region HA spec 2's placement policy reads as headroom's age.
+ Generation string `json:"generation"`
+
// The moves protocol (moves spec §11), read only by the Phase 1 move
// door's floor check (selectMoveBoxes); Task 13 adds the writer's side.
// MovesProtocol is the protocol the box speaks (absent: 0).
@@ -246,10 +253,11 @@ func (s Snapshot) BoxChoice() boxchoice.Snapshot {
Box: f.Box, BoxID: f.BoxID, EIP: f.EIP, Region: f.Region, Placement: f.Placement,
HibernationConfigured: f.HibernationConfigured, ServesSharedPool: f.ServesSharedPool,
AcceptsInbound: f.AcceptsInbound, SecretsKeyFP: f.SecretsKeyFP, Capabilities: f.Capabilities,
- Draining: f.Draining, Fresh: f.Fresh,
+ Draining: f.Draining, Generation: f.Generation, Fresh: f.Fresh,
}
if f.Headroom != nil {
- b.Headroom = &boxchoice.Headroom{MemFreeMiB: f.Headroom.MemFreeMiB, DiskFreeMiB: f.Headroom.DiskFreeMiB, Awake: f.Headroom.Awake}
+ b.Headroom = &boxchoice.Headroom{MemFreeMiB: f.Headroom.MemFreeMiB, DiskFreeMiB: f.Headroom.DiskFreeMiB,
+ MemTotalMiB: f.Headroom.MemTotalMiB, DiskTotalMiB: f.Headroom.DiskTotalMiB, Awake: f.Headroom.Awake}
}
out.Boxes = append(out.Boxes, b)
}
diff --git a/internal/edgesfile/snapshot_headroom_test.go b/internal/edgesfile/snapshot_headroom_test.go
new file mode 100644
index 000000000..93764dc9b
--- /dev/null
+++ b/internal/edgesfile/snapshot_headroom_test.go
@@ -0,0 +1,25 @@
+package edgesfile
+
+import (
+ "encoding/json"
+ "testing"
+)
+
+// Region HA spec 2 §7: the placement policy reads the headroom totals and
+// the file's generation, so BoxChoice carries both.
+func TestBoxChoiceCarriesHeadroomTotalsAndGeneration(t *testing.T) {
+ var f PlacementView
+ body := `{"v":1,"box":"box-a","box_id":"i-a","generation":"2026-10-16T11:55:00Z","headroom":{"mem_free_mib":512,"disk_free_mib":2048,"mem_total_mib":4096,"disk_total_mib":20480,"awake":3},"hosts":{}}`
+ if err := json.Unmarshal([]byte(body), &f); err != nil {
+ t.Fatal(err)
+ }
+ f.Fresh = true
+ got := Snapshot{Files: []PlacementView{f}, Complete: true, BoxesComplete: true}.BoxChoice()
+ if len(got.Boxes) != 1 {
+ t.Fatalf("boxes %+v", got.Boxes)
+ }
+ b := got.Boxes[0]
+ if b.Generation != "2026-10-16T11:55:00Z" || b.Headroom == nil || b.Headroom.MemTotalMiB != 4096 || b.Headroom.DiskTotalMiB != 20480 || b.Headroom.MemFreeMiB != 512 || b.Headroom.DiskFreeMiB != 2048 || b.Headroom.Awake != 3 {
+ t.Fatalf("box lost a field: %+v %+v", b, b.Headroom)
+ }
+}
diff --git a/internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv b/internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv
index 34d954cab..7edff1385 100644
--- a/internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv
+++ b/internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv
@@ -1,5 +1,8 @@
# Plan 14 census overlay region-ha-2a (region HA spec 2, plan 2a: docs/superpowers/plans/2026-10-16-region-ha-2a-core.md), loaded last.
# Task 2: the new internal/lbhealth package (the load balancer health state). Envelope-only: no root-read unit changed or was added, nothing runs as root that did not before, and no registry, /data, release or claim read was added, so the legacy seal stands.
# Task 3: box and group roles in the signed roster (internal/hostactors roles.go; Box.Role and Box.Group, Region.Role, Replace lets the role change, automatic enrollment inherits the group's role). Envelope-only: Directory.Replace, Roster.Validate and Directory.enroll are not root-read units, no root-read unit changed or was added, and the legacy seal stands.
+# Task 4: the placement policy (internal/boxchoice policy.go; headroom totals and the edges file's generation carried through Eligible and edgesfile.Snapshot.BoxChoice). Envelope-only: no root-read unit changed or was added, and the legacy seal stands.
@coverage internal/lbhealth ad1211a01a4eb3d02cd8ed95cbff0f8b17b3976ce9c0279ebd2cae2f20c07124
@coverage internal/hostactors 6736698380a247764ada687f2476f89382fc32a1183a50c06afe75967d716eab
+@coverage internal/boxchoice 1bfd561bcb48b51e60e63f99e086fd44d3b1be2160e28b3ed0253a14bbb9ff66
+@coverage internal/edgesfile da42c799d62e065817b5cce4f9a3ead1308dbd379615a85a7b14533c16d7e804
diff --git a/region_ha_boundary_test.go b/region_ha_boundary_test.go
index 7f68ca3fc..6f0ad3804 100644
--- a/region_ha_boundary_test.go
+++ b/region_ha_boundary_test.go
@@ -21,6 +21,7 @@ import (
var regionHACoreFiles = []string{
"internal/lbhealth/lbhealth.go",
"internal/hostactors/roles.go",
+ "internal/boxchoice/policy.go",
}
// regionHAAdapterFiles are spec 2's AWS adapters. Each is named for AWS so