amadan

carlos / platform Public

region HA 2a: publish each region's eligible capacity

The console counts, once a minute, the placeable boxes of each region its
boxes publish, through the scoped placement chooser, and publishes the
counts to CloudWatch (Carlos/Region EligibleBoxes) in the AWS region the
signed roster names. A region whose read is incomplete or fails gets no
data point, never 0. The pass starts beside claim recovery, with region HA
on and CARLOS_REGION_METRICS=cloudwatch.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude 5c5f40d5b20edc6b4db82034444aaa1e91d1e16c parent 7722ad7
9 files changed, +449 −20
  • internal/console/awsregionmetric.go +70 −0
  • internal/console/hostclaimops.go +9 −0
  • internal/console/nocapacity_test.go +2 −2
  • internal/console/regioncapacity.go +127 −0
  • internal/console/regioncapacity_test.go +209 −0
  • internal/hostops/executor.go +17 −6
  • internal/hostops/executor_test.go +10 −10
  • internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv +3 −2
  • region_ha_boundary_test.go +2 −0
diff --git a/internal/console/awsregionmetric.go b/internal/console/awsregionmetric.go
new file mode 100644
index 000000000..077db6170
--- /dev/null
+++ b/internal/console/awsregionmetric.go
@@ -0,0 +1,70 @@
+package console
+
+import (
+ "context"
+ "errors"
+ "log"
+ "strings"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/service/cloudwatch"
+ "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types"
+
+ "github.com/carlosframework/platform/internal/rootaws"
+)
+
+// The AWS adapter of the eligible-capacity metric (region HA spec 2 §4,
+// §25): CloudWatch, Carlos/Region EligibleBoxes, keyed by the placement
+// code, in each region's own CloudWatch, where the region-host module's
+// alarm reads it. Armed by CARLOS_REGION_METRICS=cloudwatch on the
+// console. The platform box's role needs cloudwatch:PutMetricData on the
+// Carlos/Region namespace: plan 2c grants it.
+
+const (
+ capacityNamespace = "Carlos/Region"
+ capacityMetric = "EligibleBoxes"
+)
+
+// capacityData is the exact PutMetricData input for one region's count.
+func capacityData(c regionCapacity, now time.Time) *cloudwatch.PutMetricDataInput {
+ return &cloudwatch.PutMetricDataInput{Namespace: aws.String(capacityNamespace), MetricData: []types.MetricDatum{{
+ MetricName: aws.String(capacityMetric),
+ Dimensions: []types.Dimension{{Name: aws.String("Region"), Value: aws.String(c.Placement)}},
+ Timestamp: aws.Time(now),
+ Unit: types.StandardUnitCount,
+ Value: aws.Float64(float64(c.Eligible)),
+ }}}
+}
+
+// cloudWatchCapacity publishes each count to its region's CloudWatch.
+type cloudWatchCapacity struct {
+ put func(ctx context.Context, awsRegion string, in *cloudwatch.PutMetricDataInput) error
+ now func() time.Time
+}
+
+func (c cloudWatchCapacity) PublishCapacity(ctx context.Context, counts []regionCapacity) error {
+ var errs []error
+ for _, n := range counts {
+ errs = append(errs, c.put(ctx, n.AWSRegion, capacityData(n, c.now())))
+ }
+ return errors.Join(errs...)
+}
+
+// capacityPublisherFromEnv is the console's publisher: CloudWatch when
+// CARLOS_REGION_METRICS is "cloudwatch", and nil otherwise, or when the
+// AWS configuration cannot be loaded (logged).
+func capacityPublisherFromEnv(ctx context.Context, getenv func(string) string) capacityPublisher {
+ if strings.TrimSpace(getenv("CARLOS_REGION_METRICS")) != "cloudwatch" {
+ return nil
+ }
+ cfg, err := rootaws.LoadDefaultConfig(ctx)
+ if err != nil {
+ log.Printf("console: eligible-capacity metric disabled: load aws config: %v", err)
+ return nil
+ }
+ return cloudWatchCapacity{now: time.Now, put: func(ctx context.Context, awsRegion string, in *cloudwatch.PutMetricDataInput) error {
+ _, err := cloudwatch.NewFromConfig(cfg, func(o *cloudwatch.Options) { o.Region = awsRegion }).PutMetricData(ctx, in)
+ return err
+ }}
+}
diff --git a/internal/console/hostclaimops.go b/internal/console/hostclaimops.go
index 0f236058c..8e0a09f9b 100644
--- a/internal/console/hostclaimops.go
+++ b/internal/console/hostclaimops.go
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"log"
+ "os"
"time"
"github.com/carlosframework/platform/internal/hostclaim"
@@ -43,6 +44,14 @@ func (s *Server) StartClaimRecoveryPass(ctx context.Context) {
log.Printf("console: claim recovery: %v", hostops.ErrContract)
return
}
+ // The eligible-capacity metric (region HA spec 2 §4, regioncapacity.go)
+ // runs beside claim recovery, on a host-claim console with region HA
+ // on and a metrics adapter armed.
+ if s.cfg.RegionHA {
+ if pub := capacityPublisherFromEnv(ctx, os.Getenv); pub != nil {
+ go s.runCapacityPass(ctx, pub)
+ }
+ }
t := time.NewTicker(claimRecoveryInterval)
defer t.Stop()
for {
diff --git a/internal/console/nocapacity_test.go b/internal/console/nocapacity_test.go
index 9facac57d..ff4007b8a 100644
--- a/internal/console/nocapacity_test.go
+++ b/internal/console/nocapacity_test.go
@@ -532,7 +532,7 @@ func TestPlacementChooserIsScoped(t *testing.T) {
if x.Namespace == "" || x.Namespace != cx.srv.cfg.ClaimAuthority.Namespace {
t.Fatalf("pool %q: chooser namespace %q, want the authority's %q", pool, x.Namespace, cx.srv.cfg.ClaimAuthority.Namespace)
}
- n, err := x.Capacity(ctx, ie)
+ n, _, err := x.Capacity(ctx, ie)
if err != nil || n == 0 {
t.Fatalf("pool %q: capacity %d %v, want a placeable box", pool, n, err)
}
@@ -544,7 +544,7 @@ func TestPlacementChooserIsScoped(t *testing.T) {
t.Fatalf("pool %q: roomIn with a placeable box: %v", pool, err)
}
}
- if n, err := cx.srv.cfg.ClaimAuthority.Executors.Capacity(ctx, ie); err != nil || n != 0 {
+ if n, _, err := cx.srv.cfg.ClaimAuthority.Executors.Capacity(ctx, ie); err != nil || n != 0 {
t.Fatalf("the unscoped chooser counted %d (%v): the scope is not what makes the count", n, err)
}
cx.srv.cfg.RegionHA = false
diff --git a/internal/console/regioncapacity.go b/internal/console/regioncapacity.go
new file mode 100644
index 000000000..4076fd586
--- /dev/null
+++ b/internal/console/regioncapacity.go
@@ -0,0 +1,127 @@
+package console
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log"
+ "sort"
+ "time"
+
+ "github.com/carlosframework/platform/internal/boxchoice"
+ "github.com/carlosframework/platform/internal/hostops"
+)
+
+// The eligible-capacity metric (region HA spec 2 §4, A1-11): once a
+// minute the console counts, for each region its boxes publish, the boxes
+// that could take a new instance now (hostops.ExecutorChooser.Capacity:
+// enrolled, active, unfenced and placeable), and publishes the counts. An
+// alarm on the count catches what the cloud's own group metrics cannot: a
+// box that is in service but never enrolled, and a launch loop. Where the
+// counts go is an adapter's business (awsregionmetric.go); a console with
+// none publishes nothing.
+
+// capacityInterval is how often the counts are published.
+const capacityInterval = time.Minute
+
+// regionCapacity is one region's count. Placement is the CARLOS placement
+// code the metric is keyed by; AWSRegion is where its boxes run.
+type regionCapacity struct {
+ Placement, AWSRegion string
+ Eligible int
+}
+
+// capacityPublisher sends the counts to a metrics service.
+type capacityPublisher interface {
+ PublishCapacity(ctx context.Context, counts []regionCapacity) error
+}
+
+// regionCapacities counts each placement the snapshot's files name, sorted
+// by placement, and takes each one's AWS region from capacity's answer:
+// the signed roster's, never the files' own claim, so a stale or
+// misconfigured file naming another AWS region cannot send a region's
+// count elsewhere (Astra 2a round 2 finding 3). A placement the roster
+// does not know (no region entry, so no box there can be chosen) is left
+// out. A placement whose count fails, or whose read is incomplete
+// (hostops.ErrEvidenceIncomplete), is left out too and its error returned
+// with the counts that did succeed: publishing 0 for it would trip the
+// eligible-capacity alarm on a transient read failure, whereas a missing
+// data point is the alarm's own signal if the failure lasts (spec §4,
+// A2-8). Only a complete read that finds no placeable box is a 0.
+func regionCapacities(ctx context.Context, snap boxchoice.Snapshot, capacity func(context.Context, string) (int, string, error)) ([]regionCapacity, error) {
+ named := map[string]bool{}
+ for _, b := range snap.Boxes {
+ if b.Placement != "" {
+ named[b.Placement] = true
+ }
+ }
+ placements := make([]string, 0, len(named))
+ for p := range named {
+ placements = append(placements, p)
+ }
+ sort.Strings(placements)
+ var out []regionCapacity
+ var errs []error
+ for _, p := range placements {
+ n, aws, err := capacity(ctx, p)
+ if errors.Is(err, hostops.ErrContract) {
+ continue
+ }
+ if err != nil {
+ errs = append(errs, fmt.Errorf("count %s: %w", p, err))
+ continue
+ }
+ out = append(out, regionCapacity{Placement: p, AWSRegion: aws, Eligible: n})
+ }
+ return out, errors.Join(errs...)
+}
+
+// publishCapacity counts and publishes once.
+func (s *Server) publishCapacity(ctx context.Context, pub capacityPublisher) error {
+ x := s.placementChooser("")
+ if x == nil {
+ return hostops.ErrContract
+ }
+ snap, err := s.liveEdges(ctx)
+ if err != nil {
+ return err
+ }
+ counts, countErr := regionCapacities(ctx, boxSnapshot(snap), x.Capacity)
+ if len(counts) == 0 {
+ return countErr
+ }
+ // The regions that counted are published even when another failed:
+ // the failed one has no data point, which is the alarm's signal.
+ return errors.Join(pub.PublishCapacity(ctx, counts), countErr)
+}
+
+// runCapacityPass publishes every capacityInterval until ctx ends. A
+// failed pass is logged on its first failure and every tenth; the alarm
+// treats missing data as breaching, so a console that stops publishing is
+// itself an alert.
+func (s *Server) runCapacityPass(ctx context.Context, pub capacityPublisher) {
+ log.Printf("console: eligible-capacity metric armed (every %s)", capacityInterval)
+ t := time.NewTicker(capacityInterval)
+ defer t.Stop()
+ fails := 0
+ for {
+ pass, cancel := context.WithTimeout(ctx, capacityInterval)
+ err := s.publishCapacity(pass, pub)
+ cancel()
+ switch {
+ case err != nil:
+ fails++
+ if fails == 1 || fails%10 == 0 {
+ log.Printf("console: eligible-capacity metric (failure #%d): %v", fails, err)
+ }
+ case fails > 0:
+ log.Printf("console: eligible-capacity metric recovered after %d failure(s)", fails)
+ fails = 0
+ }
+ select {
+ case <-ctx.Done():
+ return
+ case <-t.C:
+ }
+ }
+}
diff --git a/internal/console/regioncapacity_test.go b/internal/console/regioncapacity_test.go
new file mode 100644
index 000000000..d4c0b214d
--- /dev/null
+++ b/internal/console/regioncapacity_test.go
@@ -0,0 +1,209 @@
+package console
+
+import (
+ "context"
+ "errors"
+ "reflect"
+ "testing"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/service/cloudwatch"
+
+ "github.com/carlosframework/platform/internal/boxchoice"
+ "github.com/carlosframework/platform/internal/hostops"
+)
+
+// The count runs without any cloud adapter (§25): it reads only the
+// edges snapshot and the chooser. Each region's AWS region is the
+// chooser's (the signed roster's), whatever a file claims: box z's file
+// names uk in eu-west-1.
+func TestRegionCapacitiesCountsEachPlacement(t *testing.T) {
+ snap := boxchoice.Snapshot{Complete: true, Boxes: []boxchoice.Box{
+ {BoxID: "a", Placement: "uk", Region: "eu-west-2"},
+ {BoxID: "b", Placement: "uk", Region: "eu-west-2"},
+ {BoxID: "c", Placement: "ie", Region: "eu-west-1"},
+ {BoxID: "d", Placement: "fleet/pi", Region: "eu-west-1"},
+ {BoxID: "e", Placement: ""},
+ {BoxID: "z", Placement: "uk", Region: "eu-west-1"},
+ }}
+ asked := []string{}
+ capacity := func(_ context.Context, p string) (int, string, error) {
+ asked = append(asked, p)
+ switch p {
+ case "uk":
+ return 2, "eu-west-2", nil
+ case "ie":
+ return 0, "eu-west-1", nil
+ }
+ return 0, "", hostops.ErrContract // no roster region: left out
+ }
+ got, err := regionCapacities(context.Background(), snap, capacity)
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := []regionCapacity{{Placement: "ie", AWSRegion: "eu-west-1", Eligible: 0}, {Placement: "uk", AWSRegion: "eu-west-2", Eligible: 2}}
+ if !reflect.DeepEqual(got, want) || !reflect.DeepEqual(asked, []string{"fleet/pi", "ie", "uk"}) {
+ t.Fatalf("got %+v asked %v", got, asked)
+ }
+}
+
+// Controller ruling: a region whose count fails, or whose read is
+// incomplete, gets no data point (never a 0, which would trip the alarm on
+// a transient read failure; missing data is the alarm's own signal if it
+// lasts). The other regions' counts are unaffected, and the error is
+// returned so the pass logs it.
+func TestRegionCapacitiesOmitsARegionWhoseReadFailed(t *testing.T) {
+ snap := boxchoice.Snapshot{Complete: true, Boxes: []boxchoice.Box{
+ {BoxID: "a", Placement: "ie"}, {BoxID: "b", Placement: "uk"}, {BoxID: "c", Placement: "us"},
+ }}
+ for name, failure := range map[string]error{
+ "incomplete": hostops.ErrEvidenceIncomplete,
+ "other": errors.New("SlowDown"),
+ } {
+ capacity := func(_ context.Context, p string) (int, string, error) {
+ if p == "uk" {
+ return 0, "", failure
+ }
+ return 1, "eu-" + p, nil
+ }
+ got, err := regionCapacities(context.Background(), snap, capacity)
+ want := []regionCapacity{{Placement: "ie", AWSRegion: "eu-ie", Eligible: 1}, {Placement: "us", AWSRegion: "eu-us", Eligible: 1}}
+ if !errors.Is(err, failure) || !reflect.DeepEqual(got, want) {
+ t.Fatalf("%s: got %+v, %v; want %+v and the failure", name, got, err, want)
+ }
+ }
+}
+
+// capacityRecorder keeps what one pass would publish.
+type capacityRecorder struct {
+ got []regionCapacity
+ calls int
+}
+
+func (c *capacityRecorder) PublishCapacity(_ context.Context, counts []regionCapacity) error {
+ c.got = counts
+ c.calls++
+ return nil
+}
+
+// Astra 2a round 2 finding 3, end to end: a stray box's edges file naming
+// ie in another AWS region (and sorting after the real box) does not move
+// ie's count out of the roster's eu-west-1.
+func TestPublishCapacityTakesTheRegionFromTheRoster(t *testing.T) {
+ cx := haClaimCase(t, "")
+ publishHABox(t, cx, "box-ie-default", false, nil)
+ publishHABox(t, cx, "box-ie-stray", false, func(m map[string]any) { m["region"] = "eu-west-2" })
+ var pub capacityRecorder
+ if err := cx.srv.publishCapacity(context.Background(), &pub); err != nil {
+ t.Fatal(err)
+ }
+ want := []regionCapacity{{Placement: "ie", AWSRegion: "eu-west-1", Eligible: 1}}
+ if !reflect.DeepEqual(pub.got, want) {
+ t.Fatalf("published %+v, want %+v", pub.got, want)
+ }
+}
+
+// Controller ruling 1: the pass counts through the scoped chooser, so a
+// region with a placeable box publishes a non-zero count (an unscoped
+// chooser answers 0 for every region).
+func TestPublishCapacityCountsPlaceableBoxes(t *testing.T) {
+ cx := haClaimCase(t, "")
+ publishHABox(t, cx, "box-ie-default", false, nil)
+ var pub capacityRecorder
+ if err := cx.srv.publishCapacity(context.Background(), &pub); err != nil {
+ t.Fatal(err)
+ }
+ if len(pub.got) != 1 || pub.got[0].Placement != "ie" || pub.got[0].Eligible != 1 {
+ t.Fatalf("published %+v, want ie with 1", pub.got)
+ }
+}
+
+// Controller ruling 2: an incomplete read publishes nothing for the
+// region (not 0), while a complete read with no placeable box publishes 0.
+func TestPublishCapacityIncompleteReadPublishesNothingButNoRoomPublishesZero(t *testing.T) {
+ ctx := context.Background()
+ cx := haClaimCase(t, "")
+ publishHABox(t, cx, "box-ie-default", true, nil) // stale: named, not placeable
+ var pub capacityRecorder
+ if err := cx.srv.publishCapacity(ctx, &pub); err != nil {
+ t.Fatal(err)
+ }
+ want := []regionCapacity{{Placement: "ie", AWSRegion: "eu-west-1", Eligible: 0}}
+ if pub.calls != 1 || !reflect.DeepEqual(pub.got, want) {
+ t.Fatalf("a complete read with no placeable box published %+v (%d calls), want %+v", pub.got, pub.calls, want)
+ }
+
+ publishHABox(t, cx, "box-ie-default", false, nil)
+ pub = capacityRecorder{}
+ edges := cx.srv.cfg.ClaimAuthority.Executors.Edges
+ cx.srv.cfg.ClaimAuthority.Executors.Edges = func(ctx context.Context) (boxchoice.Snapshot, error) {
+ snap, err := edges(ctx)
+ snap.Complete = false
+ return snap, err
+ }
+ err := cx.srv.publishCapacity(ctx, &pub)
+ if !errors.Is(err, hostops.ErrEvidenceIncomplete) {
+ t.Fatalf("an incomplete read: %v, want ErrEvidenceIncomplete", err)
+ }
+ if pub.calls != 0 || pub.got != nil {
+ t.Fatalf("an incomplete read published %+v (%d calls), want nothing", pub.got, pub.calls)
+ }
+}
+
+// The same, with the only box's own file naming another AWS region: the
+// chooser does not count a box whose file disagrees with the roster, and
+// the 0 still goes to the roster's eu-west-1, not the file's eu-west-2.
+func TestPublishCapacityIgnoresTheBoxsOwnAWSRegion(t *testing.T) {
+ cx := haClaimCase(t, "")
+ publishHABox(t, cx, "box-ie-default", false, func(m map[string]any) { m["region"] = "eu-west-2" })
+ var pub capacityRecorder
+ if err := cx.srv.publishCapacity(context.Background(), &pub); err != nil {
+ t.Fatal(err)
+ }
+ want := []regionCapacity{{Placement: "ie", AWSRegion: "eu-west-1", Eligible: 0}}
+ if !reflect.DeepEqual(pub.got, want) {
+ t.Fatalf("published %+v, want %+v", pub.got, want)
+ }
+}
+
+func TestPlacementChooserIsScopedLikeCreate(t *testing.T) {
+ s := &Server{cfg: Config{RegionHA: true}}
+ if s.placementChooser("") != nil {
+ t.Fatal("a chooser with no claim runtime")
+ }
+ x := &hostops.ExecutorChooser{Namespace: "stale", Pool: "acme"}
+ s.cfg.ClaimAuthority = &hostops.ProductionAuthority{Namespace: "platform", Executors: x}
+ got := s.placementChooser("")
+ if got == nil || got.Namespace != "platform" || got.Pool != "" || x.Namespace != "stale" {
+ t.Fatalf("chooser %+v, original %+v", got, x)
+ }
+ s.cfg.RegionHA = false
+ if s.placementChooser("") != nil {
+ t.Fatal("a chooser with region HA off")
+ }
+}
+
+func TestCloudWatchCapacityPublishesEachRegionInItsOwnRegion(t *testing.T) {
+ now := time.Date(2026, 10, 16, 12, 0, 0, 0, time.UTC)
+ type call struct {
+ region, ns, metric, dim string
+ value float64
+ }
+ var calls []call
+ c := cloudWatchCapacity{now: func() time.Time { return now }, put: func(_ context.Context, region string, in *cloudwatch.PutMetricDataInput) error {
+ d := in.MetricData[0]
+ calls = append(calls, call{region, *in.Namespace, *d.MetricName, *d.Dimensions[0].Name + "=" + *d.Dimensions[0].Value, *d.Value})
+ if !d.Timestamp.Equal(now) || d.Unit != "Count" {
+ t.Fatalf("datum %+v", d)
+ }
+ return nil
+ }}
+ err := c.PublishCapacity(context.Background(), []regionCapacity{{"ie", "eu-west-1", 2}, {"uk", "eu-west-2", 0}})
+ want := []call{{"eu-west-1", "Carlos/Region", "EligibleBoxes", "Region=ie", 2}, {"eu-west-2", "Carlos/Region", "EligibleBoxes", "Region=uk", 0}}
+ if err != nil || !reflect.DeepEqual(calls, want) {
+ t.Fatalf("calls %+v %v", calls, err)
+ }
+ if capacityPublisherFromEnv(context.Background(), func(string) string { return "" }) != nil {
+ t.Fatal("published with CARLOS_REGION_METRICS unset")
+ }
+}
diff --git a/internal/hostops/executor.go b/internal/hostops/executor.go
index 228f38098..ffd4cc752 100644
--- a/internal/hostops/executor.go
+++ b/internal/hostops/executor.go
@@ -113,12 +113,23 @@ func (x ExecutorChooser) PlaceableBoxes(ctx context.Context, placement string) (
return boxes, nil
}
-// Capacity is how many boxes PlaceableBoxes names. The console publishes
-// it per region as the eligible-capacity metric (region HA spec 2 §4,
-// A1-11).
-func (x ExecutorChooser) Capacity(ctx context.Context, placement string) (int, error) {
- boxes, err := x.PlaceableBoxes(ctx, placement)
- return len(boxes), err
+// Capacity is how many boxes PlaceableBoxes names, and the AWS region the
+// signed roster names for placement. The console publishes the count per
+// region as the eligible-capacity metric (region HA spec 2 §4, A1-11), in
+// that AWS region: the roster's mapping, the one eligible reads, never an
+// edges file's own claim (Astra 2a round 2 finding 3). An incomplete read
+// is ErrEvidenceIncomplete, never a count of 0.
+func (x ExecutorChooser) Capacity(ctx context.Context, placement string) (int, string, error) {
+ r := Record{Holder: hostclaim.NewHolder{Region: placement}}
+ o, cands, err := x.judged(ctx, hostclaim.Observed{Absent: true}, r)
+ if err != nil {
+ return 0, "", err
+ }
+ // eligible's own home mapping, so the region is the one it read.
+ if placement == park.HomeRegion && x.HomeName != "" && park.PlacementRegion(x.HomeName) == x.HomeName {
+ placement = x.HomeName
+ }
+ return len(x.placeable(o, cands)), o.Roster.Regions[placement].AWSRegion, nil
}
// judged is eligible's read, refused with ErrEvidenceIncomplete when the
diff --git a/internal/hostops/executor_test.go b/internal/hostops/executor_test.go
index 890e93d37..a2d697a26 100644
--- a/internal/hostops/executor_test.go
+++ b/internal/hostops/executor_test.go
@@ -150,8 +150,8 @@ func TestExecutorChooserAppliesRolesAndHeadroom(t *testing.T) {
if err := x.Permits(ctx, absent, chooserRecord(), "box-b"); err != nil {
t.Fatalf("Permits(box-b) = %v", err)
}
- if n, err := x.Capacity(ctx, "ie"); err != nil || n != 1 {
- t.Fatalf("Capacity = %d, %v; want 1", n, err)
+ if n, aws, err := x.Capacity(ctx, "ie"); err != nil || n != 1 || aws != "eu-west-1" {
+ t.Fatalf("Capacity = %d in %q, %v; want 1 in eu-west-1", n, aws, err)
}
if boxes, err := x.PlaceableBoxes(ctx, "ie"); err != nil || len(boxes) != 1 || boxes[0] != "box-b" {
t.Fatalf("PlaceableBoxes = %v, %v; want [box-b]", boxes, err)
@@ -159,10 +159,10 @@ func TestExecutorChooserAppliesRolesAndHeadroom(t *testing.T) {
})
}
x, _, _, _ := chooserFixture(t, "box-a", "box-b")
- if n, err := x.Capacity(ctx, "ie"); err != nil || n != 2 {
- t.Fatalf("Capacity with both placeable = %d, %v; want 2", n, err)
+ if n, aws, err := x.Capacity(ctx, "ie"); err != nil || n != 2 || aws != "eu-west-1" {
+ t.Fatalf("Capacity with both placeable = %d in %q, %v; want 2 in eu-west-1", n, aws, err)
}
- if _, err := x.Capacity(ctx, "us"); !errors.Is(err, ErrContract) {
+ if _, _, err := x.Capacity(ctx, "us"); !errors.Is(err, ErrContract) {
t.Fatalf("Capacity of a region the roster lacks = %v, want ErrContract", err)
}
}
@@ -546,7 +546,7 @@ func TestExecutorChooserAnIncompleteReadIsNotNoRoom(t *testing.T) {
if boxes != nil {
t.Fatalf("PlaceableBoxes named %v from a partial read", boxes)
}
- _, err = x.Capacity(ctx, "ie")
+ _, _, err = x.Capacity(ctx, "ie")
incomplete("Capacity", err)
_, _, err = (ProductionAuthority{Namespace: "platform", Executors: &x}).SelectExecutor(ctx, absent, chooserRecord())
incomplete("SelectExecutor", err)
@@ -558,15 +558,15 @@ func TestExecutorChooserAnIncompleteReadIsNotNoRoom(t *testing.T) {
if err := x.Permits(ctx, absent, chooserRecord(), "box-a"); err != nil {
t.Fatalf("Permits on a complete read: %v", err)
}
- if n, err := x.Capacity(ctx, "ie"); err != nil || n != 1 {
- t.Fatalf("Capacity on a complete read: %d %v", n, err)
+ if n, aws, err := x.Capacity(ctx, "ie"); err != nil || n != 1 || aws != "eu-west-1" {
+ t.Fatalf("Capacity on a complete read: %d in %q %v", n, aws, err)
}
// A complete read with no placeable box is still no room.
snap.Boxes[0].Fresh = false
if _, err := x.Choose(ctx, absent, chooserRecord()); !errors.Is(err, ErrNoEligibleBox) {
t.Fatalf("Choose with every box stale: %v, want ErrNoEligibleBox", err)
}
- if n, err := x.Capacity(ctx, "ie"); err != nil || n != 0 {
- t.Fatalf("Capacity with every box stale: %d %v", n, err)
+ if n, aws, err := x.Capacity(ctx, "ie"); err != nil || n != 0 || aws != "eu-west-1" {
+ t.Fatalf("Capacity with every box stale: %d in %q %v", n, aws, err)
}
}
diff --git a/internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv b/internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv
index fb4c940d5..ae8566fbf 100644
--- a/internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv
+++ b/internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv
@@ -12,17 +12,18 @@
# Task 11: H27 on the create doors (internal/console nocapacity.go; hostops chosen.go): beginClaimCreate chooses the executor before it registers anything (chooseExecutorFirst) and carries the choice to the create (hostops.WithChosenExecutor, honoured by ProductionAuthority.SelectExecutor), claimResponseRefusal answers no room with a 503 on instance create, domain attach's first route (attachClaimFirstRoute, unchanged) and standalone sidecar create, and handleAPIInstanceCreate judges an app-derived region by the chooser's eligible set. handleAPIInstanceCreate chooses before its first write, the environment binding (chooseCreateFirst), and beginClaimCreate keeps that choice; with region HA on, an explicit region with no box that has room is the no-room refusal before the armed-placement gate (roomIn). Census ruling region-ha-2a covers two accepted-legacy units: Server.beginClaimCreate re-frozen (one call added before RegisterStart, its one root-read site, the claim read file:Read@470, unchanged), and Server.chooseCreateFirst, new: the same claim read, made before the binding, feeding only the placement choice; it returns its choice with the claim observation it decided on, and beginClaimCreate keeps the choice only if its own read is that observation (ruling item 5). The seal moves.
# Task 11 fix round 1: an incomplete placement read is not "no room" (controller ruling). hostops.ExecutorChooser's Choose, Permits and PlaceableBoxes ask judged, which wraps the chooser's own Edges read and answers ErrEvidenceIncomplete when that snapshot is incomplete; ExecutorChooser.eligible (accepted-legacy, merge ruling 2) is unchanged. The no-capacity alert is recorded only once sent (noCapacityAlerts.failed gives a failed send's slot back), and the no-room 503 carries Retry-After: 60 (setNoCapacityRetryAfter in writeClaimResponseError, renderDoorRefusal and handleAPIDomainAttach's refusal write). Envelope-only: no root-read unit changed or was added, and the legacy seal stands.
# Task 12: H27 on the move door's destination, through hostops.ExecutorChooser.PlaceableBoxes and only with region HA on: roomIn refuses a destination with no box that has room before the placement checks (no edges file, blank public addresses), and moveRoomCheck (internal/console boxselect.go) refuses when none of selectMoveBoxes' destination candidates is placeable. Both only refuse: the candidates and the destination are main's. An incomplete placement read (hostops.ErrEvidenceIncomplete) is the temporarily unavailable 503, never no room. Census ruling region-ha-2a is asked to re-freeze one accepted-legacy unit, Server.handleAPIInstanceMoveRequest: two refusals added before any write; its read sites unchanged, offsets moved. No write site is added or changed. The seal moves.
+# Task 13: the eligible-capacity metric (internal/console regioncapacity.go, counted through hostops.ExecutorChooser.Capacity, which now also answers the AWS region the signed roster names for the placement; awsregionmetric.go, the CloudWatch adapter), started beside claim recovery (StartClaimRecoveryPass). No root-read unit changed or was added: Capacity reads through judged and eligible, both unchanged by this task. A region whose read is incomplete or fails publishes no data point, never 0. Envelope-only, and the seal stands.
internal/activator/activator.go type Service authority B,C,D,F,J4 rev3=PROPOSED-seed; existing activator lifecycle and due-wake state plus owned seed exclusion and incarnation-keyed diagnostics; normal refusal and preparation ordering remain authoritative be4ca3a3ea2bd768e7e52c1056513d4fef1c1702d2e04656c540cdcbb858865c
@coverage internal/lbhealth ad1211a01a4eb3d02cd8ed95cbff0f8b17b3976ce9c0279ebd2cae2f20c07124
@coverage internal/hostactors 6736698380a247764ada687f2476f89382fc32a1183a50c06afe75967d716eab
@coverage internal/boxchoice 1bfd561bcb48b51e60e63f99e086fd44d3b1be2160e28b3ed0253a14bbb9ff66
@coverage internal/edgesfile f62c3973bb48fa36f561d26f496c0ba9a14fe882fe5843867e9f7a9eaf0abbdf
@coverage internal/host 4ad9bdcd8b94516a6ccdad3bc900b166833faf1f0334ac83478be5b04de47069
-@coverage internal/hostops 116489befd32c2afed09a4de9c60dd56cc3f78089b7752670faeb388a38e3645
+@coverage internal/hostops 0f8a9f4787c0438eb93bde1cb6369c40b89115fed460fd3be959b6128fce02a0
@coverage internal/edge 20177c9e7dc961937677bbf2763c73004d7c95f68512591800f6fe841c36152f
internal/host/edgesfile.go edgesFileWriteClaims authority F,J1 root-route;kept=F;scope=protected-decision-only registry:Account@10835,registry:Alias@5297,registry:Alias@5332,registry:Alias@5754,registry:Alias@5911,registry:Alias@5981,registry:Alias@6013,registry:Alias@6482,registry:Alias@8772,registry:App@10847,registry:DNSStatus@5429,registry:DNSStatus@5920,registry:Host@10480,registry:Host@2878,registry:Host@5389,registry:Host@6234,registry:Host@6567,registry:Host@8819,registry:Host@8866,registry:Host@8916,registry:Host@8951,registry:Host@9000,registry:Platform@2921,registry:Provisioned@2935,registry:Replica@2959,registry:Status@877,whole:Route@3212,whole:Route@323,whole:Route@3505,whole:Route@351,whole:RouteAlias@5282,whole:RouteAlias@8725,whole:RoutePath@3520,whole:WildcardDomain@8905 8c7cdffba66886b92f9a2ac663a86dbdd5f80699afd2e6afb4571da916c727ad
@coverage internal/activator fd9cf14712f55895541bb8bfbf8137cfd446298bcaa8634b0aba7b8344c29ec3
-@coverage internal/console e80945195061ccda68851a1fb043b2624d243b62a2f85f4175485e58a3f7d747
+@coverage internal/console 33b2399507ea5e8860614c42b25ad5d4976c259b392eb5923bc88722cc72daa8
@coverage . 0e6e4542758248000de1882a01f13e3449c880b4ae23e8b2ff0d6dce076b614c
@legacy b02b534b6617f57381ba12e9d9a2bf1bd63f737dc68005de926ff18b7db93dd3 internal/console/apiboxroles.go Server.boxRoster legacy accepted-legacy accepted-legacy;rev3=J4;basis=root-input;census-ruling-region-ha-2a file:Read@484 9be9925de14dbbd7d86123bf9faf1535c7b116575cadcb8ba983a665af0513f4
internal/console/apiboxroles.go Server.boxRoster legacy accepted-legacy accepted-legacy;rev3=J4;basis=root-input;census-ruling-region-ha-2a file:Read@484 9be9925de14dbbd7d86123bf9faf1535c7b116575cadcb8ba983a665af0513f4
diff --git a/region_ha_boundary_test.go b/region_ha_boundary_test.go
index c2668a896..c17537ff6 100644
--- a/region_ha_boundary_test.go
+++ b/region_ha_boundary_test.go
@@ -29,12 +29,14 @@ var regionHACoreFiles = []string{
"system_roles.go",
"internal/console/nocapacity.go",
"internal/hostops/chosen.go",
+ "internal/console/regioncapacity.go",
}
// regionHAAdapterFiles are spec 2's AWS adapters. Each is named for AWS so
// a reader can see where the provider lives.
var regionHAAdapterFiles = []string{
"internal/host/awsaddr.go",
+ "internal/console/awsregionmetric.go",
}
// regionHAPackages are the packages spec 2 touches. Every S3 client built