amadan

carlos / platform Public

region HA 2a: the load balancer health state and the provider boundary tests

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude c3783b1a48735a3f15d9c9748724ed6948de8fdc parent 28fa379
5 files changed, +528 −0
  • internal/lbhealth/lbhealth.go +200 −0
  • internal/lbhealth/lbhealth_test.go +133 −0
  • internal/provision/plan14_root_reads_test.go +11 −0
  • internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv +3 −0
  • region_ha_boundary_test.go +181 −0
diff --git a/internal/lbhealth/lbhealth.go b/internal/lbhealth/lbhealth.go
new file mode 100644
index 000000000..af3260bf2
--- /dev/null
+++ b/internal/lbhealth/lbhealth.go
@@ -0,0 +1,200 @@
+// Package lbhealth is a box's own liveness, as the load balancer's health
+// check reads it (region HA spec 2 §6). Every fact here is this box's own:
+// a flag or a timestamp that one of its own loops sets. Nothing here reads
+// a disk, a bucket or the network, so an outage of something the boxes
+// share (the deployment bucket) never turns a working box unhealthy and
+// never gets it terminated.
+//
+// The edge answers GET or HEAD /.carlos/lb-health on port 80 from State.
+// The loops that feed it are the agent's: the claim agent sets hosting,
+// the home map latches forwarding and stamps its refresh attempts, the
+// certificate mirror latches its first load, and the edge's registry watch
+// and the activator's sweeper each beat.
+package lbhealth
+
+import (
+ "io"
+ "net/http"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+)
+
+// Path is the health check's path, matched exactly.
+const Path = "/.carlos/lb-health"
+
+// HeartbeatWindow is how recent the edge's and the activator's beats must
+// be. Both loops beat every 30 seconds or faster.
+const HeartbeatWindow = 60 * time.Second
+
+// RefreshWindow is how recent the home map's last completed refresh
+// attempt must be: its 30-second interval plus its two-minute request
+// timeout, with room to spare. A failed attempt counts, so a bucket outage
+// where every attempt fails fast keeps this satisfied; a worker stuck
+// inside one attempt does not.
+const RefreshWindow = 5 * time.Minute
+
+// The condition names a 503 lists, in this order.
+const (
+ CondHosting = "hosting"
+ CondForwarding = "forwarding"
+ CondCerts = "certificates"
+ CondEdge = "edge"
+ CondActivator = "activator"
+ CondRefresh = "refresh"
+)
+
+// State is one box's liveness facts. The zero value is a box that has
+// proven nothing, and answers 503. A nil *State ignores every update.
+type State struct {
+ hosting atomic.Bool
+ mapLoaded atomic.Bool
+ keyLoaded atomic.Bool
+ certs atomic.Bool
+ edgeBeat atomic.Int64 // unix nanoseconds; 0 is never
+ refreshed atomic.Int64 // unix nanoseconds; 0 is never
+
+ mu sync.RWMutex
+ activator func() time.Time
+}
+
+// Default is the process's state. The agent runs the edge and the
+// activator in one process, so the claim wiring, the edge and the
+// activator share it.
+var Default = &State{}
+
+// SetHosting records whether the agent is in hosting mode: enrolled, with
+// its guard armed (§5). Region HA spec 2's plan 2b adds joining mode, whose
+// mode selection is what sets this false; until then the claim agent sets
+// it true once its claim wiring is running (ClaimWiring.startRegional).
+func (s *State) SetHosting(on bool) {
+ if s != nil {
+ s.hosting.Store(on)
+ }
+}
+
+// Hosting reports what SetHosting last recorded.
+func (s *State) Hosting() bool { return s != nil && s.hosting.Load() }
+
+// MapLoaded latches the first successful load of the forwarding map.
+func (s *State) MapLoaded() {
+ if s != nil {
+ s.mapLoaded.Store(true)
+ }
+}
+
+// KeyLoaded latches the first successful load of a mesh key.
+func (s *State) KeyLoaded() {
+ if s != nil {
+ s.keyLoaded.Store(true)
+ }
+}
+
+// CertsLoaded latches the first successful load of the certificate mirror
+// into the local cache the TLS handshake reads.
+func (s *State) CertsLoaded() {
+ if s != nil {
+ s.certs.Store(true)
+ }
+}
+
+// EdgeBeat records that the edge's registry watch completed a poll at t.
+func (s *State) EdgeBeat(t time.Time) {
+ if s != nil {
+ s.edgeBeat.Store(t.UnixNano())
+ }
+}
+
+// RefreshAttempted records that the home map's refresh finished an
+// attempt at t, whether it succeeded or failed.
+func (s *State) RefreshAttempted(t time.Time) {
+ if s != nil {
+ s.refreshed.Store(t.UnixNano())
+ }
+}
+
+// SetActivator gives the state the activator's own beat: the time its
+// sweeper last finished a sweep. nil means there is no activator, which
+// fails the activator condition.
+func (s *State) SetActivator(last func() time.Time) {
+ if s == nil {
+ return
+ }
+ s.mu.Lock()
+ s.activator = last
+ s.mu.Unlock()
+}
+
+func (s *State) activatorBeat() time.Time {
+ s.mu.RLock()
+ last := s.activator
+ s.mu.RUnlock()
+ if last == nil {
+ return time.Time{}
+ }
+ return last()
+}
+
+// within reports whether at is set and no older than window at now.
+func within(at, now time.Time, window time.Duration) bool {
+ return !at.IsZero() && now.Sub(at) <= window
+}
+
+func stamp(n int64) time.Time {
+ if n == 0 {
+ return time.Time{}
+ }
+ return time.Unix(0, n)
+}
+
+// Failing lists the conditions that do not hold at now, in the order of
+// the Cond names. An empty list is a healthy box. A nil *State fails
+// every condition.
+func (s *State) Failing(now time.Time) []string {
+ if s == nil {
+ return []string{CondHosting, CondForwarding, CondCerts, CondEdge, CondActivator, CondRefresh}
+ }
+ var out []string
+ if !s.hosting.Load() {
+ out = append(out, CondHosting)
+ }
+ if !s.mapLoaded.Load() || !s.keyLoaded.Load() {
+ out = append(out, CondForwarding)
+ }
+ if !s.certs.Load() {
+ out = append(out, CondCerts)
+ }
+ if !within(stamp(s.edgeBeat.Load()), now, HeartbeatWindow) {
+ out = append(out, CondEdge)
+ }
+ if !within(s.activatorBeat(), now, HeartbeatWindow) {
+ out = append(out, CondActivator)
+ }
+ if !within(stamp(s.refreshed.Load()), now, RefreshWindow) {
+ out = append(out, CondRefresh)
+ }
+ return out
+}
+
+// Serve answers the health check when r is one, and reports whether it
+// did. It matches as narrowly as the steering probe: the exact path, GET
+// or HEAD, any host. It answers 200 "ok" when every condition holds and
+// 503 naming the failing conditions otherwise, never cached. It only reads
+// s.
+func Serve(w http.ResponseWriter, r *http.Request, s *State, now time.Time) bool {
+ if r.URL.Path != Path || (r.Method != http.MethodGet && r.Method != http.MethodHead) {
+ return false
+ }
+ body, code := "ok\n", http.StatusOK
+ if failing := s.Failing(now); len(failing) > 0 {
+ body, code = "unavailable: "+strings.Join(failing, ",")+"\n", http.StatusServiceUnavailable
+ }
+ w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+ w.Header().Set("Cache-Control", "no-store")
+ w.WriteHeader(code)
+ if r.Method == http.MethodGet {
+ _, _ = io.WriteString(w, body)
+ }
+ return true
+}
diff --git a/internal/lbhealth/lbhealth_test.go b/internal/lbhealth/lbhealth_test.go
new file mode 100644
index 000000000..9bf211ec9
--- /dev/null
+++ b/internal/lbhealth/lbhealth_test.go
@@ -0,0 +1,133 @@
+package lbhealth
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "reflect"
+ "strings"
+ "testing"
+ "time"
+)
+
+var t0 = time.Date(2026, 10, 16, 12, 0, 0, 0, time.UTC)
+
+// healthy is a state with every condition met at t0.
+func healthy() *State {
+ s := &State{}
+ s.SetHosting(true)
+ s.MapLoaded()
+ s.KeyLoaded()
+ s.CertsLoaded()
+ s.EdgeBeat(t0)
+ s.RefreshAttempted(t0)
+ s.SetActivator(func() time.Time { return t0 })
+ return s
+}
+
+func TestFailingNamesEachCondition(t *testing.T) {
+ if got := healthy().Failing(t0); len(got) != 0 {
+ t.Fatalf("healthy box fails %v", got)
+ }
+ if got := (&State{}).Failing(t0); !reflect.DeepEqual(got, []string{CondHosting, CondForwarding, CondCerts, CondEdge, CondActivator, CondRefresh}) {
+ t.Fatalf("zero state fails %v", got)
+ }
+ var nilState *State
+ if got := nilState.Failing(t0); len(got) != 6 {
+ t.Fatalf("nil state fails %v", got)
+ }
+ cases := []struct {
+ name string
+ spoil func(*State)
+ now time.Time
+ want string
+ }{
+ {"joining", func(s *State) { s.SetHosting(false) }, t0, CondHosting},
+ {"map never loaded", func(s *State) { s.mapLoaded.Store(false) }, t0, CondForwarding},
+ {"key never loaded", func(s *State) { s.keyLoaded.Store(false) }, t0, CondForwarding},
+ {"no certificate cache", func(s *State) { s.certs.Store(false) }, t0, CondCerts},
+ {"edge beat stale", func(s *State) { s.EdgeBeat(t0.Add(-61 * time.Second)) }, t0, CondEdge},
+ {"activator beat stale", func(s *State) { s.SetActivator(func() time.Time { return t0.Add(-61 * time.Second) }) }, t0, CondActivator},
+ {"no activator", func(s *State) { s.SetActivator(nil) }, t0, CondActivator},
+ {"refresh stuck", func(s *State) { s.RefreshAttempted(t0.Add(-5*time.Minute - time.Second)) }, t0, CondRefresh},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ s := healthy()
+ c.spoil(s)
+ if got := s.Failing(c.now); !reflect.DeepEqual(got, []string{c.want}) {
+ t.Fatalf("Failing = %v, want [%s]", got, c.want)
+ }
+ })
+ }
+ // The windows are inclusive: a beat exactly 60s old and a refresh
+ // exactly 5 minutes old still count.
+ s := healthy()
+ s.EdgeBeat(t0.Add(-HeartbeatWindow))
+ s.RefreshAttempted(t0.Add(-RefreshWindow))
+ if got := s.Failing(t0); len(got) != 0 {
+ t.Fatalf("boundary fails %v", got)
+ }
+}
+
+// A bucket outage after arming: refreshes keep failing, but each failed
+// attempt is still a completed attempt, and the latches hold. The box
+// stays healthy (A2-3, A3-3).
+func TestBucketOutageAfterArmingStaysHealthy(t *testing.T) {
+ s := healthy()
+ for i := 1; i <= 20; i++ {
+ now := t0.Add(time.Duration(i) * 30 * time.Second)
+ s.RefreshAttempted(now) // a failed attempt: no latch is re-set
+ s.EdgeBeat(now)
+ s.SetActivator(func() time.Time { return now })
+ if got := s.Failing(now); len(got) != 0 {
+ t.Fatalf("after %d failed refreshes: %v", i, got)
+ }
+ }
+}
+
+func TestServe(t *testing.T) {
+ get := func(s *State, method, target, host string) (*httptest.ResponseRecorder, bool) {
+ w := httptest.NewRecorder()
+ r := httptest.NewRequest(method, target, nil)
+ r.Host = host
+ return w, Serve(w, r, s, t0)
+ }
+ w, ok := get(healthy(), http.MethodGet, Path, "10.0.1.7")
+ if !ok || w.Code != http.StatusOK || w.Body.String() != "ok\n" || w.Header().Get("Cache-Control") != "no-store" {
+ t.Fatalf("healthy GET: ok=%v %d %q %v", ok, w.Code, w.Body.String(), w.Header())
+ }
+ w, ok = get(healthy(), http.MethodHead, Path, "any.example.com")
+ if !ok || w.Code != http.StatusOK || w.Body.Len() != 0 {
+ t.Fatalf("healthy HEAD: ok=%v %d %q", ok, w.Code, w.Body.String())
+ }
+ joining := healthy()
+ joining.SetHosting(false)
+ w, ok = get(joining, http.MethodGet, Path, "10.0.1.7")
+ if !ok || w.Code != http.StatusServiceUnavailable || !strings.Contains(w.Body.String(), CondHosting) {
+ t.Fatalf("joining GET: ok=%v %d %q", ok, w.Code, w.Body.String())
+ }
+ for _, c := range []struct{ method, target string }{
+ {http.MethodPost, Path},
+ {http.MethodGet, Path + "/"},
+ {http.MethodGet, "/.carlos/lb-healthz"},
+ {http.MethodGet, "/"},
+ } {
+ if _, ok := get(healthy(), c.method, c.target, "10.0.1.7"); ok {
+ t.Fatalf("%s %s answered", c.method, c.target)
+ }
+ }
+}
+
+func TestNilStateIgnoresUpdates(t *testing.T) {
+ var s *State
+ s.SetHosting(true)
+ s.MapLoaded()
+ s.KeyLoaded()
+ s.CertsLoaded()
+ s.EdgeBeat(t0)
+ s.RefreshAttempted(t0)
+ s.SetActivator(func() time.Time { return t0 })
+ if s.Hosting() {
+ t.Fatal("nil state hosting")
+ }
+}
diff --git a/internal/provision/plan14_root_reads_test.go b/internal/provision/plan14_root_reads_test.go
index ba4764854..4c5bc4c89 100644
--- a/internal/provision/plan14_root_reads_test.go
+++ b/internal/provision/plan14_root_reads_test.go
@@ -787,6 +787,17 @@ func TestPlan14RootReadCensus(t *testing.T) {
t.Fatal(err)
}
seal = "5d1b9b6addf6c9964ed18b417ee5b3d0d49fd43e1acabb775e054bba4b6e314e"
+ if errs := checkRootReadLegacy(inv, seal); len(errs) != 0 {
+ t.Fatal(errs)
+ }
+ // Census overlay region-ha-2a (region HA spec 2, plan 2a), loaded last.
+ // Its header says what each plan task added. The accepted-legacy rows
+ // it adds are census ruling region-ha-2a's (platform-1e), and move the
+ // seal.
+ if err = loadRootReadInventory(filepath.Join(root, "internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv"), &inv, true); err != nil {
+ t.Fatal(err)
+ }
+ seal = "5d1b9b6addf6c9964ed18b417ee5b3d0d49fd43e1acabb775e054bba4b6e314e"
errs := append(compareRootReads(got, inv), checkRootReadLegacy(inv, seal)...)
errs = append(errs, checkRootReadDeclarations(got, inv)...)
if len(errs) > 0 {
diff --git a/internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv b/internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv
new file mode 100644
index 000000000..d98906ccd
--- /dev/null
+++ b/internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv
@@ -0,0 +1,3 @@
+# Plan 14 census overlay region-ha-2a (region HA spec 2, plan 2a: docs/superpowers/plans/2026-10-16-region-ha-2a-core.md), loaded last.
+# Task 2: the new internal/lbhealth package (the load balancer health state). Envelope-only: no root-read unit changed or was added, nothing runs as root that did not before, and no registry, /data, release or claim read was added, so the legacy seal stands.
+@coverage internal/lbhealth ad1211a01a4eb3d02cd8ed95cbff0f8b17b3976ce9c0279ebd2cae2f20c07124
diff --git a/region_ha_boundary_test.go b/region_ha_boundary_test.go
new file mode 100644
index 000000000..2499e2ef4
--- /dev/null
+++ b/region_ha_boundary_test.go
@@ -0,0 +1,181 @@
+package main
+
+import (
+ "go/ast"
+ "go/parser"
+ "go/token"
+ "os"
+ "path/filepath"
+ "sort"
+ "strconv"
+ "strings"
+ "testing"
+)
+
+// Region HA spec 2 §25: the provider boundary. Spec 2's core names no
+// cloud provider; its AWS parts are adapters in files named for AWS. Each
+// plan task that adds a core file lists it here in the same commit.
+
+// regionHACoreFiles are spec 2's provider-neutral files. None may import
+// an AWS package.
+var regionHACoreFiles = []string{
+ "internal/lbhealth/lbhealth.go",
+}
+
+// regionHAAdapterFiles are spec 2's AWS adapters. Each is named for AWS so
+// a reader can see where the provider lives.
+var regionHAAdapterFiles = []string{}
+
+// regionHAPackages are the packages spec 2 touches. Every S3 client built
+// in them goes through s3compat.Options(), so CARLOS_S3_ENDPOINT applies.
+var regionHAPackages = []string{
+ "internal/activator",
+ "internal/boxchoice",
+ "internal/console",
+ "internal/edge",
+ "internal/edgesfile",
+ "internal/host",
+ "internal/hostactors",
+ "internal/hostops",
+ "internal/lbhealth",
+}
+
+// regionHAS3Exceptions are S3 clients on main that skip s3compat, each
+// with the discussion that fixes it outside spec 2. A listed file that has
+// since been fixed passes.
+var regionHAS3Exceptions = map[string]string{
+ "internal/host/claimproduction_marker.go": "#298",
+}
+
+func TestRegionHACoreImportsNoAWS(t *testing.T) {
+ for _, f := range regionHACoreFiles {
+ file, err := parser.ParseFile(token.NewFileSet(), f, nil, parser.ImportsOnly)
+ if err != nil {
+ t.Fatalf("%s: %v", f, err)
+ }
+ for _, imp := range file.Imports {
+ path, _ := strconv.Unquote(imp.Path.Value)
+ if strings.HasPrefix(path, "github.com/aws/") {
+ t.Errorf("%s imports %s: spec 2's core names no provider (§25); move it to an AWS adapter", f, path)
+ }
+ }
+ }
+}
+
+func TestRegionHAAdaptersAreNamedForAWS(t *testing.T) {
+ for _, f := range regionHAAdapterFiles {
+ if _, err := os.Stat(f); err != nil {
+ t.Errorf("%s: %v", f, err)
+ }
+ if !strings.Contains(filepath.Base(f), "aws") && !strings.Contains(filepath.Dir(f), "aws") {
+ t.Errorf("%s is an AWS adapter but its name does not say so", f)
+ }
+ }
+}
+
+// s3NewFromConfigCalls returns the line of each s3.NewFromConfig call in
+// src that does not pass s3compat.Options() as an argument.
+func s3NewFromConfigCalls(t *testing.T, path string) []int {
+ t.Helper()
+ fset := token.NewFileSet()
+ f, err := parser.ParseFile(fset, path, nil, 0)
+ if err != nil {
+ t.Fatalf("%s: %v", path, err)
+ }
+ s3Names, compatNames := map[string]bool{}, map[string]bool{}
+ for _, imp := range f.Imports {
+ p, _ := strconv.Unquote(imp.Path.Value)
+ name := filepath.Base(p)
+ if imp.Name != nil {
+ name = imp.Name.Name
+ }
+ switch p {
+ case "github.com/aws/aws-sdk-go-v2/service/s3":
+ s3Names[name] = true
+ case "github.com/carlosframework/platform/internal/s3compat":
+ compatNames[name] = true
+ }
+ }
+ var bad []int
+ ast.Inspect(f, func(n ast.Node) bool {
+ call, ok := n.(*ast.CallExpr)
+ if !ok {
+ return true
+ }
+ sel, ok := call.Fun.(*ast.SelectorExpr)
+ if !ok || sel.Sel.Name != "NewFromConfig" {
+ return true
+ }
+ pkg, ok := sel.X.(*ast.Ident)
+ if !ok || !s3Names[pkg.Name] {
+ return true
+ }
+ for _, arg := range call.Args {
+ inner, ok := arg.(*ast.CallExpr)
+ if !ok {
+ continue
+ }
+ isel, ok := inner.Fun.(*ast.SelectorExpr)
+ if !ok || isel.Sel.Name != "Options" {
+ continue
+ }
+ if id, ok := isel.X.(*ast.Ident); ok && compatNames[id.Name] {
+ return true
+ }
+ }
+ bad = append(bad, fset.Position(call.Pos()).Line)
+ return true
+ })
+ return bad
+}
+
+func TestRegionHAS3ClientsUseS3Compat(t *testing.T) {
+ for _, dir := range regionHAPackages {
+ paths, err := filepath.Glob(filepath.Join(dir, "*.go"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ sort.Strings(paths)
+ for _, p := range paths {
+ if strings.HasSuffix(p, "_test.go") {
+ continue
+ }
+ if _, ok := regionHAS3Exceptions[filepath.ToSlash(p)]; ok {
+ continue
+ }
+ for _, line := range s3NewFromConfigCalls(t, p) {
+ t.Errorf("%s:%d builds an S3 client without s3compat.Options() (§25): CARLOS_S3_ENDPOINT would not apply", p, line)
+ }
+ }
+ }
+}
+
+// The checker itself: a bare client is caught, one with Options() is not,
+// whatever the import names.
+func TestRegionHAS3CheckerCatchesABareClient(t *testing.T) {
+ dir := t.TempDir()
+ write := func(name, body string) string {
+ p := filepath.Join(dir, name)
+ if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ return p
+ }
+ bare := write("bare.go", `package x
+import awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
+func f(cfg any) { _ = awss3.NewFromConfig(cfg) }
+`)
+ good := write("good.go", `package x
+import (
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/carlosframework/platform/internal/s3compat"
+)
+func f(cfg any) { _ = s3.NewFromConfig(cfg, s3compat.Options()) }
+`)
+ if got := s3NewFromConfigCalls(t, bare); len(got) != 1 {
+ t.Fatalf("bare client: %v", got)
+ }
+ if got := s3NewFromConfigCalls(t, good); len(got) != 0 {
+ t.Fatalf("compat client flagged: %v", got)
+ }
+}