amadan

carlos / platform Public

region HA 2b: mode selection and group references

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude ee21b94aeb16809b8cbc0b57bf3ffe5e8b40472f parent 35d24e8
12 files changed, +1032 −7
  • internal/host/agent.go +4 −1
  • internal/host/boxrecord.go +9 −3
  • internal/host/boxreference.go +103 −0
  • internal/host/boxreference_test.go +157 −0
  • internal/host/joinmode.go +344 −0
  • internal/host/joinmode_test.go +300 −0
  • internal/host/metrics_loop.go +4 −0
  • internal/provision/plan14_root_reads_test.go +1 −1
  • internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv +5 −1
  • internal/status/fleet.go +35 −1
  • internal/status/fleet_listing_test.go +67 −0
  • region_ha_boundary_test.go +3 −0
diff --git a/internal/host/agent.go b/internal/host/agent.go
index d2461402f..dd9441505 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -3708,7 +3708,10 @@ func statusLoop(reg *registry.Registry, svc *activator.Service, buildStates func
}
}
- if err := statusTick(ctx, store, reg, p, publish, fleetLabel); err != nil {
+ // groupStatusTick is statusTick for every box but one its region's
+ // group launched, which publishes only under its group reference
+ // (census ruling region-ha-2b, boxreference.go).
+ if err := groupStatusTick(ctx, store, reg, p, publish, fleetLabel, os.Getenv); err != nil {
log.Printf("agent: status tick: %v", err)
}
diff --git a/internal/host/boxrecord.go b/internal/host/boxrecord.go
index c338b6a25..4caee88ea 100644
--- a/internal/host/boxrecord.go
+++ b/internal/host/boxrecord.go
@@ -170,11 +170,17 @@ func boxRecordLoop() {
log.Printf("agent: box record not published: %v", err)
return
}
- box := metricsBoxLabel(os.Getenv, readMachineID())
client := &http.Client{Timeout: 2 * time.Second}
- read := func() (boxid.Record, error) { return readIdentityRecord(client, imdsBase, box, time.Now()) }
for attempt := 1; ; attempt++ {
- done, err := publishBoxRecord(ctx, store, box, read)
+ // A group box's reference comes from its identity document, which
+ // IMDS may not answer yet: that is a retry, never a refusal
+ // (boxreference.go).
+ box := metricsBoxLabel(os.Getenv, readMachineID())
+ read := func() (boxid.Record, error) { return readIdentityRecord(client, imdsBase, box, time.Now()) }
+ done, err := false, errors.New("this box's group reference could not be read")
+ if box != "" {
+ done, err = publishBoxRecord(ctx, store, box, read)
+ }
if done {
if err != nil {
log.Printf("agent: box record for %s: %v (region HA cannot enroll this box until an operator resolves it)", box, err)
diff --git a/internal/host/boxreference.go b/internal/host/boxreference.go
new file mode 100644
index 000000000..bd79f0d07
--- /dev/null
+++ b/internal/host/boxreference.go
@@ -0,0 +1,103 @@
+package host
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "net/http"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostenv"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/status"
+)
+
+// A box its region's group launched is referenced by its group reference
+// (census ruling region-ha-2b): hostactors.GroupBoxReference of its account
+// and instance, "g-" and 20 hex digits. Its host.env says so:
+// carlos system host-env write adds CARLOS_BOX_REFERENCE=group. Status
+// labels come from 1,600 values and identity records are never deleted, so
+// as a group replaces its boxes, labels collide and a box whose label
+// another record holds can never publish its own; group references don't
+// collide, and are a pseudonym, so the public status feed shows no
+// instance id. Every other box, fleet boxes included, keeps its label and
+// its status key exactly as before.
+
+// groupReferenced reports whether this box takes its group reference.
+func groupReferenced(getenv func(string) string) bool {
+ return strings.TrimSpace(getenv(hostenv.BoxReference)) == hostenv.GroupReference
+}
+
+// groupIdentity reads this box's account and instance from its signed
+// identity document (IMDS); a seam for tests.
+var groupIdentity = func() (account, instance string, err error) {
+ rec, err := readIdentityRecord(&http.Client{Timeout: 2 * time.Second}, imdsBase, "", time.Now())
+ if err != nil {
+ return "", "", err
+ }
+ doc, err := base64.StdEncoding.DecodeString(rec.IdentityDocument)
+ if err != nil {
+ return "", "", err
+ }
+ var d struct {
+ AccountID string `json:"accountId"`
+ InstanceID string `json:"instanceId"`
+ }
+ if err := json.Unmarshal(doc, &d); err != nil {
+ return "", "", err
+ }
+ if d.AccountID == "" || d.InstanceID == "" {
+ return "", "", errors.New("the identity document names no account or instance")
+ }
+ return d.AccountID, d.InstanceID, nil
+}
+
+var groupRef struct {
+ sync.Mutex
+ ref string
+}
+
+// groupBoxReference is this box's group reference, read once and kept. ""
+// while IMDS has not answered: never a label in its place.
+func groupBoxReference() string {
+ groupRef.Lock()
+ defer groupRef.Unlock()
+ if groupRef.ref != "" {
+ return groupRef.ref
+ }
+ account, instance, err := groupIdentity()
+ if err != nil {
+ return ""
+ }
+ groupRef.ref = hostactors.GroupBoxReference(account, instance)
+ return groupRef.ref
+}
+
+// errNoGroupReference is a tick of a group box whose reference has not
+// resolved: it publishes nothing.
+var errNoGroupReference = errors.New("this group box's reference could not be read, so it publishes no status report this tick")
+
+// groupStatusTick is statusLoop's tick (census ruling region-ha-2b). A box
+// whose host.env carries the group marker publishes its status report
+// only under its group reference, the key the console derives, resolved
+// afresh on every tick until it resolves (groupBoxReference keeps only a
+// success). Until then it publishes nothing at all: never under
+// status.Label, which may be another box's key, since WriteBox replaces
+// whatever is there. That holds whatever the claim and region HA switches
+// say. Every other box's tick is statusTick's with statusLoop's label,
+// exactly as before.
+func groupStatusTick(ctx context.Context, s release.Store, reg *registry.Registry, p status.Probes, publish bool, label string, getenv func(string) string) error {
+ if groupReferenced(getenv) {
+ ref := groupBoxReference()
+ if ref == "" {
+ return errNoGroupReference
+ }
+ label = ref
+ }
+ return statusTick(ctx, s, reg, p, publish, label)
+}
diff --git a/internal/host/boxreference_test.go b/internal/host/boxreference_test.go
new file mode 100644
index 000000000..2079f3b64
--- /dev/null
+++ b/internal/host/boxreference_test.go
@@ -0,0 +1,157 @@
+package host
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "path/filepath"
+ "reflect"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostenv"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/status"
+)
+
+// withGroupIdentity makes IMDS answer account and instance (or fail) for
+// one test, with no reference kept from before.
+func withGroupIdentity(t *testing.T, account, instance string, err error) {
+ t.Helper()
+ was := groupIdentity
+ groupIdentity = func() (string, string, error) { return account, instance, err }
+ groupRef.Lock()
+ groupRef.ref = ""
+ groupRef.Unlock()
+ t.Cleanup(func() {
+ groupIdentity = was
+ groupRef.Lock()
+ groupRef.ref = ""
+ groupRef.Unlock()
+ })
+}
+
+func groupEnv(vals map[string]string) func(string) string {
+ return func(k string) string { return vals[k] }
+}
+
+// A box its region's group launched takes its group reference everywhere
+// it names itself: the identity record, mode selection, root's join. An
+// identity IMDS has not given is no reference, never a label in its place.
+func TestAGroupBoxTakesItsGroupReference(t *testing.T) {
+ group := groupEnv(map[string]string{hostenv.BoxReference: hostenv.GroupReference})
+ withGroupIdentity(t, "", "", errors.New("IMDS answered 500"))
+ if got := metricsBoxLabel(group, "machine"); got != "" {
+ t.Fatalf("no identity: %q", got)
+ }
+ withGroupIdentity(t, "123456789012", "i-0a61f580df1acfa65", nil)
+ want := hostactors.GroupBoxReference("123456789012", "i-0a61f580df1acfa65")
+ if got := metricsBoxLabel(group, "machine"); got != want {
+ t.Fatalf("group box: %q, want %q", got, want)
+ }
+}
+
+// statusFixture is a registry with one route and a status store holding
+// another box's report under the label this box's machine id hashes to.
+func statusFixture(t *testing.T) (*registry.Registry, *release.DirStore, []byte) {
+ t.Helper()
+ reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := reg.Upsert(registry.Route{Host: "a.example.com", Kind: "static"}); err != nil {
+ t.Fatal(err)
+ }
+ s := release.NewDirStore(t.TempDir())
+ victim := []byte(`{"label":"victim","metrics":{"instance_id":"i-victim"}}`)
+ if _, err := s.Put(context.Background(), "status/boxes/"+status.Label("machine-1")+".json", victim, ""); err != nil {
+ t.Fatal(err)
+ }
+ return reg, s, victim
+}
+
+// T1 (census ruling region-ha-2b): a group box whose reference has not
+// resolved publishes nothing, whatever the claim and region HA switches
+// say; above all never under status.Label, where another box's report may
+// stand. The reference is asked again each tick, and once it resolves the
+// box publishes under it. Through statusLoop's own tick and WriteBox.
+func TestAGroupBoxPublishesOnlyUnderItsReference(t *testing.T) {
+ for name, env := range map[string]map[string]string{
+ "switches on": {hostenv.BoxReference: hostenv.GroupReference, "CARLOS_HOST_CLAIM": "1", "CARLOS_REGION_HA": "1"},
+ "switches off": {hostenv.BoxReference: hostenv.GroupReference},
+ } {
+ t.Run(name, func(t *testing.T) {
+ ctx := context.Background()
+ reg, s, victim := statusFixture(t)
+ withGroupIdentity(t, "", "", errors.New("IMDS answered 500"))
+ asked := 0
+ groupIdentity = func() (string, string, error) {
+ asked++
+ if asked < 3 {
+ return "", "", errors.New("IMDS answered 500")
+ }
+ return "123456789012", "i-0a61f580df1acfa65", nil
+ }
+ probes := status.Probes{MachineID: "machine-1"}
+ for i := 0; i < 2; i++ {
+ if err := groupStatusTick(ctx, s, reg, probes, false, "", groupEnv(env)); !errors.Is(err, errNoGroupReference) {
+ t.Fatalf("tick %d: %v", i, err)
+ }
+ }
+ keys, _ := s.List(ctx, "status/boxes/")
+ got, _, _ := s.Get(ctx, "status/boxes/"+status.Label("machine-1")+".json")
+ if len(keys) != 1 || !bytes.Equal(got, victim) {
+ t.Fatalf("published with no reference: %v, victim now %s", keys, got)
+ }
+ if err := groupStatusTick(ctx, s, reg, probes, false, "", groupEnv(env)); err != nil {
+ t.Fatal(err)
+ }
+ ref := hostactors.GroupBoxReference("123456789012", "i-0a61f580df1acfa65")
+ if _, _, err := s.Get(ctx, "status/boxes/"+ref+".json"); err != nil || asked != 3 {
+ t.Fatalf("after recovery: %v, IMDS asked %d times", err, asked)
+ }
+ if got, _, _ := s.Get(ctx, "status/boxes/"+status.Label("machine-1")+".json"); !bytes.Equal(got, victim) {
+ t.Fatalf("the victim's report was replaced: %s", got)
+ }
+ })
+ }
+}
+
+// The ruling's third test: a box that is not a group box keeps its label
+// and its status key byte for byte: statusLoop's tick is statusTick's, with
+// statusLoop's own label (CollectBox's pseudonym, or a fleet box's label).
+func TestABoxThatIsNotAGroupBoxKeepsItsLabelAndKey(t *testing.T) {
+ withGroupIdentity(t, "123456789012", "i-0a61f580df1acfa65", nil)
+ for name, env := range map[string]map[string]string{
+ "no marker": {},
+ "another value": {hostenv.BoxReference: "instance-id"},
+ "region HA, no marker": {"CARLOS_REGION_HA": "1", "CARLOS_HOST_CLAIM": "1"},
+ } {
+ getenv := groupEnv(env)
+ if got := metricsBoxLabel(getenv, "machine-1"); got != status.Label("machine-1") {
+ t.Fatalf("%s: label %q, want %q", name, got, status.Label("machine-1"))
+ }
+ for _, label := range []string{"", "kitchen"} {
+ ctx := context.Background()
+ reg, _, _ := statusFixture(t)
+ mine, theirs := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir())
+ probes := status.Probes{MachineID: "machine-1"}
+ if err := groupStatusTick(ctx, mine, reg, probes, false, label, getenv); err != nil {
+ t.Fatal(err)
+ }
+ if err := statusTick(ctx, theirs, reg, probes, false, label); err != nil {
+ t.Fatal(err)
+ }
+ a, _ := mine.List(ctx, "status/boxes/")
+ b, _ := theirs.List(ctx, "status/boxes/")
+ if !reflect.DeepEqual(a, b) || len(a) != 1 {
+ t.Fatalf("%s, label %q: keys %v, statusTick's %v", name, label, a, b)
+ }
+ }
+ }
+ fleet := groupEnv(map[string]string{"CARLOS_FLEET": "pi", "CARLOS_FLEET_BOX": "kitchen"})
+ if got := metricsBoxLabel(fleet, "machine-1"); got != "kitchen" {
+ t.Fatalf("fleet box: %q", got)
+ }
+}
diff --git a/internal/host/joinmode.go b/internal/host/joinmode.go
new file mode 100644
index 000000000..8f2ff7d5c
--- /dev/null
+++ b/internal/host/joinmode.go
@@ -0,0 +1,344 @@
+package host
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "net/http"
+ "os"
+ "runtime"
+ "strings"
+ "sync/atomic"
+ "time"
+
+ "github.com/carlosframework/platform/internal/boxid"
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostfloor"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/status"
+ "github.com/carlosframework/platform/internal/version"
+)
+
+// Joining and hosting (region HA spec 2 §5, A1-2, A2-4).
+//
+// With CARLOS_HOST_CLAIM and CARLOS_REGION_HA both on, the agent chooses
+// its mode at every start, from three facts and never from one file's
+// presence. It hosts only when all three hold:
+//
+// - /etc/carlos/host-actor.json names this box;
+// - the signed roster has an active entry for that enrollment and
+// instance;
+// - the host lifecycle guard is armed under the roster's floor, with
+// this process's own binary one it admits.
+//
+// Anything else is joining. A joining box hosts nothing, publishes no
+// edges file, and answers the load balancer 503 (joining.go). Joining
+// resumes from whatever it finds, and each stage names the next step:
+//
+// - no invitation: the console has not invited the box yet;
+// - invited: root arms the guard (carlos system join, joinroot.go);
+// - guard installed: the box publishes its guard-ready report;
+// - guard-ready reported: the console enrolls the box;
+// - held: the roster names this box, but not as an active entry for its
+// enrollment and armed guard, or, for a box its region's group
+// launched, the instance was stopped and started since the identity
+// record it was enrolled on (C3: it must prove its group membership
+// again, and never hosts on its old enrollment). Nothing automatic
+// can fix that: a held group box answers 503 and the group replaces
+// it; an operator sorts out any other.
+//
+// A fact that cannot be read is never taken as its negative: the mode is
+// then unknown (errModeUnknown), and the caller asks again later.
+
+type agentMode string
+
+const (
+ modeHosting agentMode = "hosting"
+ modeJoining agentMode = "joining"
+)
+
+type joinStage string
+
+const (
+ stageNoInvitation joinStage = "no invitation"
+ stageInvited joinStage = "invited"
+ stageGuardInstalled joinStage = "guard installed"
+ stageGuardReady joinStage = "guard-ready reported"
+ stageHeld joinStage = "held"
+)
+
+// modeDecision is one start's answer. Stage and Why are joining's.
+type modeDecision struct {
+ Mode agentMode
+ Stage joinStage
+ Box string
+ Why string
+}
+
+// errModeUnknown is a fact that could not be read: no mode is chosen.
+var errModeUnknown = errors.New("agent mode unknown")
+
+func modeUnknown(what string, err error) error {
+ return fmt.Errorf("%w: %s could not be read: %v", errModeUnknown, what, err)
+}
+
+// reportFreshness is how old a guard-ready report may be and still count:
+// the console's own limit (hostactors.VerifyReport).
+const reportFreshness = 15 * time.Minute
+
+// hostActorPath is the box's enrollment, written by guard-arm.
+const hostActorPath = "/etc/carlos/host-actor.json"
+
+// modeFacts are where selectMode reads from; production's are
+// productionModeFacts.
+type modeFacts struct {
+ // box is this box's reference: the label its identity record and its
+ // status report are published under.
+ box string
+ // enrollment reads host-actor.json; os.ErrNotExist when there is none.
+ enrollment func() (hostfloor.Enrollment, error)
+ // roster reads the signed roster.
+ roster func(context.Context) (hostactors.Roster, error)
+ // guard is the guard's view of this process (hostfloor.Disk.Running).
+ // hostfloor.ErrFloor or os.ErrNotExist: not armed, or not for this
+ // binary.
+ guard func() (hostfloor.Running, error)
+ // invited reports whether the console has written box's invitation.
+ invited func(context.Context, string) (bool, error)
+ // report reads box's own published status report; os.ErrNotExist when
+ // there is none.
+ report func(context.Context, string) (status.Box, error)
+ // relaunch says why this instance is not the launch the roster entry
+ // row was enrolled on ("" when it is): asked only of a box its
+ // region's group launched (row.Group set).
+ relaunch func(context.Context, hostactors.Box) (string, error)
+ // marked is host.env's group marker (CARLOS_BOX_REFERENCE=group), and
+ // region its CARLOS_REGION: values from the parameter store, which
+ // select this box's authority, so they are checked against the signed
+ // roster before the box hosts (crossCheck).
+ marked bool
+ region string
+ now func() time.Time
+}
+
+// selectMode chooses this start's mode.
+func selectMode(ctx context.Context, f modeFacts) (modeDecision, error) {
+ d := modeDecision{Mode: modeJoining, Box: f.box}
+ e, err := f.enrollment()
+ if errors.Is(err, os.ErrNotExist) {
+ invited, err := f.invited(ctx, f.box)
+ if err != nil {
+ return d, modeUnknown("the invitation", err)
+ }
+ d.Stage = stageNoInvitation
+ if invited {
+ d.Stage = stageInvited
+ }
+ return d, nil
+ }
+ if err != nil {
+ return d, modeUnknown(hostActorPath, err)
+ }
+ if e.Box != f.box {
+ d.Stage, d.Why = stageHeld, fmt.Sprintf("%s names %s, not this box (%s)", hostActorPath, e.Box, f.box)
+ return d, nil
+ }
+ r, err := f.roster(ctx)
+ if err != nil {
+ return d, modeUnknown("the roster", err)
+ }
+ g, err := f.guard()
+ if err != nil && !errors.Is(err, hostfloor.ErrFloor) && !errors.Is(err, os.ErrNotExist) {
+ return d, modeUnknown("the host lifecycle guard", err)
+ }
+ armed := err == nil && g.GuardReady && g.FloorSHA256 == r.FloorSHA256
+ row, listed := r.Boxes[e.Box]
+ switch {
+ case listed && row.Enrollment == e.Enrollment && row.Cloud.InstanceID == e.InstanceID && row.State == "active" && armed:
+ if why := crossCheck(f, row); why != "" {
+ d.Stage, d.Why = stageHeld, why
+ return d, nil
+ }
+ if row.Group != "" {
+ why, err := f.relaunch(ctx, row)
+ if err != nil {
+ return d, modeUnknown("this instance's launch", err)
+ }
+ if why != "" {
+ d.Stage, d.Why = stageHeld, why
+ return d, nil
+ }
+ }
+ d.Mode, d.Stage = modeHosting, ""
+ case listed:
+ d.Stage = stageHeld
+ d.Why = fmt.Sprintf("the roster's entry for %s is %s, enrollment %s on %s; this box holds enrollment %s on %s, guard armed %t",
+ e.Box, row.State, row.Enrollment, row.Cloud.InstanceID, e.Enrollment, e.InstanceID, armed)
+ case !armed:
+ // guard-arm writes host-actor.json before it persists the armed
+ // policy: an arm that was cut off is still the invited stage.
+ d.Stage = stageInvited
+ default:
+ rep, err := f.report(ctx, f.box)
+ if err != nil && !errors.Is(err, os.ErrNotExist) {
+ return d, modeUnknown("this box's status report", err)
+ }
+ d.Stage = stageGuardInstalled
+ if err == nil && guardReadyReport(rep, e, g, f.now()) {
+ d.Stage = stageGuardReady
+ }
+ }
+ return d, nil
+}
+
+// crossCheck says why host.env's authority-selecting values disagree with
+// the roster entry the box would host under ("" when they agree). The
+// group marker must match whether the console enrolled the box from a
+// group, and a group box's CARLOS_REGION must name the region it was
+// enrolled in. It is made here, at the agent's use, and not when
+// host-env write assembles the file: at boot the box has no roster entry
+// to check against (it is not enrolled yet), and before enrollment
+// neither value grants anything, since the console derives the box's
+// reference and region from its proven instance itself. A box an operator
+// enrolled keeps its own CARLOS_REGION, unchecked, as on main.
+func crossCheck(f modeFacts, row hostactors.Box) string {
+ switch {
+ case f.marked && row.Group == "":
+ return "host.env marks this box as one its region's group launched, but the roster enrolled it by hand; it does not host"
+ case !f.marked && row.Group != "":
+ return "the roster enrolled this box from group " + row.Group + ", but host.env does not mark it so; it does not host"
+ case f.marked && f.region != "" && f.region != row.Region:
+ return "host.env names region " + f.region + ", but the roster enrolled this box in " + row.Region + "; it does not host"
+ }
+ return ""
+}
+
+// relaunched says why current, this instance's identity record read from
+// IMDS now, is not the launch row was enrolled on: the record published
+// under the box's reference (enrolled) must be the one the roster entry
+// binds (its digest), and the two identity documents must name the same
+// pendingTime. EC2 sets pendingTime each time an instance enters the
+// pending state: at launch, and again at every start after a stop, but
+// not at a reboot. "" when they agree.
+func relaunched(row hostactors.Box, enrolled []byte, current boxid.Record) string {
+ if hostfloor.Digest(enrolled) != row.EvidenceSHA256 {
+ return "the identity record under this box's reference is not the one it was enrolled on"
+ }
+ var rec boxid.Record
+ if json.Unmarshal(enrolled, &rec) != nil {
+ return "the identity record it was enrolled on cannot be read"
+ }
+ then, now := recordPendingTime(rec), recordPendingTime(current)
+ switch {
+ case then == "" || now == "":
+ return "an identity document names no pendingTime, so this launch cannot be matched to its enrollment"
+ case then != now:
+ return fmt.Sprintf("this instance was stopped and started since it was enrolled (launched %s, enrolled on the launch of %s): it must prove its group membership again, so it does not host", now, then)
+ }
+ return ""
+}
+
+// recordPendingTime is the pendingTime of a record's identity document, ""
+// when it has none.
+func recordPendingTime(rec boxid.Record) string {
+ doc, err := base64.StdEncoding.DecodeString(rec.IdentityDocument)
+ if err != nil {
+ return ""
+ }
+ var d struct {
+ PendingTime string `json:"pendingTime"`
+ }
+ if json.Unmarshal(doc, &d) != nil {
+ return ""
+ }
+ return d.PendingTime
+}
+
+// guardReadyReport reports whether rep is this box's fresh guard-ready
+// report for enrollment e under the guard g.
+func guardReadyReport(rep status.Box, e hostfloor.Enrollment, g hostfloor.Running, now time.Time) bool {
+ m := rep.Metrics
+ return m != nil && m.Enrollment == e.Enrollment && m.FloorGuardReady && m.FloorSHA256 == g.FloorSHA256 && m.BinarySHA256 == g.Hash &&
+ !rep.GeneratedAt.IsZero() && now.Sub(rep.GeneratedAt) <= reportFreshness
+}
+
+// productionModeFacts reads the deployment store, the status store and
+// this box's own files.
+func productionModeFacts(ctx context.Context, getenv func(string) string) (modeFacts, error) {
+ store, err := openDeployStoreEnv(ctx, getenv)
+ if err != nil {
+ return modeFacts{}, err
+ }
+ statusStore, _, err := openStatusStore(ctx, getenv)
+ if err != nil {
+ return modeFacts{}, err
+ }
+ pub, err := release.ParsePublicKey(getenv("CARLOS_RELEASE_PUBKEY"), "CARLOS_RELEASE_PUBKEY")
+ if err != nil {
+ return modeFacts{}, err
+ }
+ disk, err := floorDisk(release.OptDir())
+ if err != nil {
+ return modeFacts{}, err
+ }
+ bucket := getenv("CARLOS_DEPLOYMENT_BUCKET")
+ box := metricsBoxLabel(getenv, readMachineID())
+ if box == "" {
+ return modeFacts{}, errors.New("this box's group reference could not be read")
+ }
+ imds := &http.Client{Timeout: 2 * time.Second}
+ return modeFacts{
+ box: box,
+ marked: groupReferenced(getenv),
+ region: strings.TrimSpace(getenv("CARLOS_REGION")),
+ enrollment: func() (hostfloor.Enrollment, error) { return hostfloor.ReadEnrollment(hostActorPath) },
+ roster: func(ctx context.Context) (hostactors.Roster, error) {
+ return readGuardRoster(ctx, store, pub, bucket)
+ },
+ guard: func() (hostfloor.Running, error) {
+ return disk.Running(version.Build, runtime.GOOS+"-"+runtime.GOARCH)
+ },
+ invited: func(ctx context.Context, box string) (bool, error) {
+ _, _, err := store.Get(ctx, invitationKey(box))
+ if errors.Is(err, os.ErrNotExist) {
+ return false, nil
+ }
+ return err == nil, err
+ },
+ report: func(ctx context.Context, box string) (status.Box, error) {
+ raw, _, err := statusStore.Get(ctx, "status/boxes/"+box+".json")
+ if err != nil {
+ return status.Box{}, err
+ }
+ var b status.Box
+ if err := json.Unmarshal(raw, &b); err != nil {
+ return status.Box{}, err
+ }
+ return b, nil
+ },
+ relaunch: func(ctx context.Context, row hostactors.Box) (string, error) {
+ enrolled, err := boxid.Load(ctx, store, box)
+ if err != nil {
+ return "", err
+ }
+ current, err := readIdentityRecord(imds, imdsBase, box, time.Now())
+ if err != nil {
+ return "", err
+ }
+ return relaunched(row, enrolled, current), nil
+ },
+ now: time.Now,
+ }, nil
+}
+
+// invitationKey is where the console writes box's invitation
+// (hostactors.Directory.Invite).
+func invitationKey(box string) string { return "control/host-actor-invitations/" + box + ".json" }
+
+// agentHosting is this process's mode, decided once by RunAgent before
+// the agent starts (hostingGate): true for the claim agent, false while
+// joining. The claim wiring reports it to the load balancer's health
+// answer (ClaimWiring.startRegional).
+var agentHosting atomic.Bool
diff --git a/internal/host/joinmode_test.go b/internal/host/joinmode_test.go
new file mode 100644
index 000000000..7b0bdff39
--- /dev/null
+++ b/internal/host/joinmode_test.go
@@ -0,0 +1,300 @@
+package host
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "io/fs"
+ "os"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/boxid"
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostfloor"
+ "github.com/carlosframework/platform/internal/status"
+)
+
+var joinNow = time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC)
+
+// joinFacts is a box, "ada-hiking", in each state the tests name. The
+// zero value has no host-actor.json and no invitation.
+type joinFacts struct {
+ enrollment *hostfloor.Enrollment
+ enrollmentErr error
+ roster hostactors.Roster
+ rosterErr error
+ guard hostfloor.Running
+ guardErr error
+ invited bool
+ invitedErr error
+ report *status.Box
+ reportErr error
+ relaunchWhy string
+ relaunchErr error
+ relaunchAsked *bool
+ marked bool
+ region string
+}
+
+func (j joinFacts) facts() modeFacts {
+ return modeFacts{
+ box: "ada-hiking",
+ enrollment: func() (hostfloor.Enrollment, error) {
+ if j.enrollmentErr != nil {
+ return hostfloor.Enrollment{}, j.enrollmentErr
+ }
+ if j.enrollment == nil {
+ return hostfloor.Enrollment{}, &fs.PathError{Op: "open", Path: hostActorPath, Err: fs.ErrNotExist}
+ }
+ return *j.enrollment, nil
+ },
+ roster: func(context.Context) (hostactors.Roster, error) { return j.roster, j.rosterErr },
+ guard: func() (hostfloor.Running, error) {
+ if j.guardErr != nil {
+ return hostfloor.Running{}, j.guardErr
+ }
+ return j.guard, nil
+ },
+ invited: func(context.Context, string) (bool, error) { return j.invited, j.invitedErr },
+ report: func(context.Context, string) (status.Box, error) {
+ if j.reportErr != nil {
+ return status.Box{}, j.reportErr
+ }
+ if j.report == nil {
+ return status.Box{}, os.ErrNotExist
+ }
+ return *j.report, nil
+ },
+ relaunch: func(context.Context, hostactors.Box) (string, error) {
+ if j.relaunchAsked != nil {
+ *j.relaunchAsked = true
+ }
+ return j.relaunchWhy, j.relaunchErr
+ },
+ marked: j.marked,
+ region: j.region,
+ now: func() time.Time { return joinNow },
+ }
+}
+
+const joinEnrollment = "0123456789abcdef0123456789abcdef"
+
+func enrolledFacts() joinFacts {
+ e := hostfloor.Enrollment{Box: "ada-hiking", Enrollment: joinEnrollment, InstanceID: "i-ada"}
+ return joinFacts{
+ enrollment: &e,
+ roster: hostactors.Roster{FloorSHA256: "floor", Boxes: map[string]hostactors.Box{"ada-hiking": {
+ Enrollment: joinEnrollment, State: "active", Cloud: hostactors.Cloud{InstanceID: "i-ada"}}}},
+ guard: hostfloor.Running{GuardReady: true, FloorSHA256: "floor", Hash: "image"},
+ }
+}
+
+// guardInstalled is a box that guard-arm has armed but the console has not
+// enrolled yet.
+func guardInstalled() joinFacts {
+ f := enrolledFacts()
+ f.roster.Boxes = map[string]hostactors.Box{}
+ return f
+}
+
+func freshReport(at time.Time) *status.Box {
+ return &status.Box{GeneratedAt: at, Metrics: &status.BoxMetrics{Enrollment: joinEnrollment, FloorGuardReady: true, FloorSHA256: "floor", BinarySHA256: "image"}}
+}
+
+func TestSelectModeHostsOnlyWhenEnrolledAndArmed(t *testing.T) {
+ d, err := selectMode(context.Background(), enrolledFacts().facts())
+ if err != nil || d.Mode != modeHosting || d.Box != "ada-hiking" {
+ t.Fatalf("enrolled and armed: %+v %v", d, err)
+ }
+ held := map[string]func(*joinFacts){
+ "the guard is not armed": func(f *joinFacts) { f.guardErr = hostfloor.ErrFloor },
+ "the guard has no policy": func(f *joinFacts) { f.guardErr = &fs.PathError{Op: "open", Path: "policy.json", Err: fs.ErrNotExist} },
+ "the guard is for another floor": func(f *joinFacts) { f.guard.FloorSHA256 = "old" },
+ "the roster entry is quarantined": func(f *joinFacts) {
+ b := f.roster.Boxes["ada-hiking"]
+ b.State = "quarantined"
+ f.roster.Boxes["ada-hiking"] = b
+ },
+ "the roster entry is another enrollment": func(f *joinFacts) {
+ b := f.roster.Boxes["ada-hiking"]
+ b.Enrollment = strings.Repeat("f", 32)
+ f.roster.Boxes["ada-hiking"] = b
+ },
+ "the roster entry is another instance": func(f *joinFacts) {
+ b := f.roster.Boxes["ada-hiking"]
+ b.Cloud.InstanceID = "i-old"
+ f.roster.Boxes["ada-hiking"] = b
+ },
+ "host-actor.json names another box": func(f *joinFacts) { f.enrollment.Box = "grace-hiking" },
+ }
+ for name, spoil := range held {
+ t.Run(name, func(t *testing.T) {
+ f := enrolledFacts()
+ spoil(&f)
+ d, err := selectMode(context.Background(), f.facts())
+ if err != nil || d.Mode != modeJoining || d.Stage != stageHeld || d.Why == "" {
+ t.Fatalf("%+v %v", d, err)
+ }
+ })
+ }
+}
+
+// A restart at each joining state resumes joining at that state (spec 2
+// §11): nothing but an active roster entry with an armed guard hosts.
+func TestSelectModeResumesEachJoiningStage(t *testing.T) {
+ for _, c := range []struct {
+ name string
+ facts joinFacts
+ want joinStage
+ }{
+ {"no invitation", joinFacts{}, stageNoInvitation},
+ {"invited", joinFacts{invited: true}, stageInvited},
+ {"guard-arm cut off after host-actor.json", func() joinFacts { f := guardInstalled(); f.guardErr = hostfloor.ErrFloor; return f }(), stageInvited},
+ {"guard installed", guardInstalled(), stageGuardInstalled},
+ {"guard-ready reported", func() joinFacts { f := guardInstalled(); f.report = freshReport(joinNow.Add(-time.Minute)); return f }(), stageGuardReady},
+ {"a stale report", func() joinFacts {
+ f := guardInstalled()
+ f.report = freshReport(joinNow.Add(-16 * time.Minute))
+ return f
+ }(), stageGuardInstalled},
+ {"a report for another image", func() joinFacts {
+ f := guardInstalled()
+ f.report = freshReport(joinNow)
+ f.report.Metrics.BinarySHA256 = "bootstrap"
+ return f
+ }(), stageGuardInstalled},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ d, err := selectMode(context.Background(), c.facts.facts())
+ if err != nil || d.Mode != modeJoining || d.Stage != c.want {
+ t.Fatalf("%+v %v, want joining at %s", d, err, c.want)
+ }
+ })
+ }
+}
+
+// A fact that cannot be read is unknown, never its negative: an enrolled
+// box whose roster read fails is not sent back to joining.
+func TestSelectModeIsUnknownWhenAFactCannotBeRead(t *testing.T) {
+ outage := errors.New("connection reset")
+ for name, f := range map[string]joinFacts{
+ "the invitation": {invitedErr: outage},
+ "host-actor.json": {enrollmentErr: hostfloor.ErrFloor},
+ "the roster": func() joinFacts { f := enrolledFacts(); f.rosterErr = outage; return f }(),
+ "the guard": func() joinFacts { f := enrolledFacts(); f.guardErr = errors.New("permission denied"); return f }(),
+ "the status report": func() joinFacts { f := guardInstalled(); f.reportErr = outage; return f }(),
+ "a roster that does not verify": func() joinFacts { f := guardInstalled(); f.rosterErr = hostactors.ErrRoster; return f }(),
+ } {
+ t.Run(name, func(t *testing.T) {
+ d, err := selectMode(context.Background(), f.facts())
+ if !errors.Is(err, errModeUnknown) || d.Mode == modeHosting {
+ t.Fatalf("%+v %v", d, err)
+ }
+ })
+ }
+}
+
+// C3: a box its region's group launched hosts only on the launch it was
+// enrolled on. Stopped and started since, it is held, never hosting on its
+// old enrollment; a launch that cannot be read is unknown. A box an
+// operator enrolled (no group) is not asked.
+func TestSelectModeHoldsAGroupBoxStartedAgain(t *testing.T) {
+ grouped := func() joinFacts {
+ f := enrolledFacts()
+ b := f.roster.Boxes["ada-hiking"]
+ b.Group, b.Region = "routers-ie", "ie"
+ f.roster.Boxes["ada-hiking"] = b
+ f.relaunchAsked = new(bool)
+ f.marked, f.region = true, "ie"
+ return f
+ }
+ f := grouped()
+ if d, err := selectMode(context.Background(), f.facts()); err != nil || d.Mode != modeHosting || !*f.relaunchAsked {
+ t.Fatalf("same launch: %+v %v", d, err)
+ }
+ f = grouped()
+ f.relaunchWhy = "this instance was stopped and started since it was enrolled"
+ if d, err := selectMode(context.Background(), f.facts()); err != nil || d.Mode != modeJoining || d.Stage != stageHeld || d.Why != f.relaunchWhy {
+ t.Fatalf("started again: %+v %v", d, err)
+ }
+ f = grouped()
+ f.relaunchErr = errors.New("IMDS answered 500")
+ if d, err := selectMode(context.Background(), f.facts()); !errors.Is(err, errModeUnknown) || d.Mode == modeHosting {
+ t.Fatalf("unreadable launch: %+v %v", d, err)
+ }
+ f = enrolledFacts()
+ f.relaunchWhy, f.relaunchAsked = "never asked", new(bool)
+ if d, err := selectMode(context.Background(), f.facts()); err != nil || d.Mode != modeHosting || *f.relaunchAsked {
+ t.Fatalf("an operator-enrolled box: %+v %v, asked %v", d, err, *f.relaunchAsked)
+ }
+}
+
+func TestRelaunchedComparesTheLaunchTheRosterBinds(t *testing.T) {
+ record := func(pending string) (boxid.Record, []byte) {
+ doc := `{"accountId":"123","region":"eu-west-1","instanceId":"i-ada"` + pending + `}`
+ rec := boxid.Record{V: 1, Box: "i-ada", IdentityDocument: base64.StdEncoding.EncodeToString([]byte(doc)), IdentitySignature: "c2ln"}
+ raw, _ := json.Marshal(rec)
+ return rec, raw
+ }
+ enrolledRec, enrolled := record(`,"pendingTime":"2026-10-17T09:00:00Z"`)
+ row := hostactors.Box{EvidenceSHA256: hostfloor.Digest(enrolled), Group: "routers-ie"}
+ if why := relaunched(row, enrolled, enrolledRec); why != "" {
+ t.Fatalf("the same launch: %q", why)
+ }
+ again, _ := record(`,"pendingTime":"2026-10-17T11:30:00Z"`)
+ if why := relaunched(row, enrolled, again); !strings.Contains(why, "stopped and started") {
+ t.Fatalf("started again: %q", why)
+ }
+ none, _ := record("")
+ if why := relaunched(row, enrolled, none); why == "" {
+ t.Fatal("a document with no pendingTime matched")
+ }
+ _, other := record(`,"pendingTime":"2026-10-17T09:00:00Z","x":1`)
+ if why := relaunched(row, other, enrolledRec); !strings.Contains(why, "not the one it was enrolled on") {
+ t.Fatalf("another record: %q", why)
+ }
+}
+
+// Condition 3 of census ruling region-ha-2b: the group marker and a group
+// box's CARLOS_REGION come from the parameter store, and a value that
+// disagrees with the signed roster holds the box instead of hosting it.
+func TestSelectModeHoldsAStoreValueTheRosterDisagreesWith(t *testing.T) {
+ grouped := func() joinFacts {
+ f := enrolledFacts()
+ b := f.roster.Boxes["ada-hiking"]
+ b.Group, b.Region = "routers-ie", "ie"
+ f.roster.Boxes["ada-hiking"] = b
+ f.marked, f.region = true, "ie"
+ return f
+ }
+ if d, err := selectMode(context.Background(), grouped().facts()); err != nil || d.Mode != modeHosting {
+ t.Fatalf("agreeing: %+v %v", d, err)
+ }
+ for name, spoil := range map[string]func(*joinFacts){
+ "marked, enrolled by hand": func(f *joinFacts) {
+ b := f.roster.Boxes["ada-hiking"]
+ b.Group = ""
+ f.roster.Boxes["ada-hiking"] = b
+ },
+ "enrolled from a group, not marked": func(f *joinFacts) { f.marked = false },
+ "another region": func(f *joinFacts) { f.region = "uk" },
+ } {
+ t.Run(name, func(t *testing.T) {
+ f := grouped()
+ spoil(&f)
+ d, err := selectMode(context.Background(), f.facts())
+ if err != nil || d.Mode != modeJoining || d.Stage != stageHeld || !strings.Contains(d.Why, "does not host") {
+ t.Fatalf("%+v %v", d, err)
+ }
+ })
+ }
+ // A box an operator enrolled keeps its own region, unchecked.
+ f := enrolledFacts()
+ f.region = "anything"
+ if d, err := selectMode(context.Background(), f.facts()); err != nil || d.Mode != modeHosting {
+ t.Fatalf("an operator-enrolled box: %+v %v", d, err)
+ }
+}
diff --git a/internal/host/metrics_loop.go b/internal/host/metrics_loop.go
index 2e0dad037..845876a0c 100644
--- a/internal/host/metrics_loop.go
+++ b/internal/host/metrics_loop.go
@@ -715,6 +715,10 @@ func loadAlertState(path, box string) metrics.AlertState {
// label rather than disabling metrics outright, mirroring statusLoop's own
// degrade-don't-disable choice.
func metricsBoxLabel(getenv func(string) string, machineID string) string {
+ if groupReferenced(getenv) {
+ // A box its region's group launched (boxreference.go).
+ return groupBoxReference()
+ }
if identity, err := resolveFleetIdentity(getenv); err == nil {
return identity.Label
} else if !errors.Is(err, errNotFleetBox) {
diff --git a/internal/provision/plan14_root_reads_test.go b/internal/provision/plan14_root_reads_test.go
index 2ee14760a..bbd2e5a47 100644
--- a/internal/provision/plan14_root_reads_test.go
+++ b/internal/provision/plan14_root_reads_test.go
@@ -816,7 +816,7 @@ func TestPlan14RootReadCensus(t *testing.T) {
if err = loadRootReadInventory(filepath.Join(root, "internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv"), &inv, true); err != nil {
t.Fatal(err)
}
- seal = "8afff5c9874f6d31079ffb2e722274b0b1f82eef7ae228e3c816245c239a93e1"
+ seal = "fc584e113d0c6786bdeb3344354ae567c112c607e68f85056352545557a91b5d"
errs := append(compareRootReads(got, inv), checkRootReadLegacy(inv, seal)...)
errs = append(errs, checkRootReadDeclarations(got, inv)...)
if len(errs) > 0 {
diff --git a/internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv b/internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv
index faf32cdd5..47442f423 100644
--- a/internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv
+++ b/internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv
@@ -3,12 +3,13 @@
# Task 3: carlos system install-channel (internal/host installchannel.go, installchannel_linux.go): a fresh box installs its signed channel release into both protected slots through verifyPointer, hostfloor.CheckImage and stageCandidate, all unchanged, and on every run hands each directory on the agent's cache path (system, system/router and the version's) to the service user through bootstrap's own descriptor-anchored open, refusing a symlinked parent. One new root-read unit, channelInstall.run, whose one read site opens root's apply lock (systemApplier's lockPath). Ruled authority F,J4. stageCandidate, accepted-legacy, is called and not changed. The legacy seal stands. Fix rounds 1 and 2: more root-read units. slotState (Linux): each protected slot opened with O_NOFOLLOW and hashed from the descriptor against the signed artifact's sha256, to refuse a slot holding a build this command did not install. removeBeneath (Linux): removes the agent's cache entry only by openat2 and unlinkat on descriptors anchored at the router directory, opened with no symlink followed, never by path name, and refuses without openat2. var openat2: the unix.Openat2 seam for the test that it refuses. All ruled F,J4 on the same footing as channelInstall.run.
# Task 4: group membership, read by the console (internal/hostactors membershipread.go: MembershipReader, the unexported confirmation and the write bound, the single enforcement point of the 60-second deadline, each store call cut off at 15 seconds; enrollment_confirmation.go: EnrollAutomatic and CheckConfirmed make the read themselves and write on the bound store, and the box-written confirmation object is removed; groupref.go: GroupBoxReference; private.go: a group box's report read under the reference the console derives; awsmembership, the Auto Scaling and EC2 describe adapter) and the Auto Scaling SDK in go.mod. No root-read unit is added or changed: the read is the cloud's, through the reader. Envelope-only, and the seal stands. The record-write census gains deadlineStore.Put (nonrecord, ruled).
# Task 5: the console's automatic enrollment pass (internal/hostactors autoenroll.go: AutoEnroller: an enrolling write the deadline left unknown is settled by the next pass's roster read; internal/console enrollpass.go, started in StartClaimRecoveryPass with region HA on, and awsmembership.go, its membership reader). One new root-read unit, AutoEnroller.inputs, whose two read sites are the signed roster (Directory.Read) and the host floor (hostfloor.Read), in the console. Ruled accepted-legacy (J4 root-input). Invite (only on the bound store, after a membership read), enrollmentNonce (the read-only invitation check), BeginEnrollmentAttempt and checkMembership, accepted-legacy, are called and not changed. Fix round 1: CheckConfirmed is unexported (enrollment_confirmation.go), the pass binds the invitation to the confirmation checkConfirmed returns, and an invited box is found with enrollmentNonce, never Invite. The seal moves.
+# Task 6: mode selection and group references (internal/host joinmode.go: selectMode, productionModeFacts, relaunched and crossCheck: a group box stopped and started since its enrollment, or whose host.env disagrees with the roster, is held; boxreference.go, metrics_loop.go's metricsBoxLabel and boxrecord.go's boxRecordLoop: a group box takes its group reference; internal/status fleet.go: the status listing drops reports no box has rewritten in a day before the key cap). statusLoop, accepted-legacy, is re-sealed as ruled: its tick is groupStatusTick, which publishes a group box's report only under its group reference, resolved afresh each tick, and nothing until it resolves; every other box's tick is statusTick's, unchanged. No other root-read unit is added or changed. The seal moves.
@coverage . 3678cfe22582d35fe0d8e93d3fed898cdffa6fb2ce2d81e0252c8a8da77993b3
@coverage internal/hostenv 45d9d4b90a5d441020f5ba4573821ca92701246c80b32d6f1823665fb0f65208
internal/hostenv/cli.go runHostEnv authority A,J4 rev3=census-ruling-region-ha-2b;basis=carlos system host-env: write runs as root at a box's start-up and writes /etc/carlos/host.env, 0600, whole (temporary file and rename), from allowlisted parameters with safe values only, refusing a missing required value; its one read site is seed's --from file, an operator's host.env whose values go only to the parameter store; write's path is fixed and reached only through writeHostEnv file:ReadFile@1500 df4470176b229b311927ff0ac4f268d354c3a2357cac4300ac0992d2ebb90899
internal/hostenv/write_linux.go checkedDir authority A,J4 rev3=census-ruling-region-ha-2b;basis=carlos system host-env write (root, Linux): opens one directory under one already held, O_NOFOLLOW|O_DIRECTORY, creating it 0755 first only when asked and missing, and refuses it unless fstat shows a directory owned by this process's user (root, as write requires) that no group or other can write file:Fstat@576,file:Openat@168,file:Openat@372 012257ac0866f3385044785d8abae9c31e61134928c394e22a2aeaef39ec9f3c
internal/hostenv/write_linux.go writeHostEnv authority A,J4 rev3=census-ruling-region-ha-2b;basis=carlos system host-env write (root, Linux): writes /etc/carlos/host.env only through /etc and /etc/carlos held by descriptor and checked by checkedDir; the new file is created 0600, O_EXCL|O_NOFOLLOW, in that directory, synced, renamed over host.env within it, and the directory synced; its read sites open the root and the temporary file file:Open@68,file:Openat@509 1245fc0f819ba73fb45545ffdcf92d5d4941802175e73bac209ab4d4238adff1
-@coverage internal/host fca6227146690ccc5c14310d24728c69acd42465d1ec277e08c68e7a7f11997c
+@coverage internal/host 814d15fae7ab34f51237950ec34bbe646d93120bf8c7a4cf53ad5019eeef555b
internal/host/installchannel.go channelInstall.run authority F,J4 rev3=census-ruling-region-ha-2b;basis=carlos system install-channel (root, at a fresh box's start-up): its one read site opens root's apply lock; the install goes through verifyPointer, hostfloor.CheckImage and stageCandidate into both protected slots, and refuses a box whose guard is armed or that already runs another release file:OpenFile@196 f0cdb6dd28e83da282c3b0d7bc2012dcd80db181a8cecbf1a2db6bcc41d1d5e3
internal/host/installchannel_linux.go slotState authority F,J4 rev3=census-ruling-region-ha-2b;basis=carlos system install-channel (root, Linux): opens one protected slot with O_NOFOLLOW (a symlink is never followed), fstats the descriptor and hashes from it, to compare with the sha256 of the signed, verified artifact; a slot holding anything else, a symlink or a non-regular file is refused and left as it is, and nothing in it is executed file:Open@66,file:Stat@388 bc533a29f92ae95d31123b551354ab9f8960576e5ffef1bd6a74a06df0b57a24
internal/host/installchannel_linux.go var openat2 authority F,J4 rev3=census-ruling-region-ha-2b;basis=carlos system install-channel (root, Linux): unix.Openat2 held in a variable so a test can make it fail; removeBeneath is its only user and has no path-based fallback when it fails file:Openat2@14 6081e86f758f4ae3948ec2d4f4c037888e1362f122a410c110e31290ba914816
@@ -18,3 +19,6 @@ internal/host/installchannel_linux.go removeBeneath authority F,J4 rev3=census-r
@coverage internal/console 80fa5bba8bd23325ababaedc2b8ce810b25161c93a81c6d97dd7033bcc5df103
@legacy 2f4ca2a51661d8aae1e0f38a41efdcc9df9f1dd75411ca6742d6ee3ae71f1756 internal/hostactors/autoenroll.go AutoEnroller.inputs legacy accepted-legacy accepted-legacy;rev3=J4;basis=root-input;census-ruling-region-ha-2b file:Read@102,file:Read@305 c0ae921ee9d8df9de403d488d9708c565d659a7bf880199313f50bdc7c8f8127
internal/hostactors/autoenroll.go AutoEnroller.inputs legacy accepted-legacy accepted-legacy;rev3=J4;basis=root-input;census-ruling-region-ha-2b file:Read@102,file:Read@305 c0ae921ee9d8df9de403d488d9708c565d659a7bf880199313f50bdc7c8f8127
+@coverage internal/status a1eee66fc906d6a7275017fd30c346b64aaafe4f88a501538248e99e9d95c33d
+@legacy 30a3ee6b3365c202bf93b26087888a582c6b8d6df1a9f1fc6ba12e639ddd8684 internal/host/agent.go statusLoop legacy accepted-legacy accepted-legacy;rev3=census-ruling-region-ha-2b;basis=the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos): main's body with its tick through groupStatusTick, so a box whose host.env carries the group marker (CARLOS_BOX_REFERENCE=group) publishes its status report only under its group reference, resolved afresh each tick, and nothing until it resolves, never under status.Label; every other box, fleet boxes included, gets statusTick with today's label and key; read sites unchanged whole:Route@6154,whole:Route@6181,whole:Route@7423,whole:Route@9437 666ddbc4779442db4c868253cf5062c690e20ad643197688305e0742f6de9283
+internal/host/agent.go statusLoop legacy accepted-legacy accepted-legacy;rev3=census-ruling-region-ha-2b;basis=the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos): main's body with its tick through groupStatusTick, so a box whose host.env carries the group marker (CARLOS_BOX_REFERENCE=group) publishes its status report only under its group reference, resolved afresh each tick, and nothing until it resolves, never under status.Label; every other box, fleet boxes included, gets statusTick with today's label and key; read sites unchanged whole:Route@6154,whole:Route@6181,whole:Route@7423,whole:Route@9437 666ddbc4779442db4c868253cf5062c690e20ad643197688305e0742f6de9283
diff --git a/internal/status/fleet.go b/internal/status/fleet.go
index eea3f3d2d..3d4920043 100644
--- a/internal/status/fleet.go
+++ b/internal/status/fleet.go
@@ -235,8 +235,42 @@ func LoadBoxesComplete(ctx context.Context, s release.Store) (boxes []Box, compl
return loadBoxesReport(ctx, s)
}
+// statusListingHorizon is how long ago a report's object may last have
+// been written and still be listed as a box (region HA plan 2b). Each box
+// rewrites its own report every SnapshotPeriod, so an object older than a
+// day is a box that has gone: with automatic replacement such reports pile
+// up, one per replaced box, and could fill the key cap below, hiding the
+// boxes that are running. They are dropped from the listing before any
+// read and before the cap, so the cap bounds current reports only. This is
+// an optimisation, never authority: every reader still judges each
+// report's own GeneratedAt as before (BoxStale, the console's
+// VerifyReport), and a day keeps every box an operator would still look
+// for as stale. A store that cannot say when an object was written
+// (release.ModLister) is listed whole, as before.
+const statusListingHorizon = 24 * time.Hour
+
+// listBoxKeys lists status/boxes/, without the reports the horizon drops.
+func listBoxKeys(ctx context.Context, s release.Store) ([]string, error) {
+ ml, ok := s.(release.ModLister)
+ if !ok {
+ return s.List(ctx, "status/boxes/")
+ }
+ mods, err := ml.ListMod(ctx, "status/boxes/")
+ if err != nil {
+ return nil, err
+ }
+ cut := time.Now().Add(-statusListingHorizon)
+ keys := make([]string, 0, len(mods))
+ for _, m := range mods {
+ if m.Mod.IsZero() || !m.Mod.Before(cut) {
+ keys = append(keys, m.Key)
+ }
+ }
+ return keys, nil
+}
+
func loadBoxesReport(ctx context.Context, s release.Store) (boxes []Box, complete bool, err error) {
- keys, err := s.List(ctx, "status/boxes/")
+ keys, err := listBoxKeys(ctx, s)
if err != nil {
return nil, false, err
}
diff --git a/internal/status/fleet_listing_test.go b/internal/status/fleet_listing_test.go
new file mode 100644
index 000000000..0f354ce23
--- /dev/null
+++ b/internal/status/fleet_listing_test.go
@@ -0,0 +1,67 @@
+package status
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// listOnly hides the store's ModLister, as a store that cannot say when an
+// object was written does.
+type listOnly struct{ release.Store }
+
+// T2 (region HA plan 2b): 250 reports of boxes a group replaced long ago
+// do not hide the two boxes running now. The old reports are dropped from
+// the listing before any read and before the key cap, and the reading is
+// complete.
+func TestStaleReportsDoNotHideLiveBoxes(t *testing.T) {
+ ctx := context.Background()
+ root := t.TempDir()
+ s := release.NewDirStore(root)
+ now := time.Now()
+ put := func(label string, at time.Time) {
+ raw, _ := json.Marshal(Box{Label: label, GeneratedAt: at})
+ key := "status/boxes/" + label + ".json"
+ if _, err := s.Put(ctx, key, raw, ""); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Chtimes(filepath.Join(root, filepath.FromSlash(key)), at, at); err != nil {
+ t.Fatal(err)
+ }
+ }
+ for i := 0; i < 250; i++ {
+ put(fmt.Sprintf("g-%020d", i), now.Add(-48*time.Hour))
+ }
+ put("g-live-one", now.Add(-time.Minute))
+ put("g-live-two", now.Add(-2*time.Minute))
+ // A report written an hour ago that says it is old: still listed (the
+ // listing only drops what is plainly history) and judged stale by its
+ // readers, as before.
+ put("g-quiet", now.Add(-time.Hour))
+
+ boxes, complete, err := LoadBoxesComplete(ctx, s)
+ if err != nil || !complete {
+ t.Fatalf("complete %v, err %v", complete, err)
+ }
+ var labels []string
+ for _, b := range boxes {
+ labels = append(labels, b.Label)
+ }
+ if fmt.Sprint(labels) != "[g-live-one g-live-two g-quiet]" {
+ t.Fatalf("boxes %v", labels)
+ }
+ if !BoxStale(boxes[2], now, SnapshotPeriod) {
+ t.Fatal("an old report read as fresh")
+ }
+ // A store with no write times is listed whole, as before: the cap
+ // applies to all 253, and the reading says it is incomplete.
+ if _, complete, err := LoadBoxesComplete(ctx, listOnly{s}); err != nil || complete {
+ t.Fatalf("listed whole: complete %v, err %v", complete, err)
+ }
+}
diff --git a/region_ha_boundary_test.go b/region_ha_boundary_test.go
index c321421c0..d6b5f3f2a 100644
--- a/region_ha_boundary_test.go
+++ b/region_ha_boundary_test.go
@@ -39,6 +39,9 @@ var regionHACoreFiles = []string{
"internal/hostactors/groupref.go",
"internal/hostactors/autoenroll.go",
"internal/console/enrollpass.go",
+ "internal/host/joinmode.go",
+ "internal/host/boxreference.go",
+ "internal/status/fleet.go",
}
// regionHAAdapterFiles are spec 2's AWS adapters. Each is named for AWS so