carlos / platform Public

region HA 2b: a marked box with no region is held, the status tick says why its reference is missing

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Paul Campbell pushed by claude 1 week ago 12d011e69d8fad15855892f3077728413595d320 parent ee21b94

6 files changed, +55 −12

internal/host/boxrecord.go +6 −2
internal/host/boxreference.go +15 −6
internal/host/boxreference_test.go +29 −2
internal/host/joinmode.go +3 −1
internal/host/joinmode_test.go +1 −0
internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv +1 −1

diff --git a/internal/host/boxrecord.go b/internal/host/boxrecord.go

index 4caee88ea..f220f1593 100644

--- a/internal/host/boxrecord.go

+++ b/internal/host/boxrecord.go

@@ -181,14 +181,18 @@ func boxRecordLoop() {

if box != "" {

done, err = publishBoxRecord(ctx, store, box, read)

}

+ name := box

+ if name == "" {

+ name = "this group box"

+ }

if done {

if err != nil {

- log.Printf("agent: box record for %s: %v (region HA cannot enroll this box until an operator resolves it)", box, err)

+ log.Printf("agent: box record for %s: %v (region HA cannot enroll this box until an operator resolves it)", name, err)

}

return

}

if attempt == 1 || attempt%60 == 0 {

- log.Printf("agent: box record for %s not published yet (#%d): %v", box, attempt, err)

+ log.Printf("agent: box record for %s not published yet (#%d): %v", name, attempt, err)

}

time.Sleep(boxRecordRetry)

}

diff --git a/internal/host/boxreference.go b/internal/host/boxreference.go

index bd79f0d07..47bbf843f 100644

--- a/internal/host/boxreference.go

+++ b/internal/host/boxreference.go

@@ -5,6 +5,7 @@ import (

"encoding/base64"

"encoding/json"

"errors"

+ "fmt"

"net/http"

"strings"

"sync"

@@ -65,17 +66,25 @@ var groupRef struct {

// groupBoxReference is this box's group reference, read once and kept. ""

// while IMDS has not answered: never a label in its place.

func groupBoxReference() string {

+ ref, _ := resolveGroupBoxReference()

+ return ref

+// resolveGroupBoxReference is groupBoxReference with the reason it has

+// none: the IMDS read's error, kept so the caller can say why. Only a

+// success is kept; a failure is asked again next time.

+func resolveGroupBoxReference() (string, error) {

groupRef.Lock()

defer groupRef.Unlock()

if groupRef.ref != "" {

- return groupRef.ref

+ return groupRef.ref, nil

}

account, instance, err := groupIdentity()

if err != nil {

- return ""

+ return "", err

}

groupRef.ref = hostactors.GroupBoxReference(account, instance)

- return groupRef.ref

+ return groupRef.ref, nil

}

// errNoGroupReference is a tick of a group box whose reference has not

@@ -93,9 +102,9 @@ var errNoGroupReference = errors.New("this group box's reference could not be re

// exactly as before.

func groupStatusTick(ctx context.Context, s release.Store, reg *registry.Registry, p status.Probes, publish bool, label string, getenv func(string) string) error {

if groupReferenced(getenv) {

- ref := groupBoxReference()

- if ref == "" {

- return errNoGroupReference

+ ref, err := resolveGroupBoxReference()

+ if err != nil {

+ return fmt.Errorf("%w: %v", errNoGroupReference, err)

}

label = ref

}

diff --git a/internal/host/boxreference_test.go b/internal/host/boxreference_test.go

index 2079f3b64..4bfd1feb1 100644

--- a/internal/host/boxreference_test.go

+++ b/internal/host/boxreference_test.go

@@ -6,6 +6,7 @@ import (

"errors"

"path/filepath"

"reflect"

+ "strings"

"testing"

"github.com/carlosframework/platform/internal/hostactors"

@@ -95,8 +96,9 @@ func TestAGroupBoxPublishesOnlyUnderItsReference(t *testing.T) {

}

probes := status.Probes{MachineID: "machine-1"}

for i := 0; i < 2; i++ {

- if err := groupStatusTick(ctx, s, reg, probes, false, "", groupEnv(env)); !errors.Is(err, errNoGroupReference) {

- t.Fatalf("tick %d: %v", i, err)

+ err := groupStatusTick(ctx, s, reg, probes, false, "", groupEnv(env))

+ if !errors.Is(err, errNoGroupReference) || !strings.Contains(err.Error(), "IMDS answered 500") {

+ t.Fatalf("tick %d: %v, want errNoGroupReference with its cause", i, err)

}

keys, _ := s.List(ctx, "status/boxes/")

@@ -155,3 +157,28 @@ func TestABoxThatIsNotAGroupBoxKeepsItsLabelAndKey(t *testing.T) {

t.Fatalf("fleet box: %q", got)

}

+// Task 8's gate relies on this: a marked box's label is "" only until its

+// reference resolves, asked again on every call, and then the reference,

+// kept, even if IMDS fails afterwards.

+func TestAGroupBoxLabelIsEmptyOnlyUntilItsReferenceResolves(t *testing.T) {

+ group := groupEnv(map[string]string{hostenv.BoxReference: hostenv.GroupReference})

+ withGroupIdentity(t, "", "", nil)

+ asked := 0

+ groupIdentity = func() (string, string, error) {

+ asked++

+ if asked != 3 {

+ return "", "", errors.New("IMDS answered 500")

+ }

+ return "123456789012", "i-0a61f580df1acfa65", nil

+ }

+ want := hostactors.GroupBoxReference("123456789012", "i-0a61f580df1acfa65")

+ for i, w := range []string{"", "", want, want, want} {

+ if got := metricsBoxLabel(group, "machine-1"); got != w {

+ t.Fatalf("call %d: %q, want %q", i, got, w)

+ }

+ if asked != 3 {

+ t.Fatalf("IMDS asked %d times, want 3 (a resolved reference is kept)", asked)

+ }

diff --git a/internal/host/joinmode.go b/internal/host/joinmode.go

index 8f2ff7d5c..4f519b46d 100644

--- a/internal/host/joinmode.go

+++ b/internal/host/joinmode.go

@@ -209,7 +209,9 @@ func crossCheck(f modeFacts, row hostactors.Box) string {

return "host.env marks this box as one its region's group launched, but the roster enrolled it by hand; it does not host"

case !f.marked && row.Group != "":

return "the roster enrolled this box from group " + row.Group + ", but host.env does not mark it so; it does not host"

- case f.marked && f.region != "" && f.region != row.Region:

+ case f.marked && f.region == "":

+ return "host.env names no region, but the roster enrolled this box in " + row.Region + "; it does not host"

+ case f.marked && f.region != row.Region:

return "host.env names region " + f.region + ", but the roster enrolled this box in " + row.Region + "; it does not host"

}

return ""

diff --git a/internal/host/joinmode_test.go b/internal/host/joinmode_test.go

index 7b0bdff39..035f1a5f7 100644

--- a/internal/host/joinmode_test.go

+++ b/internal/host/joinmode_test.go

@@ -281,6 +281,7 @@ func TestSelectModeHoldsAStoreValueTheRosterDisagreesWith(t *testing.T) {

"enrolled from a group, not marked": func(f *joinFacts) { f.marked = false },

"another region": func(f *joinFacts) { f.region = "uk" },

+ "no region": func(f *joinFacts) { f.region = "" },

} {

t.Run(name, func(t *testing.T) {

f := grouped()

diff --git a/internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv b/internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv

index 47442f423..e544aa204 100644

--- a/internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv

+++ b/internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv

@@ -9,7 +9,7 @@

internal/hostenv/cli.go runHostEnv authority A,J4 rev3=census-ruling-region-ha-2b;basis=carlos system host-env: write runs as root at a box's start-up and writes /etc/carlos/host.env, 0600, whole (temporary file and rename), from allowlisted parameters with safe values only, refusing a missing required value; its one read site is seed's --from file, an operator's host.env whose values go only to the parameter store; write's path is fixed and reached only through writeHostEnv file:ReadFile@1500 df4470176b229b311927ff0ac4f268d354c3a2357cac4300ac0992d2ebb90899

internal/hostenv/write_linux.go checkedDir authority A,J4 rev3=census-ruling-region-ha-2b;basis=carlos system host-env write (root, Linux): opens one directory under one already held, O_NOFOLLOW|O_DIRECTORY, creating it 0755 first only when asked and missing, and refuses it unless fstat shows a directory owned by this process's user (root, as write requires) that no group or other can write file:Fstat@576,file:Openat@168,file:Openat@372 012257ac0866f3385044785d8abae9c31e61134928c394e22a2aeaef39ec9f3c

internal/hostenv/write_linux.go writeHostEnv authority A,J4 rev3=census-ruling-region-ha-2b;basis=carlos system host-env write (root, Linux): writes /etc/carlos/host.env only through /etc and /etc/carlos held by descriptor and checked by checkedDir; the new file is created 0600, O_EXCL|O_NOFOLLOW, in that directory, synced, renamed over host.env within it, and the directory synced; its read sites open the root and the temporary file file:Open@68,file:Openat@509 1245fc0f819ba73fb45545ffdcf92d5d4941802175e73bac209ab4d4238adff1

-@coverage internal/host 814d15fae7ab34f51237950ec34bbe646d93120bf8c7a4cf53ad5019eeef555b

+@coverage internal/host 3e02ff558f27445db10bf04624dc3eac63f77d106cd1fee31265c76465a42e38

internal/host/installchannel.go channelInstall.run authority F,J4 rev3=census-ruling-region-ha-2b;basis=carlos system install-channel (root, at a fresh box's start-up): its one read site opens root's apply lock; the install goes through verifyPointer, hostfloor.CheckImage and stageCandidate into both protected slots, and refuses a box whose guard is armed or that already runs another release file:OpenFile@196 f0cdb6dd28e83da282c3b0d7bc2012dcd80db181a8cecbf1a2db6bcc41d1d5e3

internal/host/installchannel_linux.go slotState authority F,J4 rev3=census-ruling-region-ha-2b;basis=carlos system install-channel (root, Linux): opens one protected slot with O_NOFOLLOW (a symlink is never followed), fstats the descriptor and hashes from it, to compare with the sha256 of the signed, verified artifact; a slot holding anything else, a symlink or a non-regular file is refused and left as it is, and nothing in it is executed file:Open@66,file:Stat@388 bc533a29f92ae95d31123b551354ab9f8960576e5ffef1bd6a74a06df0b57a24

internal/host/installchannel_linux.go var openat2 authority F,J4 rev3=census-ruling-region-ha-2b;basis=carlos system install-channel (root, Linux): unix.Openat2 held in a variable so a test can make it fail; removeBeneath is its only user and has no path-based fallback when it fails file:Openat2@14 6081e86f758f4ae3948ec2d4f4c037888e1362f122a410c110e31290ba914816