amadan

carlos / platform Public

agent: with region HA on, a box publishes its AWS-signed identity once, under its label

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude 11bde18277da01d3b682448006eccf7031944ec6 parent ab345ad
4 files changed, +296 −2
  • internal/host/agent.go +3 −0
  • internal/host/boxrecord.go +151 −0
  • internal/host/boxrecord_test.go +141 −0
  • internal/host/edgesfile.go +1 −2
diff --git a/internal/host/agent.go b/internal/host/agent.go
index 94dad4ed4..7b826b063 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -522,6 +522,9 @@ func RunAgent(args []string) {
go retentionLoop(reg)
go registrySnapshotLoop(reg)
go edgesFileLoop(reg)
+ // Region HA enrollment evidence (spec 1 §5.1): once, at start, and only
+ // with CARLOS_REGION_HA on. See boxrecord.go.
+ go boxRecordLoop()
// The mesh key's own loop (multi-region backhaul spec, Part 3 Task 2) —
// deliberately NOT a duty riding retentionLoop's tick: that loop returns
// early, never arming at all, on a box missing CARLOS_RELEASE_PUBKEY,
diff --git a/internal/host/boxrecord.go b/internal/host/boxrecord.go
new file mode 100644
index 000000000..9069d3524
--- /dev/null
+++ b/internal/host/boxrecord.go
@@ -0,0 +1,151 @@
+package host
+
+import (
+ "context"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "io"
+ "log"
+ "net/http"
+ "os"
+ "strings"
+ "time"
+
+ "github.com/carlosframework/platform/internal/boxid"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// regionHAEnabled is CARLOS_REGION_HA: on for any value but "" and "0".
+// Every region HA behaviour on a box is behind it (spec 1 §8 step 2).
+func regionHAEnabled(getenv func(string) string) bool {
+ switch strings.TrimSpace(getenv("CARLOS_REGION_HA")) {
+ case "", "0":
+ return false
+ }
+ return true
+}
+
+const (
+ imdsBase = "http://169.254.169.254"
+ imdsIdentityDocument = "/latest/dynamic/instance-identity/document"
+ imdsIdentitySignature = "/latest/dynamic/instance-identity/signature"
+ boxRecordRetry = time.Minute
+)
+
+// imdsGetRaw reads one IMDSv2 path and returns its body untouched. The
+// identity document is signed byte for byte, so unlike imdsLeaf nothing is
+// trimmed.
+func imdsGetRaw(client *http.Client, base, token, path string) ([]byte, error) {
+ req, err := http.NewRequest(http.MethodGet, base+path, nil)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("X-aws-ec2-metadata-token", token)
+ resp, err := client.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+ body, err := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
+ if err != nil {
+ return nil, err
+ }
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("IMDS %s answered %d", path, resp.StatusCode)
+ }
+ return body, nil
+}
+
+// readIdentityRecord builds this box's record (spec §5.1) from IMDS: the
+// identity document exactly as served, and its base64 RSA-SHA256
+// signature with any line breaks removed (base64 ignores them; Go's
+// decoder does not).
+func readIdentityRecord(client *http.Client, base, box string, now time.Time) (boxid.Record, error) {
+ req, err := http.NewRequest(http.MethodPut, base+"/latest/api/token", nil)
+ if err != nil {
+ return boxid.Record{}, err
+ }
+ req.Header.Set("X-aws-ec2-metadata-token-ttl-seconds", "60")
+ resp, err := client.Do(req)
+ if err != nil {
+ return boxid.Record{}, fmt.Errorf("IMDS token: %w", err)
+ }
+ tok, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
+ resp.Body.Close()
+ if resp.StatusCode != http.StatusOK || len(strings.TrimSpace(string(tok))) == 0 {
+ return boxid.Record{}, fmt.Errorf("IMDS token answered %d", resp.StatusCode)
+ }
+ token := strings.TrimSpace(string(tok))
+ doc, err := imdsGetRaw(client, base, token, imdsIdentityDocument)
+ if err != nil {
+ return boxid.Record{}, err
+ }
+ sig, err := imdsGetRaw(client, base, token, imdsIdentitySignature)
+ if err != nil {
+ return boxid.Record{}, err
+ }
+ cleaned := strings.Join(strings.Fields(string(sig)), "")
+ if _, err := base64.StdEncoding.DecodeString(cleaned); err != nil || cleaned == "" {
+ return boxid.Record{}, errors.New("IMDS signature is not base64")
+ }
+ return boxid.Record{V: 1, Box: box, IdentityDocument: base64.StdEncoding.EncodeToString(doc),
+ IdentitySignature: cleaned, WrittenAt: now.UTC().Format(time.RFC3339)}, nil
+}
+
+// publishBoxRecord makes one attempt to publish this box's enrollment
+// evidence. done is true once there is nothing left to retry: the record
+// is in place, or a different one already holds this reference, which
+// only an operator can sort out and which is logged loudly.
+func publishBoxRecord(ctx context.Context, s release.Store, box string, read func() (boxid.Record, error)) (done bool, err error) {
+ if !boxid.ValidBox(box) {
+ return true, fmt.Errorf("box reference %q is not a label; no enrollment evidence is published", box)
+ }
+ if _, err := boxid.Load(ctx, s, box); err == nil {
+ return true, nil // create-only: an existing record is never rewritten
+ } else if !errors.Is(err, os.ErrNotExist) {
+ return false, err
+ }
+ rec, err := read()
+ if err != nil {
+ return false, err
+ }
+ err = boxid.Publish(ctx, s, rec)
+ if errors.Is(err, release.ErrCASConflict) {
+ return true, fmt.Errorf("another record already holds %s: %w", box, err)
+ }
+ return err == nil, err
+}
+
+// boxRecordLoop publishes this box's enrollment evidence once, at start,
+// when CARLOS_REGION_HA is on, retrying each minute until it is in place.
+// The record grants nothing: enrollment (an operator, or for Auto Scaling
+// boxes the console with a membership confirmation) reads it later.
+// Platform-box job: skipped on fleet boxes.
+func boxRecordLoop() {
+ if !regionHAEnabled(os.Getenv) || shouldSkipPublisherElection(os.Getenv) {
+ return
+ }
+ ctx := context.Background()
+ store, err := openDeployStore(ctx)
+ if err != nil {
+ log.Printf("agent: box record not published: %v", err)
+ return
+ }
+ box := metricsBoxLabel(os.Getenv, readMachineID())
+ client := &http.Client{Timeout: 2 * time.Second}
+ read := func() (boxid.Record, error) { return readIdentityRecord(client, imdsBase, box, time.Now()) }
+ for attempt := 1; ; attempt++ {
+ done, err := publishBoxRecord(ctx, store, box, read)
+ if done {
+ if err != nil {
+ log.Printf("agent: box record for %s: %v (region HA cannot enroll this box until an operator resolves it)", box, err)
+ }
+ return
+ }
+ if attempt == 1 || attempt%60 == 0 {
+ log.Printf("agent: box record for %s not published yet (#%d): %v", box, attempt, err)
+ }
+ time.Sleep(boxRecordRetry)
+ }
+}
diff --git a/internal/host/boxrecord_test.go b/internal/host/boxrecord_test.go
new file mode 100644
index 000000000..8d55ce309
--- /dev/null
+++ b/internal/host/boxrecord_test.go
@@ -0,0 +1,141 @@
+package host
+
+import (
+ "context"
+ "encoding/base64"
+ "errors"
+ "io"
+ "net/http"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/boxid"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// fakeIMDS answers IMDSv2 in-process: no socket, so it runs in any sandbox.
+type fakeIMDS struct {
+ doc, sig string
+ status int // for the document and signature; 0 = 200
+ calls []string
+}
+
+func (f *fakeIMDS) RoundTrip(r *http.Request) (*http.Response, error) {
+ f.calls = append(f.calls, r.Method+" "+r.URL.Path)
+ answer := func(code int, body string) (*http.Response, error) {
+ return &http.Response{StatusCode: code, Body: io.NopCloser(strings.NewReader(body)), Header: http.Header{}, Request: r}, nil
+ }
+ if r.Method == http.MethodPut && r.URL.Path == "/latest/api/token" {
+ return answer(200, "tok\n")
+ }
+ if r.Header.Get("X-aws-ec2-metadata-token") != "tok" {
+ return answer(401, "")
+ }
+ code := f.status
+ if code == 0 {
+ code = 200
+ }
+ switch r.URL.Path {
+ case imdsIdentityDocument:
+ return answer(code, f.doc)
+ case imdsIdentitySignature:
+ return answer(code, f.sig)
+ }
+ return answer(404, "")
+}
+
+func imdsClient(f *fakeIMDS) *http.Client { return &http.Client{Transport: f} }
+
+var recordNow = time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
+
+// The document is signed byte for byte, so its trailing newline must
+// survive; the signature arrives wrapped and must not.
+func TestReadIdentityRecordKeepsTheDocumentsExactBytes(t *testing.T) {
+ doc := "{\n \"accountId\" : \"111122223333\",\n \"instanceId\" : \"i-0abc\"\n}\n"
+ sig := base64.StdEncoding.EncodeToString([]byte(strings.Repeat("s", 256)))
+ wrapped := sig[:64] + "\n" + sig[64:] + "\n"
+ f := &fakeIMDS{doc: doc, sig: wrapped}
+ rec, err := readIdentityRecord(imdsClient(f), "http://imds.test", "annie-welding", recordNow)
+ if err != nil {
+ t.Fatal(err)
+ }
+ got, _ := base64.StdEncoding.DecodeString(rec.IdentityDocument)
+ if string(got) != doc {
+ t.Fatalf("document = %q, want the bytes IMDS served, %q", got, doc)
+ }
+ if rec.IdentitySignature != sig || rec.V != 1 || rec.Box != "annie-welding" || rec.WrittenAt != "2026-10-06T09:00:00Z" {
+ t.Fatalf("record = %+v", rec)
+ }
+}
+
+func TestReadIdentityRecordRefusesWhatIsNotAnAnswer(t *testing.T) {
+ for name, f := range map[string]*fakeIMDS{
+ "document missing": {doc: "{}", sig: "c2lnCg==", status: 404},
+ "signature not base64": {doc: "{}", sig: "not base64!"},
+ "signature empty": {doc: "{}", sig: "\n"},
+ } {
+ t.Run(name, func(t *testing.T) {
+ if rec, err := readIdentityRecord(imdsClient(f), "http://imds.test", "annie-welding", recordNow); err == nil {
+ t.Fatalf("record = %+v, want an error", rec)
+ }
+ })
+ }
+}
+
+func TestPublishBoxRecordWritesOnceAndNeverRewrites(t *testing.T) {
+ ctx, st := context.Background(), release.NewDirStore(t.TempDir())
+ reads := 0
+ read := func() (boxid.Record, error) {
+ reads++
+ return boxid.Record{V: 1, Box: "annie-welding", IdentityDocument: "ZG9j", IdentitySignature: "c2ln", WrittenAt: "2026-10-06T09:00:00Z"}, nil
+ }
+ if done, err := publishBoxRecord(ctx, st, "annie-welding", read); !done || err != nil {
+ t.Fatalf("first publish: done=%v err=%v", done, err)
+ }
+ if done, err := publishBoxRecord(ctx, st, "annie-welding", read); !done || err != nil {
+ t.Fatalf("second publish: done=%v err=%v", done, err)
+ }
+ if reads != 1 {
+ t.Fatalf("IMDS read %d times; an existing record must not be re-read or rewritten", reads)
+ }
+}
+
+func TestPublishBoxRecordRetriesAReadFailureAndStopsOnABadLabel(t *testing.T) {
+ ctx, st := context.Background(), release.NewDirStore(t.TempDir())
+ failing := func() (boxid.Record, error) { return boxid.Record{}, errors.New("IMDS down") }
+ if done, err := publishBoxRecord(ctx, st, "annie-welding", failing); done || err == nil {
+ t.Fatalf("done=%v err=%v: a read failure is retried", done, err)
+ }
+ if done, err := publishBoxRecord(ctx, st, "Annie.Welding", failing); !done || err == nil {
+ t.Fatalf("done=%v err=%v: a reference that is not a label is not retried", done, err)
+ }
+}
+
+// A store that refuses the read with something other than not-found must
+// not be taken as "no record yet": that would publish over an outage.
+type unreadableBoxStore struct{ release.Store }
+
+func (unreadableBoxStore) Get(context.Context, string) ([]byte, string, error) {
+ return nil, "", errors.New("AccessDenied")
+}
+
+func TestPublishBoxRecordDoesNotPublishWhenItCannotRead(t *testing.T) {
+ ctx := context.Background()
+ st := unreadableBoxStore{release.NewDirStore(t.TempDir())}
+ read := func() (boxid.Record, error) {
+ t.Fatal("IMDS read although the store could not say whether a record exists")
+ return boxid.Record{}, nil
+ }
+ if done, err := publishBoxRecord(ctx, st, "annie-welding", read); done || err == nil {
+ t.Fatalf("done=%v err=%v", done, err)
+ }
+}
+
+func TestRegionHAEnabled(t *testing.T) {
+ for v, want := range map[string]bool{"": false, "0": false, " 0 ": false, "1": true, "yes": true} {
+ if got := regionHAEnabled(func(string) string { return v }); got != want {
+ t.Errorf("CARLOS_REGION_HA=%q: %v, want %v", v, got, want)
+ }
+ }
+}
diff --git a/internal/host/edgesfile.go b/internal/host/edgesfile.go
index f8861b37d..be1f47275 100644
--- a/internal/host/edgesfile.go
+++ b/internal/host/edgesfile.go
@@ -157,8 +157,7 @@ const capRegionHA1 = "region-ha-1"
// edgesFileCapabilities publishes region-ha-1 only when CARLOS_REGION_HA is
// set to something other than "" or "0".
func edgesFileCapabilities(getenv func(string) string) []string {
- switch strings.TrimSpace(getenv("CARLOS_REGION_HA")) {
- case "", "0":
+ if !regionHAEnabled(getenv) {
return nil
}
return []string{capRegionHA1}