amadan
region HA 2b: the console's automatic enrolment pass
hostactors.AutoEnroller and internal/console/enrollpass.go, started beside 2a's capacity pass on a host-claim console with region HA on. The invite is bound to the confirmation checkConfirmed itself builds, never to one rebuilt from exported evidence. Only a positive answer (not a member, stale) counts against an attempt; an unknown read or store result keeps it and its wait.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
11 files changed,
+1320
−2
internal/console/awsmembership.go+16 −0internal/console/claim_authority_census_test.go+5 −0internal/console/enrollpass.go+100 −0internal/console/enrollpass_test.go+71 −0internal/console/hostclaimops.go+3 −0internal/console/testdata/claim-authority-sites.txt+3 −0internal/hostactors/autoenroll.go+379 −0internal/hostactors/autoenroll_test.go+734 −0internal/provision/plan14_root_reads_test.go+1 −1internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv+5 −1region_ha_boundary_test.go+3 −0
diff --git a/internal/console/awsmembership.go b/internal/console/awsmembership.go| new file mode 100644 |
| index 000000000..2447bafc3 |
| --- /dev/null |
| +++ b/internal/console/awsmembership.go |
| @@ -0,0 +1,16 @@ |
| +package console |
| + |
| +import ( |
| + "context" |
| + |
| + "github.com/carlosframework/platform/internal/hostactors" |
| + "github.com/carlosframework/platform/internal/hostactors/awsmembership" |
| +) |
| + |
| +// consoleMembershipReader is the enrollment pass's membership read (region |
| +// HA spec 2 §5 step 5, H28, §25): the AWS adapter, over the console's own |
| +// instance role, which 2c grants autoscaling:DescribeAutoScalingInstances |
| +// and ec2:DescribeInstances, read-only, in the deployment's AWS regions. |
| +func consoleMembershipReader(ctx context.Context) (hostactors.MembershipReader, error) { |
| + return awsmembership.New(ctx) |
| +} |
diff --git a/internal/console/claim_authority_census_test.go b/internal/console/claim_authority_census_test.go| index c8567c565..e8646b917 100644 |
| --- a/internal/console/claim_authority_census_test.go |
| +++ b/internal/console/claim_authority_census_test.go |
| @@ -43,6 +43,11 @@ func claimAuthoritySites(t *testing.T) []string { |
| "EnrollBackfillApp": true, "OpenGated": true, "WithAppMutation": true, |
| "withClaimSQL": true, "withPreparedClaimSQL": true, "RegisterIngest": true, "RecoverIngest": true, "RecoverSQLTransaction": true, "BindTransaction": true, "PrepareSQLPermit": true, "FenceSQL": true, "FenceSQLOutcome": true, "WithProjection": true, |
| "BindMutationRunner": true, "claimEnrollmentContext": true, |
| + // Region HA spec 2's automatic enrollment pass (plan 2b Task 5): its |
| + // construction hands this console's roster directory and root |
| + // signer to hostactors.AutoEnroller, its start runs it, and its one |
| + // invocation is the pass that invites and enrolls boxes. |
| + "newAutoEnroller": true, "runEnrollPass": true, "EnrollPass": true, |
| } |
| fs := token.NewFileSet() |
| var sites []string |
diff --git a/internal/console/enrollpass.go b/internal/console/enrollpass.go| new file mode 100644 |
| index 000000000..64cae7a6f |
| --- /dev/null |
| +++ b/internal/console/enrollpass.go |
| @@ -0,0 +1,100 @@ |
| +package console |
| + |
| +import ( |
| + "context" |
| + "log" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/boxid" |
| + "github.com/carlosframework/platform/internal/hostactors" |
| + "github.com/carlosframework/platform/internal/hostops" |
| +) |
| + |
| +// The console's automatic enrollment pass (region HA spec 2 §5 step 5, |
| +// H16, H28): every enrollPassInterval it takes the next step for each box |
| +// its region's group launched that the roster does not list yet |
| +// (hostactors.AutoEnroller: attempt, invitation, guard-ready, enrollment). |
| +// It runs beside claim recovery, on a host-claim console with region HA |
| +// on, and signs with the root key this console already holds. It confirms |
| +// each box's group membership itself, with its own read-only Auto Scaling |
| +// and EC2 describes (consoleMembershipReader), immediately before it |
| +// invites and before it enrolls. The console is the only writer of the |
| +// roster and the only membership confirmer, so it is the only place this |
| +// can run. |
| + |
| +// enrollPassInterval is how often the pass runs: a box is invited, and |
| +// enrolled once its guard-ready report is published, within ten seconds. |
| +const enrollPassInterval = 10 * time.Second |
| + |
| +// enrollPatience is how long the pass keeps an attempt the cloud refused |
| +// before it begins another (doubling with each refusal). |
| +const enrollPatience = 2 * time.Minute |
| + |
| +// newAutoEnroller is the pass over this console's claim authority (its |
| +// roster, root signer, report evidence and namespace, and the default |
| +// pool) and its own membership reader. An error when either is missing. |
| +func newAutoEnroller(a *hostops.ProductionAuthority, members hostactors.MembershipReader) (*hostactors.AutoEnroller, error) { |
| + if a == nil || members == nil { |
| + return nil, hostops.ErrContract |
| + } |
| + certs, err := boxid.EmbeddedCerts() |
| + if err != nil { |
| + return nil, err |
| + } |
| + return &hostactors.AutoEnroller{Directory: a.Directory, Signer: a.Signer, Evidence: a.Evidence, Members: members, Certs: certs, |
| + Namespace: a.Namespace, Pool: "default", Now: time.Now, Patience: enrollPatience, Logf: log.Printf}, nil |
| +} |
| + |
| +// runEnrollPass runs the pass until ctx ends. Each box's step is logged |
| +// when it changes, and a failed pass on its first failure and every |
| +// tenth. |
| +func runEnrollPass(ctx context.Context, e *hostactors.AutoEnroller, every time.Duration, logf func(string, ...any)) { |
| + logf("console: automatic box enrollment armed (every %s)", every) |
| + last := map[string]string{} |
| + fails := 0 |
| + for { |
| + steps, err := e.EnrollPass(ctx) |
| + switch { |
| + case err != nil: |
| + fails++ |
| + if fails == 1 || fails%10 == 0 { |
| + logf("console: automatic box enrollment (failure #%d): %v", fails, err) |
| + } |
| + default: |
| + fails = 0 |
| + seen := map[string]bool{} |
| + for _, st := range steps { |
| + seen[st.Box] = true |
| + if last[st.Box] != st.Step { |
| + logf("console: automatic box enrollment: %s: %s", st.Box, st.Step) |
| + last[st.Box] = st.Step |
| + } |
| + } |
| + for box := range last { |
| + if !seen[box] { |
| + delete(last, box) |
| + } |
| + } |
| + } |
| + select { |
| + case <-ctx.Done(): |
| + return |
| + case <-time.After(every): |
| + } |
| + } |
| +} |
| + |
| +// startEnrollPass starts the pass for this server, or logs why it cannot. |
| +func (s *Server) startEnrollPass(ctx context.Context) { |
| + members, err := consoleMembershipReader(ctx) |
| + if err != nil { |
| + log.Printf("console: automatic box enrollment: not started: %v", err) |
| + return |
| + } |
| + e, err := newAutoEnroller(s.cfg.ClaimAuthority, members) |
| + if err != nil { |
| + log.Printf("console: automatic box enrollment: not started: %v", err) |
| + return |
| + } |
| + go runEnrollPass(ctx, e, enrollPassInterval, log.Printf) |
| +} |
diff --git a/internal/console/enrollpass_test.go b/internal/console/enrollpass_test.go| new file mode 100644 |
| index 000000000..87bbac2db |
| --- /dev/null |
| +++ b/internal/console/enrollpass_test.go |
| @@ -0,0 +1,71 @@ |
| +package console |
| + |
| +import ( |
| + "context" |
| + "crypto/ed25519" |
| + "errors" |
| + "fmt" |
| + "strings" |
| + "sync" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/hostactors" |
| + "github.com/carlosframework/platform/internal/hostops" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// noMembers is a membership reader that is never asked. |
| +type noMembers struct{} |
| + |
| +func (noMembers) Member(context.Context, string, string) (hostactors.Membership, error) { |
| + return hostactors.Membership{}, errors.New("not asked") |
| +} |
| + |
| +func TestEnrollPassIsBuiltFromTheClaimAuthority(t *testing.T) { |
| + pub, key, _ := ed25519.GenerateKey(nil) |
| + a := &hostops.ProductionAuthority{Directory: hostactors.Directory{Store: release.NewDirStore(t.TempDir()), PublicKey: pub, Bucket: "bucket"}, |
| + Signer: release.RootSigner(key), Namespace: "platform"} |
| + if _, err := newAutoEnroller(nil, noMembers{}); !errors.Is(err, hostops.ErrContract) { |
| + t.Fatalf("no authority: %v", err) |
| + } |
| + if _, err := newAutoEnroller(a, nil); !errors.Is(err, hostops.ErrContract) { |
| + t.Fatalf("no membership reader: %v", err) |
| + } |
| + e, err := newAutoEnroller(a, noMembers{}) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if e.Namespace != "platform" || e.Pool != "default" || e.Patience != enrollPatience || len(e.Certs) == 0 || e.Directory.Bucket != "bucket" || e.Members != (noMembers{}) || e.Logf == nil { |
| + t.Fatalf("enroller %+v", e) |
| + } |
| +} |
| + |
| +// A pass that cannot read the roster is logged on its first failure and |
| +// every tenth, not every ten seconds, and the loop ends with its context. |
| +func TestEnrollPassLogsFailuresSparingly(t *testing.T) { |
| + pub, key, _ := ed25519.GenerateKey(nil) |
| + e := &hostactors.AutoEnroller{Directory: hostactors.Directory{Store: release.NewDirStore(t.TempDir()), PublicKey: pub, Bucket: "bucket"}, |
| + Signer: release.RootSigner(key), Namespace: "platform", Pool: "default", Now: time.Now, Patience: time.Minute} |
| + var mu sync.Mutex |
| + var lines []string |
| + logf := func(format string, args ...any) { |
| + mu.Lock() |
| + lines = append(lines, fmt.Sprintf(format, args...)) |
| + mu.Unlock() |
| + } |
| + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) |
| + defer cancel() |
| + runEnrollPass(ctx, e, 2*time.Millisecond, logf) |
| + mu.Lock() |
| + defer mu.Unlock() |
| + if len(lines) < 2 || !strings.Contains(lines[0], "armed") { |
| + t.Fatalf("lines %q", lines) |
| + } |
| + for i, l := range lines[1:] { |
| + var n int |
| + if _, err := fmt.Sscanf(l[strings.Index(l, "#"):], "#%d", &n); err != nil || (n != 1 && n%10 != 0) || (i == 0 && n != 1) { |
| + t.Fatalf("logged %q", l) |
| + } |
| + } |
| +} |
diff --git a/internal/console/hostclaimops.go b/internal/console/hostclaimops.go| index 8e0a09f9b..152aa34c4 100644 |
| --- a/internal/console/hostclaimops.go |
| +++ b/internal/console/hostclaimops.go |
| @@ -51,6 +51,9 @@ func (s *Server) StartClaimRecoveryPass(ctx context.Context) { |
| if pub := capacityPublisherFromEnv(ctx, os.Getenv); pub != nil { |
| go s.runCapacityPass(ctx, pub) |
| } |
| + // Automatic enrollment of the boxes regions' groups launch |
| + // (region HA spec 2 §5 step 5, enrollpass.go). |
| + s.startEnrollPass(ctx) |
| } |
| t := time.NewTicker(claimRecoveryInterval) |
| defer t.Stop() |
diff --git a/internal/console/testdata/claim-authority-sites.txt b/internal/console/testdata/claim-authority-sites.txt| index 5c120d1f3..310cb54d3 100644 |
| --- a/internal/console/testdata/claim-authority-sites.txt |
| +++ b/internal/console/testdata/claim-authority-sites.txt |
| @@ -145,3 +145,6 @@ claim_loss_pages.go|pageClaimLossesAt|s.store.db.ExecContext(ctx, `INSERT INTO c |
| claim_loss_pages.go|pageClaimLossesAt|s.store.db.QueryRowContext(ctx, `SELECT last_key FROM claim_loss_pages WHERE rule = 'scan'`) |
| store_gate.go|OpenGated|s.db.Exec(`ALTER TABLE claim_loss_pages ADD COLUMN last_key TEXT NOT NULL DEFAULT ''`) |
| awsregionmetric.go|capacityPublisherFromEnv|cloudwatch.NewFromConfig(cfg, func(o *cloudwatch.Options) { o.Region = awsRegion }) |
| +enrollpass.go|startEnrollPass|newAutoEnroller(s.cfg.ClaimAuthority, members) |
| +enrollpass.go|startEnrollPass|runEnrollPass(ctx, e, enrollPassInterval, log.Printf) |
| +enrollpass.go|runEnrollPass|e.EnrollPass(ctx) |
diff --git a/internal/hostactors/autoenroll.go b/internal/hostactors/autoenroll.go| new file mode 100644 |
| index 000000000..365058d56 |
| --- /dev/null |
| +++ b/internal/hostactors/autoenroll.go |
| @@ -0,0 +1,379 @@ |
| +package hostactors |
| + |
| +import ( |
| + "context" |
| + "crypto/x509" |
| + "encoding/base64" |
| + "encoding/json" |
| + "errors" |
| + "fmt" |
| + "os" |
| + "slices" |
| + "sort" |
| + "strings" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/boxid" |
| + "github.com/carlosframework/platform/internal/hostfloor" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// The console's automatic enrollment pass (region HA spec 2 §5 step 5, |
| +// A2-1, H16, H28). A box its region's group launched publishes its signed |
| +// identity record (control/boxes/<ref>.json) and joins. For each such box |
| +// not yet in the roster, the pass takes the next step its state allows: |
| +// |
| +// 1. no invitation: begin an enrollment attempt (BeginEnrollmentAttempt), |
| +// then, on the next pass, confirm the instance's group membership with |
| +// the console's own read (CheckConfirmed, Members) and invite the box |
| +// (Invite) while that read is under a minute old; |
| +// 2. invited: wait for the box to arm its guard and publish its |
| +// guard-ready report (VerifyReport, the same check membership.go makes |
| +// when the box is admitted); |
| +// 3. guard-ready: enroll it (EnrollAutomatic), which reads membership |
| +// again, itself, immediately before the enrolling write. |
| +// |
| +// A read that fails is unknown: the pass asks again next time, with the |
| +// same attempt, and its wait does not grow. Only the cloud's positive |
| +// answer that the instance is not a current member (ErrNotMember), or a |
| +// read that was over a minute old at its write (ErrMembershipStale), |
| +// counts against the attempt: the pass then waits Patience before it |
| +// begins another, doubling each time, so the record of a box the group |
| +// has already replaced stops costing roster writes and describe calls. |
| +// The invitation never changes. |
| +// |
| +// The box takes its group's role in the enrolling write (enroll.go). Only |
| +// a deployment whose host floor is published is enrolled automatically: |
| +// a box can arm its guard only under one. |
| +// |
| +// Nothing here names a provider beyond the identity the roster already |
| +// rests on (H26, §25): the record is AWS's signed instance document, as |
| +// for every enrolled box on main, and Members is an interface. |
| + |
| +// AutoEnroller is the pass's configuration and its memory of attempts. |
| +type AutoEnroller struct { |
| + Directory Directory |
| + Signer release.Signer |
| + // Evidence reads boxes' reports and release identities (the console's |
| + // ExactEvidence). |
| + Evidence Evidence |
| + // Members is the console's own read of the cloud's group membership |
| + // (awsmembership in production). |
| + Members MembershipReader |
| + // Certs verify identity records (boxid.EmbeddedCerts in production). |
| + Certs map[string]*x509.Certificate |
| + // Namespace and Pool are what a box is enrolled into: the console's |
| + // own namespace, and the default pool. |
| + Namespace, Pool string |
| + Now func() time.Time |
| + // Patience is how long the pass waits after an attempt is refused |
| + // before it begins another. Each refused attempt doubles the wait, up |
| + // to 32 times Patience, so a record whose instance is gone (a box the |
| + // group replaced while it was joining) costs the roster a write an |
| + // hour, not one a minute. An invitation resets it. |
| + Patience time.Duration |
| + // Logf records each membership read's request ids (C2's audit: the |
| + // roster has no field for them). nil is silent. |
| + Logf func(format string, args ...any) |
| + |
| + attempts map[string]attemptSeen |
| +} |
| + |
| +// attemptSeen is what the pass remembers of a box's current attempt: its |
| +// id, when the pass first saw it, how many attempts it has begun for the |
| +// box since the last invitation, and why the cloud refused the current |
| +// one ("" while it has not). Memory only: a console restart starts every |
| +// box afresh. |
| +type attemptSeen struct { |
| + id string |
| + since time.Time |
| + tries int |
| + refused string |
| +} |
| + |
| +// wait is how long a refused attempt is kept after tries attempts. |
| +func (e *AutoEnroller) wait(tries int) time.Duration { |
| + return e.Patience << min(max(tries-1, 0), 5) |
| +} |
| + |
| +// EnrollStep is what one pass did, or is waiting for, for one box. |
| +type EnrollStep struct { |
| + Box string |
| + Step string |
| +} |
| + |
| +// ErrNoFloor is a deployment with no published host floor: no box can arm |
| +// its guard, so none is enrolled automatically. |
| +var ErrNoFloor = errors.New("hostactors: no host floor is published, so no box is enrolled automatically") |
| + |
| +// EnrollPass takes one step for every box whose identity record is |
| +// published and that the roster does not list. It returns the steps in |
| +// reference order, and the first error that stopped the pass as a whole. |
| +func (e *AutoEnroller) EnrollPass(ctx context.Context) ([]EnrollStep, error) { |
| + o, floor, err := e.inputs(ctx) |
| + if err != nil { |
| + return nil, err |
| + } |
| + var steps []EnrollStep |
| + keys, err := e.Directory.Store.List(ctx, boxid.Prefix) |
| + if err != nil { |
| + return nil, err |
| + } |
| + var refs []string |
| + for _, k := range keys { |
| + ref, ok := strings.CutSuffix(strings.TrimPrefix(k, boxid.Prefix), ".json") |
| + if !ok || strings.Contains(ref, "/") { |
| + continue |
| + } |
| + if _, listed := o.Roster.Boxes[ref]; !listed { |
| + refs = append(refs, ref) |
| + } |
| + } |
| + sort.Strings(refs) |
| + if e.attempts == nil { |
| + e.attempts = map[string]attemptSeen{} |
| + } |
| + for _, ref := range refs { |
| + next, step := e.step(ctx, o, floor, ref) |
| + if next.ETag != "" { |
| + o = next |
| + } |
| + steps = append(steps, EnrollStep{Box: ref, Step: step}) |
| + } |
| + for ref := range e.attempts { |
| + if !slices.Contains(refs, ref) { |
| + delete(e.attempts, ref) |
| + } |
| + } |
| + return steps, nil |
| +} |
| + |
| +// inputs reads the roster and the host floor it was published under. |
| +func (e *AutoEnroller) inputs(ctx context.Context) (Observed, hostfloor.Verified, error) { |
| + o, err := e.Directory.Read(ctx) |
| + if err != nil { |
| + return Observed{}, hostfloor.Verified{}, err |
| + } |
| + if o.Roster.Publication != "published" { |
| + return Observed{}, hostfloor.Verified{}, ErrNoFloor |
| + } |
| + floor, err := hostfloor.Read(ctx, e.Directory.Store, e.Directory.PublicKey, e.Directory.Bucket) |
| + if err != nil { |
| + return Observed{}, hostfloor.Verified{}, err |
| + } |
| + if floor.Digest != o.Roster.FloorSHA256 { |
| + return Observed{}, hostfloor.Verified{}, ErrRoster |
| + } |
| + return o, floor, nil |
| +} |
| + |
| +// step takes one box's next step. It returns the roster it wrote, if it |
| +// wrote one, and what it did or waits for. |
| +func (e *AutoEnroller) step(ctx context.Context, o Observed, floor hostfloor.Verified, ref string) (Observed, string) { |
| + d, now := e.Directory, e.Now() |
| + raw, err := boxid.Load(ctx, d.Store, ref) |
| + if err != nil { |
| + return Observed{}, fmt.Sprintf("its identity record could not be read: %v", err) |
| + } |
| + name, region, err := groupRegion(o.Roster, raw) |
| + if err != nil { |
| + return Observed{}, err.Error() |
| + } |
| + if !slices.Contains(region.Namespaces, e.Namespace) || !slices.Contains(region.Pools, e.Pool) { |
| + return Observed{}, fmt.Sprintf("%s does not serve namespace %s and pool %s", name, e.Namespace, e.Pool) |
| + } |
| + v := boxid.Verifier{Account: region.Account, Certs: e.Certs} |
| + expect := boxid.Expect{Box: ref, AWSRegion: region.AWSRegion} |
| + for other, b := range o.Roster.Boxes { |
| + expect.Roster = append(expect.Roster, boxid.Enrolled{Box: other, Cloud: b.Cloud}) |
| + } |
| + ev, err := v.Enroll(raw, expect) |
| + if err != nil { |
| + return Observed{}, fmt.Sprintf("its identity record does not verify: %v", err) |
| + } |
| + instance := ev.Cloud.InstanceID |
| + // A group box publishes under the reference its instance derives; |
| + // the pass derives it again and never takes the key the box chose |
| + // (census ruling region-ha-2b). |
| + if ref != GroupBoxReference(region.Account, instance) { |
| + return Observed{}, "its record is not under its group reference, so it is not enrolled automatically" |
| + } |
| + in := EnrollmentRequest{Reference: ref, Namespace: e.Namespace, Label: ref, Region: name, Pool: e.Pool, AWSRegion: region.AWSRegion} |
| + |
| + _, _, err = d.Store.Get(ctx, "control/host-actor-invitations/"+ref+".json") |
| + invited := err == nil |
| + if err != nil && !errors.Is(err, os.ErrNotExist) { |
| + return Observed{}, fmt.Sprintf("its invitation could not be read: %v", err) |
| + } |
| + if invited { |
| + inv, err := d.Invite(ctx, e.Signer, ref, raw) |
| + if err != nil { |
| + return Observed{}, fmt.Sprintf("its invitation is not for this record: %v", err) |
| + } |
| + // The report is read under the reference derived from the |
| + // attempt's proven instance (PrivateBoxReport, Group set). |
| + rep, err := e.Evidence.BoxReport(ctx, ref, Box{Namespace: e.Namespace, Label: ref, Group: region.GroupName, Cloud: ev.Cloud}) |
| + if err != nil { |
| + return Observed{}, "waiting for its guard-ready report" |
| + } |
| + if err := VerifyReport(ctx, e.Evidence, d.PublicKey, floor, rep, inv.Enrollment, instance, now); err != nil { |
| + return Observed{}, "waiting for its guard-ready report" |
| + } |
| + } |
| + a, ok := o.Roster.EnrollmentAttempts[ref] |
| + usable := ok && !a.Consumed && a.InstanceID == instance && a.EvidenceSHA256 == hostfloor.Digest(raw) && a.Region == name && |
| + a.Account == region.Account && a.AWSRegion == region.AWSRegion && a.GroupName == region.GroupName && a.GroupARN == region.GroupARN |
| + seen := e.attempts[ref] |
| + if usable { |
| + if seen.id != a.AttemptID { |
| + seen = attemptSeen{id: a.AttemptID, since: now, tries: seen.tries} |
| + e.attempts[ref] = seen |
| + } |
| + if seen.refused != "" { |
| + // The cloud refused this attempt: keep it until its wait is |
| + // over, asking nothing more, then begin another. |
| + if now.Sub(seen.since) < e.wait(seen.tries) { |
| + return Observed{}, "not confirmed (" + seen.refused + "); waiting before another attempt" |
| + } |
| + } else { |
| + in.AttemptID, in.ConfirmationID = a.AttemptID, a.ConfirmationID |
| + next, step, err := e.confirmAndWrite(ctx, o, v, in, raw, instance, invited, name, region) |
| + if err == nil { |
| + if !invited { |
| + seen.tries = 0 |
| + e.attempts[ref] = seen |
| + } else { |
| + delete(e.attempts, ref) |
| + } |
| + return next, step |
| + } |
| + if !errors.Is(err, ErrNotMember) && !errors.Is(err, ErrMembershipStale) { |
| + // Unknown: a failed read, or a write that failed. The same |
| + // attempt is asked again next time; the wait does not grow. |
| + return Observed{}, step |
| + } |
| + seen.refused, seen.since = err.Error(), now |
| + e.attempts[ref] = seen |
| + return Observed{}, "not confirmed (" + seen.refused + "); waiting before another attempt" |
| + } |
| + } |
| + next, a, err := d.BeginEnrollmentAttempt(ctx, e.Signer, o, in, raw, instance) |
| + if err != nil { |
| + return Observed{}, fmt.Sprintf("an attempt could not be begun: %v", err) |
| + } |
| + e.attempts[ref] = attemptSeen{id: a.AttemptID, since: now, tries: seen.tries + 1} |
| + return next, "began an attempt" |
| +} |
| + |
| +// confirmAndWrite reads the box's membership and makes the write it |
| +// allows: the invitation, or, for a box already invited and guard-ready, |
| +// the enrollment. The membership read is the console's own, made here, |
| +// immediately before the write. |
| +func (e *AutoEnroller) confirmAndWrite(ctx context.Context, o Observed, v boxid.Verifier, in EnrollmentRequest, raw []byte, instance string, invited bool, name string, region Region) (Observed, string, error) { |
| + d := e.Directory |
| + if invited { |
| + next, ev, err := d.EnrollAutomatic(ctx, e.Signer, o, v, in, raw, instance, e.Members, e.Now) |
| + // A write the deadline left unknown (ErrWriteDeadline) is settled |
| + // by the next pass's roster read: a box the roster lists is |
| + // enrolled, and the pass has nothing more to do for it; one it |
| + // does not list goes on with the same attempt, whose wait has not |
| + // grown. |
| + e.logRead(in, instance, ev, "enrolled", err) |
| + if err != nil { |
| + return Observed{}, fmt.Sprintf("it could not be enrolled: %v", err), err |
| + } |
| + return next, fmt.Sprintf("enrolled in %s with the group's role, %s", name, RoleName(region.Role)), nil |
| + } |
| + // The check is CheckConfirmed's own, made here through its unexported |
| + // core so the write is bound to the confirmation that read built, never |
| + // to one rebuilt from its exported evidence. |
| + _, c, err := d.checkConfirmed(ctx, o, v, in, raw, instance, e.Members, e.Now) |
| + ev := c.evidence() |
| + if err != nil { |
| + e.logRead(in, instance, ev, "", err) |
| + return Observed{}, fmt.Sprintf("its membership could not be confirmed: %v", err), err |
| + } |
| + // Invite is main's and sealed. It runs on the store bound to the |
| + // read's deadline (writeBound), the single enforcement point: its |
| + // record read, its signature's Put and any read-back are refused once |
| + // the read is a minute old; a call in flight then, or still unanswered |
| + // 15 seconds after it began, is cut off as unknown, and so is one that |
| + // answers only after the deadline. The next pass settles it from the |
| + // bucket. |
| + // Nothing that waits sits between the read and this call. |
| + bd, bctx, cancel := writeBound(ctx, d, c, e.Now) |
| + defer cancel() |
| + _, err = bd.Invite(bctx, e.Signer, in.Reference, raw) |
| + e.logRead(in, instance, ev, "invited", err) |
| + if err != nil { |
| + return Observed{}, fmt.Sprintf("it could not be invited: %v", err), err |
| + } |
| + return Observed{}, "invited", nil |
| +} |
| + |
| +// logRead records one membership read: the box, its attempt, its instance, |
| +// the cloud's two request ids and the read's time, and what came of it |
| +// (C2's audit: the roster has no field for them). A call that failed |
| +// before the cloud answered has no request ids and says so. |
| +func (e *AutoEnroller) logRead(in EnrollmentRequest, instance string, ev MembershipEvidence, done string, err error) { |
| + outcome := done |
| + if err != nil { |
| + outcome = "not " + done + ": " + err.Error() |
| + if done == "" { |
| + outcome = err.Error() |
| + } |
| + } |
| + e.logOutcome(in, instance, ev, outcome) |
| +} |
| + |
| +// logOutcome is one membership log line (S5): the box, its attempt, its |
| +// instance, both request ids and the read's time, and the outcome. |
| +func (e *AutoEnroller) logOutcome(in EnrollmentRequest, instance string, ev MembershipEvidence, outcome string) { |
| + if e.Logf == nil { |
| + return |
| + } |
| + ids := "none" |
| + if len(ev.RequestIDs) > 0 { |
| + ids = strings.Join(ev.RequestIDs, ", ") |
| + } |
| + at := "never" |
| + if !ev.ObservedAt.IsZero() { |
| + at = ev.ObservedAt.Format(time.RFC3339) |
| + } |
| + e.Logf("console: automatic box enrollment: membership read for %s (attempt %s, instance %s, request ids %s, read at %s): %s", in.Reference, in.AttemptID, instance, ids, at, outcome) |
| +} |
| + |
| +// groupRegion is the one roster region whose group could have launched |
| +// the box the record names: its account and AWS region, with a group. |
| +// The record is read here only to choose; Verifier.Enroll then proves it. |
| +func groupRegion(r Roster, raw []byte) (string, Region, error) { |
| + var rec boxid.Record |
| + if json.Unmarshal(raw, &rec) != nil { |
| + return "", Region{}, errors.New("its identity record is not JSON") |
| + } |
| + doc, err := base64.StdEncoding.DecodeString(rec.IdentityDocument) |
| + if err != nil { |
| + return "", Region{}, errors.New("its identity document is not base64") |
| + } |
| + var id struct { |
| + AccountID string `json:"accountId"` |
| + Region string `json:"region"` |
| + } |
| + if json.Unmarshal(doc, &id) != nil { |
| + return "", Region{}, errors.New("its identity document is not JSON") |
| + } |
| + var names []string |
| + for name, g := range r.Regions { |
| + if g.GroupName != "" && g.Account == id.AccountID && g.AWSRegion == id.Region { |
| + names = append(names, name) |
| + } |
| + } |
| + switch len(names) { |
| + case 0: |
| + return "", Region{}, fmt.Errorf("no region's group runs in %s for account %s", id.Region, id.AccountID) |
| + case 1: |
| + return names[0], r.Regions[names[0]], nil |
| + } |
| + sort.Strings(names) |
| + return "", Region{}, fmt.Errorf("the regions %s all have a group in %s", strings.Join(names, ", "), id.Region) |
| +} |
diff --git a/internal/hostactors/autoenroll_test.go b/internal/hostactors/autoenroll_test.go| new file mode 100644 |
| index 000000000..99baa9636 |
| --- /dev/null |
| +++ b/internal/hostactors/autoenroll_test.go |
| @@ -0,0 +1,734 @@ |
| +package hostactors |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "fmt" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/boxid" |
| + "github.com/carlosframework/platform/internal/boxid/boxidtest" |
| + "github.com/carlosframework/platform/internal/release" |
| + "github.com/carlosframework/platform/internal/status" |
| +) |
| + |
| +// enrollWorld is plan9Fixture's published deployment (boxes a, b and w, |
| +// active in ie under a published host floor) with ie's group, and a box |
| +// (instance i-new, reference newRef) that the group launched: its |
| +// identity record is published, and nothing else. The console reads the |
| +// cloud through cloud. |
| +type enrollWorld struct { |
| + plan9World |
| + e *AutoEnroller |
| + cloud *fakeMembership |
| + signer *boxidtest.Signer |
| + clock time.Time |
| + logs []string |
| +} |
| + |
| +func newEnrollWorld(t *testing.T, role string) *enrollWorld { |
| + t.Helper() |
| + w, _ := plan9Fixture(t, func(r *Roster) { |
| + r.Regions = map[string]Region{"ie": {Account: "123", AWSRegion: "eu-west-1", GroupName: "routers-ie", |
| + GroupARN: "arn:aws:autoscaling:eu-west-1:123:autoScalingGroup:x:autoScalingGroupName/routers-ie", Namespaces: []string{"platform"}, Pools: []string{"default"}, Role: role}} |
| + }) |
| + signer := boxidtest.NewSigner(t) |
| + _, raw := signer.Record(t, newRef, "123", "eu-west-1", "i-new") |
| + if _, err := w.d.Store.Put(context.Background(), boxid.Key(newRef), raw, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + ew := &enrollWorld{plan9World: w, clock: time.Now().UTC(), signer: signer, |
| + cloud: &fakeMembership{members: map[string]Membership{"i-new": member("routers-ie")}}} |
| + ew.e = &AutoEnroller{Directory: w.d, Signer: w.sg, Evidence: ExactEvidence{PublicKey: w.d.PublicKey, Store: w.d.Store, Boxes: w.d.PrivateBoxReport}, |
| + Members: ew.cloud, Certs: signer.Certs(), Namespace: "platform", Pool: "default", Now: func() time.Time { return ew.clock }, Patience: 2 * time.Minute, |
| + Logf: func(format string, args ...any) { ew.logs = append(ew.logs, fmt.Sprintf(format, args...)) }} |
| + return ew |
| +} |
| + |
| +// pass runs the enrollment pass and returns new's step. |
| +func (w *enrollWorld) pass(t *testing.T) string { |
| + t.Helper() |
| + steps, err := w.e.EnrollPass(context.Background()) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, s := range steps { |
| + if s.Box == newRef { |
| + return s.Step |
| + } |
| + } |
| + return "" |
| +} |
| + |
| +// guardReady is the joining box, its guard armed, publishing the report |
| +// every hosting box publishes. |
| +func (w *enrollWorld) guardReady(t *testing.T, enrollment string, ready bool) { |
| + t.Helper() |
| + box := status.Box{GeneratedAt: time.Now().UTC(), Metrics: &status.BoxMetrics{Enrollment: enrollment, InstanceID: "i-new", Release: "v1.0.0", ReleaseTag: "v1.0.0", |
| + Platform: "linux-amd64", BinarySHA256: w.hash, MovesProtocol: 1, BootID: "boot", FloorSHA256: w.floor.Digest, FloorGuardReady: ready, SystemChannel: "stable"}} |
| + raw, _ := json.Marshal(box) |
| + _, etag, _ := w.d.Store.Get(context.Background(), "status/boxes/"+newRef+".json") |
| + if _, err := w.d.Store.Put(context.Background(), "status/boxes/"+newRef+".json", raw, etag); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +func (w *enrollWorld) invitation(t *testing.T) (Invitation, bool) { |
| + t.Helper() |
| + if _, _, err := w.d.Store.Get(context.Background(), "control/host-actor-invitations/"+newRef+".json"); err != nil { |
| + return Invitation{}, false |
| + } |
| + raw, err := boxid.Load(context.Background(), w.d.Store, newRef) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + // Invite returns the stored invitation, verified, and writes nothing. |
| + got, err := w.d.Invite(context.Background(), w.sg, newRef, raw) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return got, true |
| +} |
| + |
| +func (w *enrollWorld) attempt(t *testing.T) EnrollmentAttempt { |
| + t.Helper() |
| + o, err := w.d.Read(context.Background()) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return o.Roster.EnrollmentAttempts[newRef] |
| +} |
| + |
| +// The whole sequence (spec 2 §5 step 5, §11): attempt, the console's own |
| +// membership read and the invitation, guard-ready, the console's second |
| +// read and the enrollment. The deployment's floor is published, so the |
| +// enrolling write passes membership.go's rule only because the guard-ready |
| +// report is there; the box takes its group's role in it. |
| +func TestAutomaticEnrollmentFromRecordToRoster(t *testing.T) { |
| + w := newEnrollWorld(t, RoleLegacy) |
| + if got := w.pass(t); got != "began an attempt" { |
| + t.Fatalf("first pass: %q", got) |
| + } |
| + if _, ok := w.invitation(t); ok || len(w.cloud.calls) != 0 { |
| + t.Fatalf("invited before membership was read: calls %v", w.cloud.calls) |
| + } |
| + if got := w.pass(t); got != "invited" { |
| + t.Fatalf("second pass: %q", got) |
| + } |
| + inv, ok := w.invitation(t) |
| + if !ok || len(w.cloud.calls) != 1 { |
| + t.Fatalf("invitation %v, calls %v", ok, w.cloud.calls) |
| + } |
| + // Invited, but the guard is not armed: nothing more, and no read. |
| + // First no report at all, then a report from a box whose guard is not |
| + // ready. |
| + if got := w.pass(t); got != "waiting for its guard-ready report" { |
| + t.Fatalf("invited, no report: %q", got) |
| + } |
| + w.guardReady(t, inv.Enrollment, false) |
| + if got := w.pass(t); got != "waiting for its guard-ready report" || len(w.cloud.calls) != 1 { |
| + t.Fatalf("invited, guard not ready: %q, calls %v", got, w.cloud.calls) |
| + } |
| + w.guardReady(t, inv.Enrollment, true) |
| + w.clock = w.clock.Add(5 * time.Minute) |
| + if got := w.pass(t); !strings.HasPrefix(got, "enrolled in ie") { |
| + t.Fatalf("ready: %q", got) |
| + } |
| + // The enrollment read membership again, itself. |
| + if len(w.cloud.calls) != 2 || len(w.logs) != 2 || !strings.Contains(w.logs[1], "attempt "+w.attempt(t).AttemptID+", instance i-new, request ids asg-req, ec2-req") || !strings.HasSuffix(w.logs[1], "): enrolled") { |
| + t.Fatalf("calls %v, logs %q", w.cloud.calls, w.logs) |
| + } |
| + o, err := w.d.Read(context.Background()) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + b, ok := o.Roster.Boxes[newRef] |
| + if !ok || b.State != "active" || b.Enrollment != inv.Enrollment || b.Cloud.InstanceID != "i-new" || b.Group != "routers-ie" || b.Role != RoleLegacy || b.Namespace != "platform" || b.Pool != "default" { |
| + t.Fatalf("roster entry %+v", b) |
| + } |
| + if !o.Roster.EnrollmentAttempts[newRef].Consumed { |
| + t.Fatal("the attempt was not consumed") |
| + } |
| + // Enrolled: the pass has nothing left to do for it. |
| + if got := w.pass(t); got != "" { |
| + t.Fatalf("enrolled box stepped again: %q", got) |
| + } |
| +} |
| + |
| +// R1: an instance in the deployment's account that the cloud does not |
| +// place in the region's group is never invited and never enrolled, however |
| +// long it waits; and one that leaves the group after its invitation is not |
| +// enrolled on a guard-ready report. |
| +func TestAnInstanceOutsideTheGroupIsNeverInvitedOrEnrolled(t *testing.T) { |
| + for name, m := range map[string]Membership{ |
| + "another group": member("routers-uk"), |
| + "no group": {Owner: "123", InstanceState: "running", AvailabilityZoneRegion: "eu-west-1", RequestIDs: []string{"a", "b"}}, |
| + } { |
| + t.Run(name, func(t *testing.T) { |
| + w := newEnrollWorld(t, "") |
| + w.cloud.members["i-new"] = m |
| + for i := 0; i < 12; i++ { |
| + w.pass(t) |
| + w.clock = w.clock.Add(time.Minute) |
| + } |
| + if _, ok := w.invitation(t); ok { |
| + t.Fatal("an instance outside the group was invited") |
| + } |
| + if o, _ := w.d.Read(context.Background()); o.Roster.Boxes[newRef].Enrollment != "" { |
| + t.Fatal("an instance outside the group was enrolled") |
| + } |
| + }) |
| + } |
| + w := newEnrollWorld(t, "") |
| + w.pass(t) |
| + if got := w.pass(t); got != "invited" { |
| + t.Fatal(got) |
| + } |
| + inv, _ := w.invitation(t) |
| + w.guardReady(t, inv.Enrollment, true) |
| + w.cloud.members["i-new"] = func() Membership { m := member("routers-ie"); m.LifecycleState = "Detaching"; return m }() |
| + if got := w.pass(t); !strings.Contains(got, "not confirmed") { |
| + t.Fatalf("left the group: %q", got) |
| + } |
| + if o, _ := w.d.Read(context.Background()); o.Roster.Boxes[newRef].Enrollment != "" { |
| + t.Fatal("an instance that left the group was enrolled") |
| + } |
| +} |
| + |
| +// R1 (#7): a read that fails is unknown. The attempt stands, its wait does |
| +// not grow, and the next read that answers goes on from it. |
| +func TestAMembershipReadThatFailsKeepsTheAttempt(t *testing.T) { |
| + w := newEnrollWorld(t, "") |
| + w.pass(t) |
| + first := w.attempt(t).AttemptID |
| + w.cloud.err = errors.New("Throttling: rate exceeded") |
| + for i := 0; i < 10; i++ { |
| + w.clock = w.clock.Add(time.Hour) |
| + if got := w.pass(t); !strings.Contains(got, "could not be read") { |
| + t.Fatalf("failed read: %q", got) |
| + } |
| + } |
| + if got := w.attempt(t).AttemptID; got != first || w.e.attempts[newRef].tries != 1 { |
| + t.Fatalf("a failed read superseded the attempt (%s, was %s) or grew its wait (%d)", got, first, w.e.attempts[newRef].tries) |
| + } |
| + w.cloud.err = nil |
| + if got := w.pass(t); got != "invited" || w.attempt(t).AttemptID != first { |
| + t.Fatalf("after the outage: %q", got) |
| + } |
| +} |
| + |
| +// C2: the invitation is written only while the console's read is under a |
| +// minute old. Here every look at the clock takes 31 seconds: the read is |
| +// fresh when CheckConfirmed has it, and over a minute old by the time the |
| +// invitation would be written. That is unknown: the attempt and its wait |
| +// stand. |
| +func TestAnInvitationNeedsAReadUnderAMinuteOldAtTheWrite(t *testing.T) { |
| + w := newEnrollWorld(t, "") |
| + w.pass(t) |
| + first := w.attempt(t).AttemptID |
| + w.e.Now = func() time.Time { w.clock = w.clock.Add(31 * time.Second); return w.clock } |
| + if got := w.pass(t); !strings.Contains(got, "more than a minute after its membership read") { |
| + t.Fatalf("slow pass: %q", got) |
| + } |
| + if _, ok := w.invitation(t); ok { |
| + t.Fatal("invited on a read over a minute old") |
| + } |
| + if w.attempt(t).AttemptID != first || w.e.attempts[newRef].tries != 1 || w.e.attempts[newRef].refused != "" { |
| + t.Fatalf("a missed deadline counted against the attempt: %+v", w.e.attempts[newRef]) |
| + } |
| +} |
| + |
| +// slowStore is the deployment bucket with one slow read: the first Get of |
| +// slow after armed is set moves the console's clock on by exactly 60 |
| +// seconds, to the deadline itself, as a bucket that answers late would. It honours its context, as the S3 |
| +// store does, and records every Put. |
| +type slowStore struct { |
| + release.Store |
| + slow string |
| + armed bool |
| + late func() |
| + puts []string |
| +} |
| + |
| +func (s *slowStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| + if err := ctx.Err(); err != nil { |
| + return nil, "", err |
| + } |
| + if s.armed && key == s.slow { |
| + s.armed = false |
| + s.late() |
| + } |
| + return s.Store.Get(ctx, key) |
| +} |
| + |
| +func (s *slowStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { |
| + if err := ctx.Err(); err != nil { |
| + return "", err |
| + } |
| + s.puts = append(s.puts, key) |
| + return s.Store.Put(ctx, key, body, ifMatch) |
| +} |
| + |
| +// slowAfterRead makes the store slow on key after the console's next |
| +// membership read. |
| +func (w *enrollWorld) slowAfterRead(key string) *slowStore { |
| + ss := &slowStore{Store: w.e.Directory.Store, slow: key, late: func() { w.clock = w.clock.Add(60 * time.Second) }} |
| + w.e.Directory.Store = ss |
| + w.cloud.during = func() { ss.armed = true } |
| + return ss |
| +} |
| + |
| +// S1: a read inside the sealed Invite that answers late (here, its own |
| +// read of the invitation key) leaves the invitation unwritten: no Put is |
| +// issued after the deadline. The attempt and its wait stand, and the next |
| +// pass, with a fresh read, invites. |
| +func TestASlowReadInsideInviteWritesNothing(t *testing.T) { |
| + w := newEnrollWorld(t, "") |
| + w.pass(t) |
| + first := w.attempt(t).AttemptID |
| + ss := w.slowAfterRead("control/host-actor-invitations/" + newRef + ".json") |
| + if got := w.pass(t); !strings.Contains(got, "more than a minute after its membership read") { |
| + t.Fatalf("slow invite: %q", got) |
| + } |
| + if len(ss.puts) != 0 { |
| + t.Fatalf("written after the deadline: %v", ss.puts) |
| + } |
| + if _, ok := w.invitation(t); ok || w.attempt(t).AttemptID != first || w.e.attempts[newRef].tries != 1 { |
| + t.Fatalf("invited %v, attempt %+v", ok, w.e.attempts[newRef]) |
| + } |
| + w.cloud.during = nil |
| + if got := w.pass(t); got != "invited" || w.attempt(t).AttemptID != first { |
| + t.Fatalf("next pass: %q", got) |
| + } |
| +} |
| + |
| +// S1: a read inside the sealed checkMembership that answers late (the |
| +// guard-ready report Replace checks before its CAS) leaves the roster |
| +// unwritten. The attempt stays unconsumed, and the next pass enrolls. |
| +func TestASlowReadInsideTheRosterCheckWritesNothing(t *testing.T) { |
| + w := newEnrollWorld(t, "") |
| + w.pass(t) |
| + if got := w.pass(t); got != "invited" { |
| + t.Fatal(got) |
| + } |
| + inv, _ := w.invitation(t) |
| + w.guardReady(t, inv.Enrollment, true) |
| + first, tries := w.attempt(t).AttemptID, w.e.attempts[newRef].tries |
| + ss := w.slowAfterRead("status/boxes/" + newRef + ".json") |
| + if got := w.pass(t); !strings.Contains(got, "more than a minute after its membership read") { |
| + t.Fatalf("slow enrolment: %q", got) |
| + } |
| + if len(ss.puts) != 0 { |
| + t.Fatalf("written after the deadline: %v", ss.puts) |
| + } |
| + if a := w.attempt(t); a.AttemptID != first || a.Consumed || w.e.attempts[newRef].tries != tries || w.e.attempts[newRef].refused != "" { |
| + t.Fatalf("attempt %+v, seen %+v", a, w.e.attempts[newRef]) |
| + } |
| + if o, _ := w.d.Read(context.Background()); o.Roster.Boxes[newRef].Enrollment != "" { |
| + t.Fatal("enrolled after the deadline") |
| + } |
| + w.cloud.during = nil |
| + w.guardReady(t, inv.Enrollment, true) |
| + if got := w.pass(t); !strings.HasPrefix(got, "enrolled in ie") { |
| + t.Fatalf("next pass: %q", got) |
| + } |
| +} |
| + |
| +// The ruling's second test: a box cannot satisfy another box's report |
| +// check. A report under newRef's key that another instance wrote is not |
| +// newRef's guard-ready report, and a record under a key that is not its |
| +// instance's group reference is never invited. |
| +func TestABoxCannotStandInForAnother(t *testing.T) { |
| + w := newEnrollWorld(t, "") |
| + w.pass(t) |
| + if got := w.pass(t); got != "invited" { |
| + t.Fatal(got) |
| + } |
| + inv, _ := w.invitation(t) |
| + box := status.Box{GeneratedAt: time.Now().UTC(), Metrics: &status.BoxMetrics{Enrollment: inv.Enrollment, InstanceID: "i-other", Release: "v1.0.0", ReleaseTag: "v1.0.0", |
| + Platform: "linux-amd64", BinarySHA256: w.hash, MovesProtocol: 1, BootID: "boot", FloorSHA256: w.floor.Digest, FloorGuardReady: true, SystemChannel: "stable"}} |
| + raw, _ := json.Marshal(box) |
| + if _, err := w.d.Store.Put(context.Background(), "status/boxes/"+newRef+".json", raw, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + if got := w.pass(t); got != "waiting for its guard-ready report" { |
| + t.Fatalf("another instance's report: %q", got) |
| + } |
| + // A record its box published under a key of its own choosing. |
| + v := newEnrollWorld(t, "") |
| + _, chosen := v.signer.Record(t, "chosen", "123", "eu-west-1", "i-chosen") |
| + if _, err := v.d.Store.Put(context.Background(), boxid.Key("chosen"), chosen, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + steps, err := v.e.EnrollPass(context.Background()) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, s := range steps { |
| + if s.Box == "chosen" && !strings.Contains(s.Step, "not under its group reference") { |
| + t.Fatalf("chosen key: %q", s.Step) |
| + } |
| + } |
| + if _, _, err := v.d.Store.Get(context.Background(), "control/host-actor-invitations/chosen.json"); err == nil { |
| + t.Fatal("a box with a chosen key was invited") |
| + } |
| +} |
| + |
| +// inflightStore is the bucket with one slow Put: the first Put of key |
| +// after armed is set commits (when commit is set) and then, by mode, |
| +// waits for its context as an S3 request cut off by the deadline does |
| +// ("cancelled"), ignores its context and never answers ("ignores"), or |
| +// answers success only after the console's clock has passed the deadline |
| +// ("late"). In mode "delayed" it commits nothing until hang is closed, |
| +// whatever its context says, then commits and reports on done: a write |
| +// the console gave up on that the bucket applies later. |
| +type inflightStore struct { |
| + release.Store |
| + key string |
| + armed, commit bool |
| + mode string |
| + late func() |
| + hang chan struct{} |
| + done chan error |
| +} |
| + |
| +func (s *inflightStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { |
| + if s.armed && key == s.key { |
| + s.armed = false |
| + if s.mode == "delayed" { |
| + <-s.hang |
| + etag, err := s.Store.Put(ctx, key, body, ifMatch) |
| + s.done <- err |
| + return etag, err |
| + } |
| + etag := "" |
| + if s.commit { |
| + var err error |
| + if etag, err = s.Store.Put(ctx, key, body, ifMatch); err != nil { |
| + return "", err |
| + } |
| + } |
| + switch s.mode { |
| + case "ignores": |
| + <-s.hang |
| + return etag, nil |
| + case "late": |
| + s.late() |
| + return etag, nil |
| + } |
| + <-ctx.Done() |
| + return "", ctx.Err() |
| + } |
| + return s.Store.Put(ctx, key, body, ifMatch) |
| +} |
| + |
| +// inflightAfterRead makes key's next Put slow in mode: the next |
| +// membership read takes 59.95 seconds, so its write has 50 milliseconds, |
| +// and a late answer comes a second after that. |
| +func (w *enrollWorld) inflightAfterRead(key string, commit bool, mode string) *inflightStore { |
| + is := &inflightStore{Store: w.e.Directory.Store, key: key, commit: commit, mode: mode, hang: make(chan struct{}), done: make(chan error, 1), |
| + late: func() { w.clock = w.clock.Add(time.Second) }} |
| + w.e.Directory.Store = is |
| + w.cloud.during = func() { w.clock = w.clock.Add(59950 * time.Millisecond); is.armed = true } |
| + return is |
| +} |
| + |
| +// S1: a Put still in flight at the deadline is cancelled there, and its |
| +// result is unknown: the attempt and its wait stand, and the next pass |
| +// reads the bucket as it does every pass. Here the invitation did not |
| +// land, then (a second time) it did, and the next pass finds it. |
| +func TestAPutInFlightAtTheDeadlineIsUnknown(t *testing.T) { |
| + w := newEnrollWorld(t, "") |
| + w.pass(t) |
| + first, tries := w.attempt(t).AttemptID, w.e.attempts[newRef].tries |
| + w.inflightAfterRead("control/host-actor-invitations/"+newRef+".json", false, "cancelled") |
| + if got := w.pass(t); !strings.Contains(got, "in flight at the deadline") { |
| + t.Fatalf("in flight: %q", got) |
| + } |
| + if _, ok := w.invitation(t); ok || w.attempt(t).AttemptID != first || w.e.attempts[newRef].tries != tries || w.e.attempts[newRef].refused != "" { |
| + t.Fatalf("invited %v, seen %+v", ok, w.e.attempts[newRef]) |
| + } |
| + w.inflightAfterRead("control/host-actor-invitations/"+newRef+".json", true, "cancelled") |
| + if got := w.pass(t); !strings.Contains(got, "in flight at the deadline") { |
| + t.Fatalf("in flight, landed: %q", got) |
| + } |
| + w.cloud.during = nil |
| + if got := w.pass(t); got != "waiting for its guard-ready report" { |
| + t.Fatalf("read back: %q", got) |
| + } |
| +} |
| + |
| +// U2: a store that ignores its context and never answers is cut off by |
| +// the hard per-call timeout, well before the deadline: the outcome is |
| +// unknown, the attempt and its wait stand, and nothing was invited. |
| +func TestAStoreThatIgnoresItsContextIsCutOffAtTheHardTimeout(t *testing.T) { |
| + was := storeCallTimeout |
| + storeCallTimeout = 100 * time.Millisecond |
| + t.Cleanup(func() { storeCallTimeout = was }) |
| + w := newEnrollWorld(t, "") |
| + w.pass(t) |
| + first, tries := w.attempt(t).AttemptID, w.e.attempts[newRef].tries |
| + is := &inflightStore{Store: w.e.Directory.Store, key: "control/host-actor-invitations/" + newRef + ".json", mode: "ignores", hang: make(chan struct{}), armed: true} |
| + t.Cleanup(func() { close(is.hang) }) |
| + w.e.Directory.Store = is |
| + start := time.Now() |
| + got := w.pass(t) |
| + if !strings.Contains(got, "did not answer in time") || time.Since(start) > 5*time.Second { |
| + t.Fatalf("after %s: %q", time.Since(start), got) |
| + } |
| + if _, ok := w.invitation(t); ok || w.attempt(t).AttemptID != first || w.e.attempts[newRef].tries != tries || w.e.attempts[newRef].refused != "" { |
| + t.Fatalf("invited %v, seen %+v", ok, w.e.attempts[newRef]) |
| + } |
| +} |
| + |
| +// U1, U2: an enrolling write whose outcome is unknown is settled by the |
| +// next pass's roster read, with nothing more. Not landed: the same |
| +// attempt, its wait unchanged, enrols on the next pass. Landed, whether |
| +// cut off in flight or answering success only after the deadline: the |
| +// roster lists the box, and the pass has nothing left to do for it. |
| +func TestAnUnknownEnrolmentIsSettledByTheRoster(t *testing.T) { |
| + ready := func(t *testing.T) *enrollWorld { |
| + w := newEnrollWorld(t, "") |
| + w.pass(t) |
| + if got := w.pass(t); got != "invited" { |
| + t.Fatal(got) |
| + } |
| + inv, _ := w.invitation(t) |
| + w.guardReady(t, inv.Enrollment, true) |
| + return w |
| + } |
| + t.Run("did not land", func(t *testing.T) { |
| + w := ready(t) |
| + first, tries := w.attempt(t).AttemptID, w.e.attempts[newRef].tries |
| + w.inflightAfterRead(Key, false, "cancelled") |
| + if got := w.pass(t); !strings.Contains(got, "in flight at the deadline") { |
| + t.Fatalf("in flight: %q", got) |
| + } |
| + if a := w.attempt(t); a.Consumed || a.AttemptID != first || w.e.attempts[newRef].tries != tries { |
| + t.Fatalf("attempt %+v, seen %+v", a, w.e.attempts[newRef]) |
| + } |
| + w.cloud.during = nil |
| + if got := w.pass(t); !strings.HasPrefix(got, "enrolled in ie") { |
| + t.Fatalf("next pass: %q", got) |
| + } |
| + }) |
| + for name, mode := range map[string]string{"landed, cut off in flight": "cancelled", "landed, answered after the deadline": "late"} { |
| + t.Run(name, func(t *testing.T) { |
| + w := ready(t) |
| + w.inflightAfterRead(Key, true, mode) |
| + want := map[string]string{"cancelled": "in flight at the deadline", "late": "answered only after the deadline"}[mode] |
| + if got := w.pass(t); !strings.Contains(got, want) { |
| + t.Fatalf("unknown: %q", got) |
| + } |
| + w.cloud.during = nil |
| + calls := len(w.cloud.calls) |
| + steps, err := w.e.EnrollPass(context.Background()) |
| + if err != nil || len(steps) != 0 || len(w.cloud.calls) != calls { |
| + t.Fatalf("settled by the roster: %+v %v, calls %v", steps, err, w.cloud.calls) |
| + } |
| + o, _ := w.d.Read(context.Background()) |
| + if b := o.Roster.Boxes[newRef]; b.State != "active" || !o.Roster.EnrollmentAttempts[newRef].Consumed { |
| + t.Fatalf("roster %+v", b) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// V1, the accepted exposure: the console waits for no write past its |
| +// bound, but a write it gave up on can still commit. Here the enrolling |
| +// Put is issued before the deadline and blocks; the pass returns unknown; |
| +// the instance is detached from its group (still running); the next pass |
| +// finds the box absent from the roster, and its fresh describe refuses |
| +// without writing anything; then the bucket applies the delayed Put, on |
| +// the roster revision it was built against, which nothing has moved. The |
| +// pass after that finds the box enrolled and does nothing more. Such an |
| +// enrolment rests on a membership read that was true when its write was |
| +// issued: the same exposure as an instance detached after enrolment, |
| +// which spec 3's regional worker reconciles. |
| +func TestAWriteTheConsoleGaveUpOnCanCommitLater(t *testing.T) { |
| + w := newEnrollWorld(t, "") |
| + w.pass(t) |
| + if got := w.pass(t); got != "invited" { |
| + t.Fatal(got) |
| + } |
| + inv, _ := w.invitation(t) |
| + w.guardReady(t, inv.Enrollment, true) |
| + before, _ := w.d.Read(context.Background()) |
| + is := w.inflightAfterRead(Key, false, "delayed") |
| + if got := w.pass(t); !strings.Contains(got, "in flight at the deadline") { |
| + t.Fatalf("unknown: %q", got) |
| + } |
| + // Detached: the group no longer lists the instance, which still runs. |
| + w.cloud.during = nil |
| + w.cloud.members["i-new"] = member("") |
| + calls := len(w.cloud.calls) |
| + if got := w.pass(t); !strings.Contains(got, "not confirmed") || len(w.cloud.calls) != calls+1 { |
| + t.Fatalf("absent, then refused: %q, calls %v", got, w.cloud.calls) |
| + } |
| + if o, _ := w.d.Read(context.Background()); o.ETag != before.ETag { |
| + t.Fatalf("the refusing pass wrote the roster: %s, was %s", o.ETag, before.ETag) |
| + } |
| + close(is.hang) |
| + if err := <-is.done; err != nil { |
| + t.Fatalf("the delayed write did not commit: %v", err) |
| + } |
| + calls = len(w.cloud.calls) |
| + steps, err := w.e.EnrollPass(context.Background()) |
| + if err != nil || len(steps) != 0 || len(w.cloud.calls) != calls { |
| + t.Fatalf("enrolled: %+v %v, calls %v", steps, err, w.cloud.calls) |
| + } |
| + o, _ := w.d.Read(context.Background()) |
| + if b := o.Roster.Boxes[newRef]; b.State != "active" || o.Roster.Revision != before.Roster.Revision+1 { |
| + t.Fatalf("roster %+v, revision %d", b, o.Roster.Revision) |
| + } |
| +} |
| + |
| +// A refused attempt is kept, with no more reads, for its wait, then begun |
| +// again; each refusal doubles the wait, up to 32 times Patience. |
| +func TestARefusedAttemptIsBegunAgainWithADoublingWait(t *testing.T) { |
| + w := newEnrollWorld(t, "") |
| + w.cloud.members["i-new"] = member("") |
| + if got := w.pass(t); got != "began an attempt" { |
| + t.Fatal(got) |
| + } |
| + first := w.attempt(t).AttemptID |
| + if got := w.pass(t); !strings.Contains(got, "not confirmed") { |
| + t.Fatal(got) |
| + } |
| + w.clock = w.clock.Add(time.Minute + 59*time.Second) |
| + if got := w.pass(t); !strings.Contains(got, "waiting before another attempt") || len(w.cloud.calls) != 1 { |
| + t.Fatalf("inside the wait: %q, calls %v", got, w.cloud.calls) |
| + } |
| + w.clock = w.clock.Add(time.Second) |
| + if got := w.pass(t); got != "began an attempt" || w.attempt(t).AttemptID == first { |
| + t.Fatalf("after two minutes: %q", got) |
| + } |
| + if got := w.pass(t); !strings.Contains(got, "not confirmed") { |
| + t.Fatal(got) |
| + } |
| + // The second refusal waits four minutes. |
| + w.clock = w.clock.Add(3*time.Minute + 59*time.Second) |
| + if got := w.pass(t); !strings.Contains(got, "waiting before another attempt") { |
| + t.Fatalf("second attempt before its four minutes: %q", got) |
| + } |
| + w.clock = w.clock.Add(time.Second) |
| + if got := w.pass(t); got != "began an attempt" { |
| + t.Fatalf("second attempt after four minutes: %q", got) |
| + } |
| + if got := w.e.wait(10); got != 64*time.Minute { |
| + t.Fatalf("the wait is not capped: %s", got) |
| + } |
| +} |
| + |
| +func TestAutomaticEnrollmentRefusesWhatItCannotProve(t *testing.T) { |
| + // A box from an account or region with no group is left alone. |
| + w := newEnrollWorld(t, "") |
| + _, raw := boxidtest.NewSigner(t).Record(t, "elsewhere", "999", "eu-west-1", "i-x") |
| + if _, err := w.d.Store.Put(context.Background(), boxid.Key("elsewhere"), raw, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + steps, err := w.e.EnrollPass(context.Background()) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, s := range steps { |
| + if s.Box == "elsewhere" && !strings.Contains(s.Step, "no region's group") { |
| + t.Fatalf("elsewhere: %q", s.Step) |
| + } |
| + } |
| + // A record whose signature does not verify under the certificates. |
| + w.e.Certs = boxidtest.NewSigner(t).Certs() |
| + if got := w.pass(t); !strings.Contains(got, "does not verify") { |
| + t.Fatalf("unverified record: %q", got) |
| + } |
| + // A deployment with no published floor enrolls nothing. |
| + d, sg, _ := fixture(t) |
| + unpublished := &AutoEnroller{Directory: d, Signer: sg, Namespace: "platform", Pool: "default", Now: time.Now} |
| + if _, err := unpublished.EnrollPass(context.Background()); !errors.Is(err, ErrNoFloor) { |
| + t.Fatalf("unpublished: %v", err) |
| + } |
| +} |
| + |
| +// S5: every membership read is logged with the box, its attempt, its |
| +// instance and the request ids the read's own confirmation kept, whatever |
| +// came of it: a failed read (the ids the cloud gave, "none" for a call it |
| +// never answered), a refusal, and an invitation. |
| +func TestEveryMembershipReadIsLoggedWithItsIds(t *testing.T) { |
| + w := newEnrollWorld(t, "") |
| + w.pass(t) |
| + id := w.attempt(t).AttemptID |
| + head := "membership read for " + newRef + " (attempt " + id + ", instance i-new, request ids " |
| + w.cloud.err, w.cloud.failIDs = errors.New("Throttling: rate exceeded"), []string{"asg-req", NoRequestID} |
| + w.pass(t) |
| + w.cloud.err, w.cloud.failIDs = nil, nil |
| + w.cloud.members["i-new"] = member("routers-uk") |
| + w.pass(t) |
| + if len(w.logs) != 2 || !strings.Contains(w.logs[0], head+"asg-req, none, read at ") || !strings.Contains(w.logs[0], "could not be read") || |
| + !strings.Contains(w.logs[1], head+"asg-req, ec2-req, read at ") || !strings.Contains(w.logs[1], "not a current member") { |
| + t.Fatalf("logs %q", w.logs) |
| + } |
| + v := newEnrollWorld(t, "") |
| + v.pass(t) |
| + if got := v.pass(t); got != "invited" { |
| + t.Fatal(got) |
| + } |
| + vhead := "membership read for " + newRef + " (attempt " + v.attempt(t).AttemptID + ", instance i-new, request ids asg-req, ec2-req, read at " |
| + if len(v.logs) != 1 || !strings.Contains(v.logs[0], vhead) || !strings.HasSuffix(v.logs[0], "): invited") { |
| + t.Fatalf("logs %q", v.logs) |
| + } |
| +} |
| + |
| +// conflictStore refuses the next Put of key with the store's own conflict, |
| +// as a conditional write another writer got to first is refused. |
| +type conflictStore struct { |
| + release.Store |
| + key string |
| + armed bool |
| +} |
| + |
| +func (s *conflictStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { |
| + if s.armed && key == s.key { |
| + s.armed = false |
| + return "", release.ErrCASConflict |
| + } |
| + return s.Store.Put(ctx, key, body, ifMatch) |
| +} |
| + |
| +// Only a positive answer counts against an attempt. A store that refuses |
| +// the invitation's or the enrolment's write with its own error (a |
| +// conflict, which a call failing after the deadline also returns) is |
| +// unknown: the attempt stands, its wait does not grow, and the next pass |
| +// goes on from it. |
| +func TestAStoreErrorIsUnknownAndKeepsTheAttempt(t *testing.T) { |
| + for _, key := range []string{"control/host-actor-invitations/" + newRef + ".json", Key} { |
| + t.Run(key, func(t *testing.T) { |
| + w := newEnrollWorld(t, "") |
| + w.pass(t) |
| + if key == Key { |
| + if got := w.pass(t); got != "invited" { |
| + t.Fatal(got) |
| + } |
| + inv, _ := w.invitation(t) |
| + w.guardReady(t, inv.Enrollment, true) |
| + } |
| + first, tries := w.attempt(t).AttemptID, w.e.attempts[newRef].tries |
| + w.e.Directory.Store = &conflictStore{Store: w.e.Directory.Store, key: key, armed: true} |
| + if got := w.pass(t); !strings.Contains(got, release.ErrCASConflict.Error()) { |
| + t.Fatalf("conflict: %q", got) |
| + } |
| + if a := w.attempt(t); a.AttemptID != first || a.Consumed || w.e.attempts[newRef].tries != tries || w.e.attempts[newRef].refused != "" { |
| + t.Fatalf("attempt %+v, seen %+v", a, w.e.attempts[newRef]) |
| + } |
| + want := map[string]string{Key: "enrolled in ie"}[key] |
| + if want == "" { |
| + want = "invited" |
| + } |
| + if got := w.pass(t); !strings.HasPrefix(got, want) || w.attempt(t).AttemptID != first { |
| + t.Fatalf("next pass: %q", got) |
| + } |
| + }) |
| + } |
| +} |
diff --git a/internal/provision/plan14_root_reads_test.go b/internal/provision/plan14_root_reads_test.go| index 2371cba3c..2ee14760a 100644 |
| --- a/internal/provision/plan14_root_reads_test.go |
| +++ b/internal/provision/plan14_root_reads_test.go |
| @@ -816,7 +816,7 @@ func TestPlan14RootReadCensus(t *testing.T) { |
| if err = loadRootReadInventory(filepath.Join(root, "internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv"), &inv, true); err != nil { |
| t.Fatal(err) |
| } |
| - seal = "8198674d8809ef90ecebac55324bfd75e69fb3d73bc27b942f29ca2f43e3fe8d" |
| + seal = "8afff5c9874f6d31079ffb2e722274b0b1f82eef7ae228e3c816245c239a93e1" |
| errs := append(compareRootReads(got, inv), checkRootReadLegacy(inv, seal)...) |
| errs = append(errs, checkRootReadDeclarations(got, inv)...) |
| if len(errs) > 0 { |
diff --git a/internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv b/internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv| index 8a254cd2f..0e496e2a7 100644 |
| --- a/internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv |
| +++ b/internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv |
| @@ -2,6 +2,7 @@ |
| # Task 2: host.env from the deployment's parameters (internal/hostenv: hostenv.go, the layout, assembly and rendering, with an allowlist of names, shared values written create-only and the group-box marker added on write; cli.go, carlos system host-env write and seed; write_linux.go, the write through directories held by descriptor, and write_other.go, its refusal off Linux; awsssm.go, the SSM adapter) and the SSM SDK in go.mod. Three new root-read units. runHostEnv: seed's --from file. Ruled authority A,J4 on conditions (no box write to the parameters; allowlisted names, safe values, atomic write, checked parent; authority-selecting values cross-checked against the roster at the agent's use). writeHostEnv and checkedDir (Linux): write's fixed path, /etc/carlos/host.env, reached only through /etc and /etc/carlos opened O_NOFOLLOW|O_DIRECTORY and checked by fstat (root's, writable by no group or other), the file written 0600 in that directory, synced, renamed and the directory synced; ruled A,J4 under census-ruling-region-ha-2b, as the checked parent runHostEnv's condition asks for. The legacy seal stands. |
| # Task 3: carlos system install-channel (internal/host installchannel.go, installchannel_linux.go): a fresh box installs its signed channel release into both protected slots through verifyPointer, hostfloor.CheckImage and stageCandidate, all unchanged, and on every run hands each directory on the agent's cache path (system, system/router and the version's) to the service user through bootstrap's own descriptor-anchored open, refusing a symlinked parent. One new root-read unit, channelInstall.run, whose one read site opens root's apply lock (systemApplier's lockPath). Ruled authority F,J4. stageCandidate, accepted-legacy, is called and not changed. The legacy seal stands. Fix rounds 1 and 2: more root-read units. slotState (Linux): each protected slot opened with O_NOFOLLOW and hashed from the descriptor against the signed artifact's sha256, to refuse a slot holding a build this command did not install. removeBeneath (Linux): removes the agent's cache entry only by openat2 and unlinkat on descriptors anchored at the router directory, opened with no symlink followed, never by path name, and refuses without openat2. var openat2: the unix.Openat2 seam for the test that it refuses. All ruled F,J4 on the same footing as channelInstall.run. |
| # Task 4: group membership, read by the console (internal/hostactors membershipread.go: MembershipReader, the unexported confirmation and the write bound, the single enforcement point of the 60-second deadline, each store call cut off at 15 seconds; enrollment_confirmation.go: EnrollAutomatic and CheckConfirmed make the read themselves and write on the bound store, and the box-written confirmation object is removed; groupref.go: GroupBoxReference; private.go: a group box's report read under the reference the console derives; awsmembership, the Auto Scaling and EC2 describe adapter) and the Auto Scaling SDK in go.mod. No root-read unit is added or changed: the read is the cloud's, through the reader. Envelope-only, and the seal stands. The record-write census gains deadlineStore.Put (nonrecord, ruled). |
| +# Task 5: the console's automatic enrollment pass (internal/hostactors autoenroll.go: AutoEnroller: an enrolling write the deadline left unknown is settled by the next pass's roster read; internal/console enrollpass.go, started in StartClaimRecoveryPass with region HA on, and awsmembership.go, its membership reader). One new root-read unit, AutoEnroller.inputs, whose two read sites are the signed roster (Directory.Read) and the host floor (hostfloor.Read), in the console. Ruled accepted-legacy (J4 root-input). Invite, BeginEnrollmentAttempt and checkMembership, accepted-legacy, are called and not changed. The seal moves. |
| @coverage . 3678cfe22582d35fe0d8e93d3fed898cdffa6fb2ce2d81e0252c8a8da77993b3 |
| @coverage internal/hostenv 45d9d4b90a5d441020f5ba4573821ca92701246c80b32d6f1823665fb0f65208 |
| internal/hostenv/cli.go runHostEnv authority A,J4 rev3=census-ruling-region-ha-2b;basis=carlos system host-env: write runs as root at a box's start-up and writes /etc/carlos/host.env, 0600, whole (temporary file and rename), from allowlisted parameters with safe values only, refusing a missing required value; its one read site is seed's --from file, an operator's host.env whose values go only to the parameter store; write's path is fixed and reached only through writeHostEnv file:ReadFile@1500 df4470176b229b311927ff0ac4f268d354c3a2357cac4300ac0992d2ebb90899 |
| @@ -12,5 +13,8 @@ internal/host/installchannel.go channelInstall.run authority F,J4 rev3=census-ru |
| internal/host/installchannel_linux.go slotState authority F,J4 rev3=census-ruling-region-ha-2b;basis=carlos system install-channel (root, Linux): opens one protected slot with O_NOFOLLOW (a symlink is never followed), fstats the descriptor and hashes from it, to compare with the sha256 of the signed, verified artifact; a slot holding anything else, a symlink or a non-regular file is refused and left as it is, and nothing in it is executed file:Open@66,file:Stat@388 bc533a29f92ae95d31123b551354ab9f8960576e5ffef1bd6a74a06df0b57a24 |
| internal/host/installchannel_linux.go var openat2 authority F,J4 rev3=census-ruling-region-ha-2b;basis=carlos system install-channel (root, Linux): unix.Openat2 held in a variable so a test can make it fail; removeBeneath is its only user and has no path-based fallback when it fails file:Openat2@14 6081e86f758f4ae3948ec2d4f4c037888e1362f122a410c110e31290ba914816 |
| internal/host/installchannel_linux.go removeBeneath authority F,J4 rev3=census-ruling-region-ha-2b;basis=carlos system install-channel (root, Linux): removes the agent's cache entry for the version, after a run that made it, only through the router directory held by descriptor (opened with no symlink anywhere on its path); its version directory is opened beneath it with RESOLVE_BENEATH|RESOLVE_NO_SYMLINKS and everything under it is unlinked by descriptor; a symlink is unlinked, never entered; when openat2 is unavailable or refuses, the removal refuses and leaves the tree, with no path-based fallback; its one read site lists the held directory's names file:Readdirnames@381 42732bb7b400abdb7f18ddbf5aacdfadc9496d8941e774dc01a471bf15c35f69 |
| -@coverage internal/hostactors be1ff5b27d88362d238ddd45d900a3ea9f6e4cbd3d6b9e5e6047b48a1ebca1c5 |
| +@coverage internal/hostactors cdea6818082bf1e6b9c291b82369cf69b6193aa48b5275a7501c18891b0b8fa9 |
| @coverage internal/hostactors/awsmembership ff967730ddb8f17acb73e3351590ba11695e82341b195539da6fbd856ac39175 |
| +@coverage internal/console 80fa5bba8bd23325ababaedc2b8ce810b25161c93a81c6d97dd7033bcc5df103 |
| +@legacy 2f4ca2a51661d8aae1e0f38a41efdcc9df9f1dd75411ca6742d6ee3ae71f1756 internal/hostactors/autoenroll.go AutoEnroller.inputs legacy accepted-legacy accepted-legacy;rev3=J4;basis=root-input;census-ruling-region-ha-2b file:Read@102,file:Read@305 c0ae921ee9d8df9de403d488d9708c565d659a7bf880199313f50bdc7c8f8127 |
| +internal/hostactors/autoenroll.go AutoEnroller.inputs legacy accepted-legacy accepted-legacy;rev3=J4;basis=root-input;census-ruling-region-ha-2b file:Read@102,file:Read@305 c0ae921ee9d8df9de403d488d9708c565d659a7bf880199313f50bdc7c8f8127 |
diff --git a/region_ha_boundary_test.go b/region_ha_boundary_test.go| index 5452a3357..c321421c0 100644 |
| --- a/region_ha_boundary_test.go |
| +++ b/region_ha_boundary_test.go |
| @@ -37,6 +37,8 @@ var regionHACoreFiles = []string{ |
| "internal/host/installchannel.go", |
| "internal/hostactors/membershipread.go", |
| "internal/hostactors/groupref.go", |
| + "internal/hostactors/autoenroll.go", |
| + "internal/console/enrollpass.go", |
| } |
| // regionHAAdapterFiles are spec 2's AWS adapters. Each is named for AWS so |
| @@ -46,6 +48,7 @@ var regionHAAdapterFiles = []string{ |
| "internal/console/awsregionmetric.go", |
| "internal/hostenv/awsssm.go", |
| "internal/hostactors/awsmembership/awsmembership.go", |
| + "internal/console/awsmembership.go", |
| } |
| // regionHAPackages are the packages spec 2 touches. Every S3 client built |