amadan
region HA 2b: group membership, read by the console
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
17 files changed,
+1241
−131
go.mod+1 −0go.sum+2 −0internal/hostactors/attempt_regression_test.go+15 −40internal/hostactors/awsmembership/awsmembership.go+89 −0internal/hostactors/awsmembership/awsmembership_test.go+64 −0internal/hostactors/boundsdk_test.go+224 −0internal/hostactors/deadlinestore_test.go+155 −0internal/hostactors/enrollment_confirmation.go+51 −78internal/hostactors/export_test.go+12 −0internal/hostactors/groupref.go+22 −0internal/hostactors/membershipread.go+280 −0internal/hostactors/membershipread_test.go+297 −0internal/hostactors/private.go+10 −1internal/hostactors/roles_test.go+6 −11internal/provision/plan9_b8_census_test.go+5 −0internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv+4 −1region_ha_boundary_test.go+4 −0
diff --git a/go.mod b/go.mod| index 63a0676ea..46f543387 100644 |
| --- a/go.mod |
| +++ b/go.mod |
| @@ -7,6 +7,7 @@ require ( |
| github.com/aws/aws-sdk-go-v2/config v1.32.6 |
| github.com/aws/aws-sdk-go-v2/credentials v1.19.6 |
| github.com/aws/aws-sdk-go-v2/service/acm v1.45.0 |
| + github.com/aws/aws-sdk-go-v2/service/autoscaling v1.74.0 |
| github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.68.0 |
| github.com/aws/aws-sdk-go-v2/service/costexplorer v1.67.5 |
| github.com/aws/aws-sdk-go-v2/service/ec2 v1.324.0 |
diff --git a/go.sum b/go.sum| index 8633e5c02..c2e4267dc 100644 |
| --- a/go.sum |
| +++ b/go.sum |
| @@ -40,6 +40,8 @@ github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.38 h1:A3UAuCmx7LyUcrixBTzKJYYIUZ2 |
| github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.38/go.mod h1:1PDUYG9Z+JrbbsobsAZHjWOm9QBT/djiK3QbykTL5Z4= |
| github.com/aws/aws-sdk-go-v2/service/acm v1.45.0 h1:C/LUyd2F3iFtityAYQbAYb56uP0x32HaHSVXfRSImAM= |
| github.com/aws/aws-sdk-go-v2/service/acm v1.45.0/go.mod h1:8TFQHTFTgR7OoBHlqJr8gsHv+ePC8z4S1lcrZVPgqbM= |
| +github.com/aws/aws-sdk-go-v2/service/autoscaling v1.74.0 h1:hm0br6zbFj1tLd5l8F6owm2/4pFK+z6NA+wIkvdMFxg= |
| +github.com/aws/aws-sdk-go-v2/service/autoscaling v1.74.0/go.mod h1:71sdt+AKd1aCexOigHqdAO5Er2oY2dEhI63220v7A6o= |
| github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.68.0 h1:RdWMeCVaJfu4hCPyRNVQMKMP/Y5J04zB5LF45DmHhRg= |
| github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.68.0/go.mod h1:HfsU8PQ6mHewSJNS77IPAFMZZgFaYSHjAhSOAFXdu9M= |
| github.com/aws/aws-sdk-go-v2/service/costexplorer v1.67.5 h1:hiJJTfio1+i/xnfLwu6RsDjDzkGSiLux54LTzDK4MyA= |
diff --git a/internal/hostactors/attempt_regression_test.go b/internal/hostactors/attempt_regression_test.go| index cd6c8c364..8657e6673 100644 |
| --- a/internal/hostactors/attempt_regression_test.go |
| +++ b/internal/hostactors/attempt_regression_test.go |
| @@ -10,12 +10,11 @@ import ( |
| "encoding/base64" |
| "encoding/json" |
| "github.com/carlosframework/platform/internal/boxid" |
| - "github.com/carlosframework/platform/internal/release" |
| "testing" |
| "time" |
| ) |
| -func TestEnrollmentRefreshRejectsExpiredSupersededAndConsumedEvidence(t *testing.T) { |
| +func TestEnrollmentRejectsSupersededAndConsumedAttempts(t *testing.T) { |
| ctx := context.Background() |
| d, sg, o := fixture(t) |
| key, err := rsa.GenerateKey(rand.Reader, 2048) |
| @@ -28,10 +27,10 @@ func TestEnrollmentRefreshRejectsExpiredSupersededAndConsumedEvidence(t *testing |
| if err != nil { |
| t.Fatal(err) |
| } |
| - rec := boxid.Record{V: 1, Box: "new", IdentityDocument: base64.StdEncoding.EncodeToString(doc), IdentitySignature: base64.StdEncoding.EncodeToString(sig)} |
| + rec := boxid.Record{V: 1, Box: newRef, IdentityDocument: base64.StdEncoding.EncodeToString(doc), IdentitySignature: base64.StdEncoding.EncodeToString(sig)} |
| raw, _ := json.Marshal(rec) |
| v := boxid.Verifier{Account: "123", Certs: map[string]*x509.Certificate{"eu-west-1": {PublicKey: &key.PublicKey, PublicKeyAlgorithm: x509.RSA}}} |
| - in := EnrollmentRequest{Reference: "new", Namespace: "platform", Label: "new", Region: "ie", Pool: "default", AWSRegion: "eu-west-1"} |
| + in := EnrollmentRequest{Reference: newRef, Namespace: "platform", Label: newRef, Region: "ie", Pool: "default", AWSRegion: "eu-west-1"} |
| next := Clone(o.Roster) |
| next.Regions = map[string]Region{"ie": {Account: "123", AWSRegion: "eu-west-1", GroupName: "routers", GroupARN: "group-arn", Namespaces: []string{"platform"}, Pools: []string{"default"}}} |
| o, err = d.Replace(ctx, sg, o, next) |
| @@ -45,27 +44,9 @@ func TestEnrollmentRefreshRejectsExpiredSupersededAndConsumedEvidence(t *testing |
| if err != nil { |
| t.Fatal(err) |
| } |
| - now := time.Now().UTC() |
| - confirmation := func(a EnrollmentAttempt, at time.Time) MembershipConfirmation { |
| - c := MembershipConfirmation{PassID: "pass-1", AttemptID: a.AttemptID, ConfirmationID: a.ConfirmationID, EvidenceSHA256: a.EvidenceSHA256, TargetPlacement: "ie", AWSRegion: "eu-west-1", GroupName: "routers", GroupARN: "group-arn", InstanceID: "i-new", Account: "123", AvailabilityZoneRegion: "eu-west-1", LifecycleState: "InService", InstanceState: "running", ObservedAt: at, RequestIDs: []string{"asg", "ec2"}} |
| - c.Confirmer.Box = "box" |
| - c.Confirmer.Enrollment = o.Roster.Boxes["box"].Enrollment |
| - c.Confirmer.Region = "ie" |
| - return c |
| - } |
| - save := func(a EnrollmentAttempt, c MembershipConfirmation) { |
| - t.Helper() |
| - wire, _ := json.Marshal(c) |
| - if err := release.CreateExact(ctx, d.Store, ConfirmationKey(in.Reference, a.AttemptID, a.ConfirmationID), wire); err != nil { |
| - t.Fatal(err) |
| - } |
| - } |
| - save(first, confirmation(first, now.Add(-2*time.Minute))) |
| - in.AttemptID = first.AttemptID |
| - in.ConfirmationID = first.ConfirmationID |
| - if _, err = d.EnrollAutomatic(ctx, sg, o, v, in, raw, "i-new", now); err == nil { |
| - t.Fatal("expired evidence admitted") |
| - } |
| + // The console's own read, which says the instance is a member. |
| + cloud := &fakeMembership{members: map[string]Membership{"i-new": member("routers")}} |
| + now := func() time.Time { return time.Now().UTC() } |
| staleRoster := o |
| o, second, err := d.BeginEnrollmentAttempt(ctx, sg, o, in, raw, "i-new") |
| if err != nil { |
| @@ -74,30 +55,24 @@ func TestEnrollmentRefreshRejectsExpiredSupersededAndConsumedEvidence(t *testing |
| if first.AttemptID == second.AttemptID || first.ConfirmationID == second.ConfirmationID { |
| t.Fatal("immutable identity reused") |
| } |
| - save(second, confirmation(second, now)) |
| - if _, err = d.EnrollAutomatic(ctx, sg, o, v, in, raw, "i-new", now); err == nil { |
| + in.AttemptID, in.ConfirmationID = first.AttemptID, first.ConfirmationID |
| + if _, _, err = d.EnrollAutomatic(ctx, sg, o, v, in, raw, "i-new", cloud, now); err == nil { |
| t.Fatal("superseded request admitted") |
| } |
| - if _, err = d.EnrollAutomatic(ctx, sg, o, v, in, raw, "i-new", now.Add(-2*time.Minute)); err == nil { |
| - t.Fatal("superseded IDs admitted while old evidence was still fresh") |
| - } |
| - // Even a still-fresh old confirmation cannot win the stale roster CAS. |
| - if _, err = d.EnrollAutomatic(ctx, sg, staleRoster, v, in, raw, "i-new", now.Add(-2*time.Minute)); err == nil { |
| + // The superseded roster snapshot loses its CAS even with its own |
| + // attempt's ids. |
| + if _, _, err = d.EnrollAutomatic(ctx, sg, staleRoster, v, in, raw, "i-new", cloud, now); err == nil { |
| t.Fatal("superseded snapshot admitted") |
| } |
| - in.AttemptID = second.AttemptID |
| - in.ConfirmationID = second.ConfirmationID |
| - admitted, err := d.EnrollAutomatic(ctx, sg, o, v, in, raw, "i-new", now) |
| + in.AttemptID, in.ConfirmationID = second.AttemptID, second.ConfirmationID |
| + admitted, _, err := d.EnrollAutomatic(ctx, sg, o, v, in, raw, "i-new", cloud, now) |
| if err != nil { |
| t.Fatal(err) |
| } |
| if !admitted.Roster.EnrollmentAttempts[in.Reference].Consumed { |
| t.Fatal("attempt not consumed with roster admission") |
| } |
| - if _, err = d.EnrollAutomatic(ctx, sg, admitted, v, in, raw, "i-new", now); err == nil { |
| - t.Fatal("consumed confirmation replayed") |
| - } |
| - if _, _, err = d.Store.Get(ctx, ConfirmationKey(in.Reference, first.AttemptID, first.ConfirmationID)); err != nil { |
| - t.Fatal("old immutable evidence removed", err) |
| + if _, _, err = d.EnrollAutomatic(ctx, sg, admitted, v, in, raw, "i-new", cloud, now); err == nil { |
| + t.Fatal("consumed attempt replayed") |
| } |
| } |
diff --git a/internal/hostactors/awsmembership/awsmembership.go b/internal/hostactors/awsmembership/awsmembership.go| new file mode 100644 |
| index 000000000..d725fe901 |
| --- /dev/null |
| +++ b/internal/hostactors/awsmembership/awsmembership.go |
| @@ -0,0 +1,89 @@ |
| +// Package awsmembership is the AWS adapter of the console's membership |
| +// read (region HA spec 2 §5 step 5, §25; H16, H28): read-only Auto Scaling |
| +// and EC2 describe calls, made in the AWS region the roster names for the |
| +// attempt's placement. The console role needs |
| +// autoscaling:DescribeAutoScalingInstances and ec2:DescribeInstances, |
| +// conditioned on aws:RequestedRegion being one of the deployment's AWS |
| +// regions. No box role has either. |
| +package awsmembership |
| + |
| +import ( |
| + "context" |
| + |
| + "github.com/aws/aws-sdk-go-v2/aws" |
| + awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" |
| + "github.com/aws/aws-sdk-go-v2/config" |
| + "github.com/aws/aws-sdk-go-v2/service/autoscaling" |
| + "github.com/aws/aws-sdk-go-v2/service/ec2" |
| + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" |
| + "github.com/aws/smithy-go/middleware" |
| + |
| + "github.com/carlosframework/platform/internal/hostactors" |
| +) |
| + |
| +// Reader is hostactors.MembershipReader over the console's own credentials. |
| +type Reader struct{ cfg aws.Config } |
| + |
| +// New loads the console's AWS configuration (its instance role). |
| +func New(ctx context.Context) (*Reader, error) { |
| + cfg, err := config.LoadDefaultConfig(ctx) |
| + if err != nil { |
| + return nil, err |
| + } |
| + return &Reader{cfg: cfg}, nil |
| +} |
| + |
| +// Member describes instance's Auto Scaling membership, then its EC2 state |
| +// and owner: two calls, two request ids. |
| +func (r *Reader) Member(ctx context.Context, awsRegion, instance string) (hostactors.Membership, error) { |
| + asg := autoscaling.NewFromConfig(r.cfg, func(o *autoscaling.Options) { o.Region = awsRegion }) |
| + ao, err := asg.DescribeAutoScalingInstances(ctx, &autoscaling.DescribeAutoScalingInstancesInput{InstanceIds: []string{instance}}) |
| + if err != nil { |
| + return hostactors.Membership{}, err |
| + } |
| + ec := ec2.NewFromConfig(r.cfg, func(o *ec2.Options) { o.Region = awsRegion }) |
| + eo, err := ec.DescribeInstances(ctx, &ec2.DescribeInstancesInput{Filters: []ec2types.Filter{{Name: aws.String("instance-id"), Values: []string{instance}}}}) |
| + if err != nil { |
| + return hostactors.Membership{}, err |
| + } |
| + return membershipFrom(instance, ao, eo), nil |
| +} |
| + |
| +// membershipFrom reads the two replies. Anything but exactly the one |
| +// instance asked about, in exactly one group, is no membership. |
| +func membershipFrom(instance string, ao *autoscaling.DescribeAutoScalingInstancesOutput, eo *ec2.DescribeInstancesOutput) hostactors.Membership { |
| + m := hostactors.Membership{RequestIDs: []string{requestID(ao.ResultMetadata), requestID(eo.ResultMetadata)}} |
| + if len(ao.AutoScalingInstances) == 1 && aws.ToString(ao.AutoScalingInstances[0].InstanceId) == instance { |
| + d := ao.AutoScalingInstances[0] |
| + m.GroupName = aws.ToString(d.AutoScalingGroupName) |
| + m.LifecycleState = aws.ToString(d.LifecycleState) |
| + m.AvailabilityZoneRegion = zoneRegion(aws.ToString(d.AvailabilityZone)) |
| + } |
| + if len(eo.Reservations) == 1 && len(eo.Reservations[0].Instances) == 1 && aws.ToString(eo.Reservations[0].Instances[0].InstanceId) == instance { |
| + res := eo.Reservations[0] |
| + m.Owner = aws.ToString(res.OwnerId) |
| + if st := res.Instances[0].State; st != nil { |
| + m.InstanceState = string(st.Name) |
| + } |
| + } |
| + return m |
| +} |
| + |
| +// zoneRegion is the region of an availability zone name ("eu-west-1a" is |
| +// in "eu-west-1"): the name less one trailing letter after a digit. Any |
| +// other shape (a Local Zone, a zone id) answers "", which matches no |
| +// region. |
| +func zoneRegion(az string) string { |
| + n := len(az) |
| + if n < 3 || az[n-1] < 'a' || az[n-1] > 'z' || az[n-2] < '0' || az[n-2] > '9' { |
| + return "" |
| + } |
| + return az[:n-1] |
| +} |
| + |
| +func requestID(md middleware.Metadata) string { |
| + id, _ := awsmiddleware.GetRequestIDMetadata(md) |
| + return id |
| +} |
| + |
| +var _ hostactors.MembershipReader = (*Reader)(nil) |
diff --git a/internal/hostactors/awsmembership/awsmembership_test.go b/internal/hostactors/awsmembership/awsmembership_test.go| new file mode 100644 |
| index 000000000..8c2ab402d |
| --- /dev/null |
| +++ b/internal/hostactors/awsmembership/awsmembership_test.go |
| @@ -0,0 +1,64 @@ |
| +package awsmembership |
| + |
| +import ( |
| + "reflect" |
| + "testing" |
| + |
| + "github.com/aws/aws-sdk-go-v2/aws" |
| + awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" |
| + "github.com/aws/aws-sdk-go-v2/service/autoscaling" |
| + asgtypes "github.com/aws/aws-sdk-go-v2/service/autoscaling/types" |
| + "github.com/aws/aws-sdk-go-v2/service/ec2" |
| + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" |
| + "github.com/aws/smithy-go/middleware" |
| + |
| + "github.com/carlosframework/platform/internal/hostactors" |
| +) |
| + |
| +func meta(id string) middleware.Metadata { |
| + var md middleware.Metadata |
| + awsmiddleware.SetRequestIDMetadata(&md, id) |
| + return md |
| +} |
| + |
| +func asgReply(instance, group, zone, state string) *autoscaling.DescribeAutoScalingInstancesOutput { |
| + return &autoscaling.DescribeAutoScalingInstancesOutput{ResultMetadata: meta("asg-req"), AutoScalingInstances: []asgtypes.AutoScalingInstanceDetails{{ |
| + InstanceId: aws.String(instance), AutoScalingGroupName: aws.String(group), AvailabilityZone: aws.String(zone), LifecycleState: aws.String(state)}}} |
| +} |
| + |
| +func ec2Reply(owner string, instances map[string]ec2types.InstanceStateName) *ec2.DescribeInstancesOutput { |
| + res := ec2types.Reservation{OwnerId: aws.String(owner)} |
| + for id, st := range instances { |
| + res.Instances = append(res.Instances, ec2types.Instance{InstanceId: aws.String(id), State: &ec2types.InstanceState{Name: st}}) |
| + } |
| + return &ec2.DescribeInstancesOutput{ResultMetadata: meta("ec2-req"), Reservations: []ec2types.Reservation{res}} |
| +} |
| + |
| +func TestMembershipFromTheTwoReplies(t *testing.T) { |
| + got := membershipFrom("i-new", asgReply("i-new", "routers-ie", "eu-west-1b", "InService"), ec2Reply("123", map[string]ec2types.InstanceStateName{"i-new": ec2types.InstanceStateNameRunning})) |
| + want := hostactors.Membership{GroupName: "routers-ie", AvailabilityZoneRegion: "eu-west-1", LifecycleState: "InService", InstanceState: "running", Owner: "123", RequestIDs: []string{"asg-req", "ec2-req"}} |
| + if !reflect.DeepEqual(got, want) { |
| + t.Fatalf("got %+v\nwant %+v", got, want) |
| + } |
| + // Not a member: Auto Scaling knows nothing of it, or answers for |
| + // another instance. EC2 knowing it changes nothing. |
| + for _, ao := range []*autoscaling.DescribeAutoScalingInstancesOutput{{ResultMetadata: meta("asg-req")}, asgReply("i-other", "routers-ie", "eu-west-1b", "InService")} { |
| + m := membershipFrom("i-new", ao, ec2Reply("123", map[string]ec2types.InstanceStateName{"i-new": ec2types.InstanceStateNameRunning})) |
| + if m.GroupName != "" || m.LifecycleState != "" { |
| + t.Fatalf("non-member read as %+v", m) |
| + } |
| + } |
| + // EC2 no longer lists it: no state and no owner. |
| + m := membershipFrom("i-new", asgReply("i-new", "routers-ie", "eu-west-1b", "Terminating"), &ec2.DescribeInstancesOutput{ResultMetadata: meta("ec2-req")}) |
| + if m.InstanceState != "" || m.Owner != "" { |
| + t.Fatalf("forgotten instance read as %+v", m) |
| + } |
| +} |
| + |
| +func TestZoneRegion(t *testing.T) { |
| + for az, want := range map[string]string{"eu-west-1a": "eu-west-1", "ap-southeast-2c": "ap-southeast-2", "us-west-2-lax-1a": "us-west-2-lax-1", "euw1-az1": "", "eu-west-1": "", "": ""} { |
| + if got := zoneRegion(az); got != want { |
| + t.Fatalf("zoneRegion(%q) = %q, want %q", az, got, want) |
| + } |
| + } |
| +} |
diff --git a/internal/hostactors/boundsdk_test.go b/internal/hostactors/boundsdk_test.go| new file mode 100644 |
| index 000000000..6b4d6ba24 |
| --- /dev/null |
| +++ b/internal/hostactors/boundsdk_test.go |
| @@ -0,0 +1,224 @@ |
| +package hostactors_test |
| + |
| +import ( |
| + "bytes" |
| + "context" |
| + "crypto/ed25519" |
| + "errors" |
| + "fmt" |
| + "io" |
| + "net/http" |
| + "strings" |
| + "sync" |
| + "testing" |
| + "time" |
| + |
| + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" |
| + |
| + "github.com/carlosframework/platform/internal/hostactors" |
| + "github.com/carlosframework/platform/internal/hostops" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// sdkRequest is one HTTP request the AWS SDK sent: its method, its path, |
| +// its If-Match condition, and how long its context had left when it was |
| +// sent (zero: no deadline). |
| +type sdkRequest struct { |
| + method, path, ifMatch string |
| + left time.Duration |
| +} |
| + |
| +// recordingHTTP stands in for the network under the SDK: a bucket that |
| +// honours S3's conditional headers. It records each request's context |
| +// deadline. A GET answers the object and its ETag, or NoSuchKey; a PUT |
| +// with If-Match succeeds only on the current ETag, one with |
| +// If-None-Match: * only when the key is absent, and gets a new ETag. |
| +type recordingHTTP struct { |
| + mu sync.Mutex |
| + seen []sdkRequest |
| + objects map[string]bucketObject |
| + next int |
| +} |
| + |
| +type bucketObject struct { |
| + body []byte |
| + etag string |
| +} |
| + |
| +func (h *recordingHTTP) Do(r *http.Request) (*http.Response, error) { |
| + var left time.Duration |
| + if dl, ok := r.Context().Deadline(); ok { |
| + left = time.Until(dl) |
| + } |
| + var body []byte |
| + if r.Body != nil { |
| + body, _ = io.ReadAll(r.Body) |
| + } |
| + h.mu.Lock() |
| + defer h.mu.Unlock() |
| + h.seen = append(h.seen, sdkRequest{r.Method, r.URL.Path, r.Header.Get("If-Match"), left}) |
| + resp := &http.Response{Header: http.Header{}, Request: r, Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, Body: io.NopCloser(strings.NewReader(""))} |
| + reply := func(code int, xml string) (*http.Response, error) { |
| + resp.StatusCode = code |
| + resp.Header.Set("Content-Type", "application/xml") |
| + resp.Body = io.NopCloser(strings.NewReader(`<?xml version="1.0" encoding="UTF-8"?><Error><Code>` + xml + `</Code><Message>fake</Message></Error>`)) |
| + return resp, nil |
| + } |
| + key := r.URL.Path |
| + cur, exists := h.objects[key] |
| + switch r.Method { |
| + case http.MethodPut: |
| + if m := r.Header.Get("If-Match"); m != "" && (!exists || strings.Trim(m, `"`) != strings.Trim(cur.etag, `"`)) { |
| + return reply(http.StatusPreconditionFailed, "PreconditionFailed") |
| + } |
| + if r.Header.Get("If-None-Match") == "*" && exists { |
| + return reply(http.StatusPreconditionFailed, "PreconditionFailed") |
| + } |
| + h.next++ |
| + obj := bucketObject{body, fmt.Sprintf(`"e%d"`, h.next)} |
| + h.objects[key] = obj |
| + resp.StatusCode = http.StatusOK |
| + resp.Header.Set("ETag", obj.etag) |
| + return resp, nil |
| + case http.MethodGet: |
| + if !exists { |
| + return reply(http.StatusNotFound, "NoSuchKey") |
| + } |
| + resp.StatusCode = http.StatusOK |
| + resp.Header.Set("ETag", cur.etag) |
| + resp.Header.Set("Content-Length", fmt.Sprint(len(cur.body))) |
| + resp.Body = io.NopCloser(bytes.NewReader(cur.body)) |
| + resp.ContentLength = int64(len(cur.body)) |
| + return resp, nil |
| + } |
| + return reply(http.StatusNotFound, "NoSuchKey") |
| +} |
| + |
| +func (h *recordingHTTP) take() []sdkRequest { |
| + h.mu.Lock() |
| + defer h.mu.Unlock() |
| + out := h.seen |
| + h.seen = nil |
| + return out |
| +} |
| + |
| +// productionStores builds the console's stores as claim_runtime.go does: |
| +// the release S3 store (its own construction, s3compat and all, with only |
| +// the network replaced), the conditional record store over it, and the |
| +// observed mutation boundary with the production mutation check over |
| +// that: the store the enrolment pass's Directory carries. |
| +func productionStores(t *testing.T) (raw *release.S3Store, data release.Store, h *recordingHTTP, root release.Signer) { |
| + t.Helper() |
| + t.Setenv("AWS_ACCESS_KEY_ID", "k") |
| + t.Setenv("AWS_SECRET_ACCESS_KEY", "s") |
| + t.Setenv("AWS_SESSION_TOKEN", "") |
| + t.Setenv("AWS_REGION", "eu-west-1") |
| + t.Setenv("AWS_EC2_METADATA_DISABLED", "true") |
| + t.Setenv("AWS_CONFIG_FILE", "/nonexistent") |
| + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", "/nonexistent") |
| + t.Setenv("CARLOS_S3_ENDPOINT", "http://127.0.0.1:9") |
| + raw, err := release.NewS3Store(context.Background(), "deployment") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + h = &recordingHTTP{objects: map[string]bucketObject{}} |
| + raw.Client = awss3.New(raw.Client.Options(), func(o *awss3.Options) { o.HTTPClient = h; o.RetryMaxAttempts = 1 }) |
| + pub, priv, err := ed25519.GenerateKey(nil) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + control, err := hostops.NewControlWriter(raw, pub, time.Now) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + check, err := hostops.NewProductionMutationCheck(raw, control) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + records, err := hostops.NewConditionalRecordStore(raw, pub) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + data, err = hostops.NewObservedMutationBoundary(true, records, check, nil) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return raw, data, h, release.RootSigner(priv) |
| +} |
| + |
| +// V2 and W1 (platform-1e): the per-call context of a bound write reaches |
| +// the AWS SDK's HTTP request through the console's production stores, so |
| +// the 15-second limit is enforced where the request is sent. The roster |
| +// write is a successful conditional PUT through the whole stack: the |
| +// bound's deadline store, the observed mutation boundary (its pre-read, |
| +// the production mutation check, which admits a root-signed roster, and |
| +// its forwarding), the conditional record store's forwarding and the S3 |
| +// store. Every request it sends, the PUT included, carries a deadline of |
| +// 15 seconds from its call, or the write deadline when that comes first. |
| +// A wrapper anywhere on the path that drops the context, or a bound that |
| +// does not give each call its own, fails here. |
| +func TestTheBoundReachesTheSDKRequest(t *testing.T) { |
| + _, data, h, root := productionStores(t) |
| + for name, c := range map[string]struct { |
| + readAge time.Duration |
| + low, high time.Duration |
| + }{ |
| + "the per-call limit": {0, 14 * time.Second, 15 * time.Second}, |
| + "the earlier write deadline": {50 * time.Second, 9 * time.Second, 10 * time.Second}, |
| + } { |
| + t.Run(name, func(t *testing.T) { |
| + check := func(what string, got []sdkRequest, methods ...string) []sdkRequest { |
| + t.Helper() |
| + var sent []string |
| + for _, r := range got { |
| + sent = append(sent, r.method) |
| + if r.left <= c.low || r.left > c.high { |
| + t.Errorf("%s: %s %s had %s left, want (%s, %s]", what, r.method, r.path, r.left, c.low, c.high) |
| + } |
| + } |
| + if strings.Join(sent, " ") != strings.Join(methods, " ") { |
| + t.Fatalf("%s: the SDK sent %v, want %v", what, sent, methods) |
| + } |
| + return got |
| + } |
| + // The bucket holds a roster the pass has read. |
| + h.mu.Lock() |
| + h.objects = map[string]bucketObject{} |
| + h.mu.Unlock() |
| + before, err := release.SignRootEnvelope(root, map[string]int{"revision": 1}) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + d := hostactors.Directory{Store: data, Bucket: "deployment"} |
| + etag, err := data.Put(context.Background(), hostactors.Key, before, "") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + h.take() |
| + bd, bctx, cancel := hostactors.BindForTest(context.Background(), d, time.Now().Add(-c.readAge), time.Now) |
| + defer cancel() |
| + if _, got, err := bd.Store.Get(bctx, hostactors.Key); err != nil || got != etag { |
| + t.Fatalf("roster read: %q %v", got, err) |
| + } |
| + check("the roster read", h.take(), http.MethodGet) |
| + // The enrolling write: conditional on that ETag, and admitted. |
| + next, err := release.SignRootEnvelope(root, map[string]int{"revision": 2}) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + after, err := bd.Store.Put(bctx, hostactors.Key, next, etag) |
| + if err != nil || after == etag { |
| + t.Fatalf("the conditional roster write: %q %v", after, err) |
| + } |
| + sent := check("the roster write", h.take(), http.MethodGet, http.MethodPut) |
| + if strings.Trim(sent[1].ifMatch, `"`) != strings.Trim(etag, `"`) { |
| + t.Fatalf("the roster PUT was sent with If-Match %q, want the ETag read, %q", sent[1].ifMatch, etag) |
| + } |
| + // And it stays conditional: the old ETag now loses. |
| + if _, err := bd.Store.Put(bctx, hostactors.Key, next, etag); !errors.Is(err, release.ErrCASConflict) { |
| + t.Fatalf("a stale ETag was not refused: %v", err) |
| + } |
| + h.take() |
| + }) |
| + } |
| +} |
diff --git a/internal/hostactors/deadlinestore_test.go b/internal/hostactors/deadlinestore_test.go| new file mode 100644 |
| index 000000000..c0c061b52 |
| --- /dev/null |
| +++ b/internal/hostactors/deadlinestore_test.go |
| @@ -0,0 +1,155 @@ |
| +package hostactors |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "sync" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// dsClock is a clock the store underneath and the test both move. |
| +type dsClock struct { |
| + mu sync.Mutex |
| + t time.Time |
| +} |
| + |
| +func (c *dsClock) now() time.Time { |
| + c.mu.Lock() |
| + defer c.mu.Unlock() |
| + return c.t |
| +} |
| + |
| +func (c *dsClock) add(d time.Duration) { |
| + c.mu.Lock() |
| + defer c.mu.Unlock() |
| + c.t = c.t.Add(d) |
| +} |
| + |
| +// dsStore is a store underneath the bound: put runs in place of each Put, |
| +// and every call it receives is counted. |
| +type dsStore struct { |
| + release.Store |
| + mu sync.Mutex |
| + calls int |
| + put func(ctx context.Context) (string, error) |
| +} |
| + |
| +func (s *dsStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { |
| + s.mu.Lock() |
| + s.calls++ |
| + s.mu.Unlock() |
| + return s.put(ctx) |
| +} |
| + |
| +func (s *dsStore) count() int { |
| + s.mu.Lock() |
| + defer s.mu.Unlock() |
| + return s.calls |
| +} |
| + |
| +// No call is issued at or after the deadline, by the read's clock, and the |
| +// store underneath never sees it. |
| +func TestDeadlineStoreIssuesNothingAtTheDeadline(t *testing.T) { |
| + clock := &dsClock{t: time.Now()} |
| + inner := &dsStore{put: func(context.Context) (string, error) { return "e1", nil }} |
| + bd, bctx, cancel := writeBound(context.Background(), Directory{Store: inner}, membershipConfirmation{observedAt: clock.now()}, clock.now) |
| + defer cancel() |
| + clock.add(membershipFreshness) |
| + if _, err := bd.Store.Put(bctx, Key, []byte("x"), "e0"); !errors.Is(err, ErrWriteDeadline) || inner.count() != 0 { |
| + t.Fatalf("a Put at the deadline: %v, %d calls reached the store", err, inner.count()) |
| + } |
| +} |
| + |
| +// A store that ignores its context is cut off by the wrapper at |
| +// storeCallTimeout, and the context it was given is cancelled: the outcome |
| +// is unknown. |
| +func TestDeadlineStoreCutsOffAStoreThatNeverAnswers(t *testing.T) { |
| + old := storeCallTimeout |
| + storeCallTimeout = 50 * time.Millisecond |
| + defer func() { storeCallTimeout = old }() |
| + unblock := make(chan struct{}) |
| + defer close(unblock) |
| + given := make(chan context.Context, 1) |
| + inner := &dsStore{put: func(ctx context.Context) (string, error) { |
| + given <- ctx |
| + <-unblock |
| + return "e1", nil |
| + }} |
| + now := time.Now |
| + bd, bctx, cancel := writeBound(context.Background(), Directory{Store: inner}, membershipConfirmation{observedAt: now()}, now) |
| + defer cancel() |
| + start := time.Now() |
| + _, err := bd.Store.Put(bctx, Key, []byte("x"), "e0") |
| + took := time.Since(start) |
| + if !errors.Is(err, ErrWriteDeadline) || took > 5*time.Second { |
| + t.Fatalf("a store that never answers: %v after %s", err, took) |
| + } |
| + ctx := <-given |
| + if dl, ok := ctx.Deadline(); !ok || dl.Sub(start) > storeCallTimeout+time.Second { |
| + t.Fatalf("the call's context had deadline %v (%v), want within %s of the call", dl, ok, storeCallTimeout) |
| + } |
| + if ctx.Err() == nil { |
| + t.Fatal("the call's context was not cancelled when the wrapper gave up") |
| + } |
| +} |
| + |
| +// A success that answers only after the deadline is unknown, never a |
| +// success. |
| +func TestDeadlineStoreCountsALateSuccessAsUnknown(t *testing.T) { |
| + clock := &dsClock{t: time.Now()} |
| + inner := &dsStore{} |
| + inner.put = func(context.Context) (string, error) { |
| + clock.add(membershipFreshness + time.Second) |
| + return "e1", nil |
| + } |
| + bd, bctx, cancel := writeBound(context.Background(), Directory{Store: inner}, membershipConfirmation{observedAt: clock.now()}, clock.now) |
| + defer cancel() |
| + if etag, err := bd.Store.Put(bctx, Key, []byte("x"), "e0"); !errors.Is(err, ErrWriteDeadline) || etag != "" || inner.count() != 1 { |
| + t.Fatalf("a late success: %q %v, %d calls", etag, err, inner.count()) |
| + } |
| +} |
| + |
| +// An unknown read, or a write the bound could not confirm, is never one of |
| +// the answers that count against the attempt, and leaves the attempt as it |
| +// was: same ids, not consumed, the roster unwritten. |
| +func TestAnUnknownOutcomeKeepsTheAttempt(t *testing.T) { |
| + check := func(t *testing.T, w *memberWorld, err error, want error) { |
| + t.Helper() |
| + if !errors.Is(err, want) || errors.Is(err, ErrNotMember) || errors.Is(err, ErrMembershipStale) { |
| + t.Fatalf("got %v, want only %v", err, want) |
| + } |
| + o, rerr := w.d.Read(context.Background()) |
| + if rerr != nil { |
| + t.Fatal(rerr) |
| + } |
| + a := o.Roster.EnrollmentAttempts[newRef] |
| + if o.Roster.Revision != w.o.Roster.Revision || a.Consumed || a.AttemptID != w.in.AttemptID || a.ConfirmationID != w.in.ConfirmationID { |
| + t.Fatalf("the attempt moved: revision %d (was %d), %+v", o.Roster.Revision, w.o.Roster.Revision, a) |
| + } |
| + } |
| + t.Run("the read fails", func(t *testing.T) { |
| + w := newMemberWorld(t, member("routers-ie")) |
| + w.cloud.err = errors.New("RequestTimeout") |
| + _, ev, err := w.enroll() |
| + check(t, w, err, ErrMembershipUnknown) |
| + if len(ev.RequestIDs) != 0 { |
| + t.Fatalf("a failed read carried request ids %v", ev.RequestIDs) |
| + } |
| + }) |
| + t.Run("the write is cut off", func(t *testing.T) { |
| + w := newMemberWorld(t, member("routers-ie")) |
| + clock := &dsClock{t: w.clock} |
| + inner := w.d.Store |
| + w.d.Store = &dsStore{Store: inner, put: func(context.Context) (string, error) { |
| + // The write answers success, but only at the deadline. |
| + clock.add(membershipFreshness) |
| + return "e-late", nil |
| + }} |
| + _, _, err := w.d.EnrollAutomatic(context.Background(), w.sg, w.o, w.v, w.in, w.raw, "i-new", w.cloud, clock.now) |
| + w.d.Store = inner |
| + check(t, w, err, ErrWriteDeadline) |
| + }) |
| +} |
diff --git a/internal/hostactors/enrollment_confirmation.go b/internal/hostactors/enrollment_confirmation.go| index 52a5d57c3..893b71ba5 100644 |
| --- a/internal/hostactors/enrollment_confirmation.go |
| +++ b/internal/hostactors/enrollment_confirmation.go |
| @@ -4,7 +4,6 @@ import ( |
| "context" |
| "crypto/rand" |
| "encoding/hex" |
| - "encoding/json" |
| "github.com/carlosframework/platform/internal/boxid" |
| "github.com/carlosframework/platform/internal/hostfloor" |
| "github.com/carlosframework/platform/internal/release" |
| @@ -22,97 +21,75 @@ type Region struct { |
| // group takes (roles.go): "" (default) or legacy. |
| Role string `json:"role,omitempty"` |
| } |
| -type TerminatedEnrollment struct { |
| - PassID string `json:"pass_id"` |
| - Box string `json:"box"` |
| - Enrollment string `json:"enrollment"` |
| - InstanceID string `json:"instance_id"` |
| - State string `json:"state"` |
| - ObservedAt time.Time `json:"observed_at"` |
| - RequestID string `json:"request_id"` |
| + |
| +// EnrollAutomatic admits a box its region's group launched (H16, H28). The |
| +// console confirms the box's group membership itself, here: Member is |
| +// asked, through r, and every store call of the enrolling write, |
| +// checkMembership's reads and Replace's Put included, runs on the store |
| +// writeBound binds to that read's deadline, the single enforcement point: |
| +// none is issued at or after it. No box writes or contributes membership |
| +// evidence, and no caller can hand a confirmation in: membershipread.go |
| +// builds it from r's answer alone. The evidence comes back whenever the |
| +// cloud answered, so every read can be logged. |
| +func (d Directory) EnrollAutomatic(ctx context.Context, sg release.Signer, o Observed, v boxid.Verifier, in EnrollmentRequest, raw []byte, instance string, r MembershipReader, now func() time.Time) (Observed, MembershipEvidence, error) { |
| + pinned, c, err := d.checkConfirmed(ctx, o, v, in, raw, instance, r, now) |
| + if err != nil { |
| + return Observed{}, c.evidence(), err |
| + } |
| + bd, bctx, cancel := writeBound(ctx, d, c, now) |
| + defer cancel() |
| + next, err := bd.enroll(bctx, sg, o, v, pinned, raw, true) |
| + if err != nil { |
| + return Observed{}, c.evidence(), err |
| + } |
| + // Callers go on with the store itself, not the bound one. |
| + return next, c.evidence(), nil |
| } |
| -type MembershipConfirmation struct { |
| - PassID string `json:"pass_id"` |
| - AttemptID string `json:"attempt_id"` |
| - ConfirmationID string `json:"confirmation_id"` |
| - EvidenceSHA256 string `json:"evidence_sha256"` |
| - TargetPlacement string `json:"target_placement"` |
| - AWSRegion string `json:"aws_region"` |
| - GroupName string `json:"group_name"` |
| - GroupARN string `json:"group_arn"` |
| - InstanceID string `json:"instance_id"` |
| - Account string `json:"account"` |
| - AvailabilityZoneRegion string `json:"availability_zone_region"` |
| - LifecycleState string `json:"lifecycle_state"` |
| - InstanceState string `json:"instance_state"` |
| - ObservedAt time.Time `json:"observed_at"` |
| - RequestIDs []string `json:"request_ids"` |
| - Confirmer struct { |
| - Box string `json:"box"` |
| - Enrollment string `json:"enrollment"` |
| - Region string `json:"region"` |
| - } `json:"confirmer"` |
| - TerminatedEnrollments []TerminatedEnrollment `json:"terminated_enrollments"` |
| +// CheckConfirmed is EnrollAutomatic's admission check, made without |
| +// writing: the attempt in the roster matches the request and the record, |
| +// the request's reference is the instance's group reference |
| +// (GroupBoxReference), and a describe made now confirms the attempt's |
| +// instance is a current member of the region's group. It returns in |
| +// pinned to that instance and what the read saw. The console's enrollment |
| +// pass asks it immediately before it invites a box, and makes the |
| +// invitation under the read's deadline (autoenroll.go). |
| +func (d Directory) CheckConfirmed(ctx context.Context, o Observed, v boxid.Verifier, in EnrollmentRequest, raw []byte, instance string, r MembershipReader, now func() time.Time) (EnrollmentRequest, MembershipEvidence, error) { |
| + pinned, c, err := d.checkConfirmed(ctx, o, v, in, raw, instance, r, now) |
| + if err != nil { |
| + return EnrollmentRequest{}, c.evidence(), err |
| + } |
| + return pinned, c.evidence(), nil |
| } |
| -// Confirmers write this create-only evidence with read-only ASG/EC2 results. |
| -// This consumer gives H16/H17 their admission boundary; spec 4 owns producing it. |
| -func (d Directory) EnrollAutomatic(ctx context.Context, sg release.Signer, o Observed, v boxid.Verifier, in EnrollmentRequest, raw []byte, instance string, now time.Time) (Observed, error) { |
| +func (d Directory) checkConfirmed(ctx context.Context, o Observed, v boxid.Verifier, in EnrollmentRequest, raw []byte, instance string, r MembershipReader, now func() time.Time) (EnrollmentRequest, membershipConfirmation, error) { |
| if err := validateScope(o.Roster, Scope{Namespace: in.Namespace, Regions: []string{in.Region}, Pools: []string{in.Pool}}); err != nil { |
| - return Observed{}, err |
| + return EnrollmentRequest{}, membershipConfirmation{}, err |
| } |
| region, ok := o.Roster.Regions[in.Region] |
| if !ok || region.Account != v.Account || region.AWSRegion != in.AWSRegion || region.GroupARN == "" || region.GroupName == "" || !segment(instance) || !segment(in.Reference) || in.ExistingReference { |
| - return Observed{}, ErrRoster |
| + return EnrollmentRequest{}, membershipConfirmation{}, ErrRoster |
| + } |
| + // A group box is enrolled only under the reference derived from its |
| + // instance, never one it chose (census ruling region-ha-2b). |
| + if in.Reference != GroupBoxReference(region.Account, instance) { |
| + return EnrollmentRequest{}, membershipConfirmation{}, ErrRoster |
| } |
| attempt, ok := o.Roster.EnrollmentAttempts[in.Reference] |
| if !ok || attempt.Consumed || !token.MatchString(in.AttemptID) || !token.MatchString(in.ConfirmationID) || in.AttemptID == in.ConfirmationID || attempt.AttemptID != in.AttemptID || attempt.ConfirmationID != in.ConfirmationID || attempt.InstanceID != instance || attempt.EvidenceSHA256 != hostfloor.Digest(raw) || attempt.Region != in.Region || attempt.Account != region.Account || attempt.AWSRegion != region.AWSRegion || attempt.GroupARN != region.GroupARN || attempt.GroupName != region.GroupName { |
| - return Observed{}, ErrRoster |
| + return EnrollmentRequest{}, membershipConfirmation{}, ErrRoster |
| } |
| - body, _, err := d.Store.Get(ctx, ConfirmationKey(in.Reference, in.AttemptID, in.ConfirmationID)) |
| + c, err := confirmMembership(ctx, r, region, instance, now) |
| if err != nil { |
| - return Observed{}, err |
| - } |
| - var c MembershipConfirmation |
| - if json.Unmarshal(body, &c) != nil || !segment(c.PassID) || c.AttemptID != attempt.AttemptID || c.ConfirmationID != attempt.ConfirmationID || c.EvidenceSHA256 != attempt.EvidenceSHA256 || c.TargetPlacement != in.Region || c.AWSRegion != region.AWSRegion || c.Account != region.Account || c.GroupARN != region.GroupARN || c.GroupName != region.GroupName || c.InstanceID != instance || c.AvailabilityZoneRegion != region.AWSRegion || (c.LifecycleState != "Pending" && c.LifecycleState != "InService") || (c.InstanceState != "pending" && c.InstanceState != "running") || !freshConfirmation(c.ObservedAt, now) || len(c.RequestIDs) < 2 { |
| - return Observed{}, ErrRoster |
| - } |
| - for _, id := range c.RequestIDs { |
| - if id == "" { |
| - return Observed{}, ErrRoster |
| - } |
| - } |
| - confirmer, ok := o.Roster.Boxes[c.Confirmer.Box] |
| - if !ok || confirmer.State != "active" || confirmer.Enrollment != c.Confirmer.Enrollment || confirmer.Region != c.Confirmer.Region { |
| - return Observed{}, ErrRoster |
| + return EnrollmentRequest{}, c, err |
| } |
| - fenced, err := d.TerminationFence(ctx, o, c.Confirmer.Box, now) |
| - if err != nil || fenced { |
| - return Observed{}, ErrRoster |
| - } |
| - if confirmer.Region != in.Region { |
| - seen := map[string]bool{} |
| - for _, read := range c.TerminatedEnrollments { |
| - b, ok := o.Roster.Boxes[read.Box] |
| - if !ok || read.PassID != c.PassID || seen[read.Box] || b.Region != in.Region || b.Enrollment != read.Enrollment || b.Cloud.InstanceID != read.InstanceID || read.State != "terminated" || read.RequestID == "" || !freshConfirmation(read.ObservedAt, now) || read.ObservedAt.Before(c.ObservedAt.Add(-time.Minute)) || read.ObservedAt.After(c.ObservedAt) { |
| - return Observed{}, ErrRoster |
| - } |
| - seen[read.Box] = true |
| - } |
| - for ref, b := range o.Roster.Boxes { |
| - if b.Region == in.Region && (b.State == "active" || b.State == "quarantined") && !seen[ref] { |
| - return Observed{}, ErrRoster |
| - } |
| - } |
| + if err := c.fresh(now()); err != nil { |
| + return EnrollmentRequest{}, c, err |
| } |
| // Pin the verified IMDS identity to the instance that membership proved. |
| in.TofuInstanceID = instance |
| in.ExistingReference = true |
| - return d.enroll(ctx, sg, o, v, in, raw, true) |
| -} |
| -func freshConfirmation(at, now time.Time) bool { |
| - return !at.IsZero() && now.Sub(at) <= time.Minute && !at.After(now.Add(5*time.Second)) |
| + return in, c, nil |
| } |
| type EnrollmentAttempt struct { |
| @@ -128,10 +105,6 @@ type EnrollmentAttempt struct { |
| Consumed bool `json:"consumed"` |
| } |
| -func ConfirmationKey(ref, attempt, confirmation string) string { |
| - return "control/box-enrollment/" + ref + "/" + attempt + "/" + confirmation + ".json" |
| -} |
| - |
| // BeginEnrollmentAttempt supersedes evidence by CAS in the signed authority. |
| // A retry after expiry obtains new IDs; old immutable confirmations remain. |
| func (d Directory) BeginEnrollmentAttempt(ctx context.Context, sg release.Signer, o Observed, in EnrollmentRequest, evidence []byte, instance string) (Observed, EnrollmentAttempt, error) { |
diff --git a/internal/hostactors/export_test.go b/internal/hostactors/export_test.go| new file mode 100644 |
| index 000000000..b64256281 |
| --- /dev/null |
| +++ b/internal/hostactors/export_test.go |
| @@ -0,0 +1,12 @@ |
| +package hostactors |
| + |
| +import ( |
| + "context" |
| + "time" |
| +) |
| + |
| +// BindForTest is writeBound for a membership read anchored at observedAt, |
| +// for the external test that drives the console's production store. |
| +func BindForTest(ctx context.Context, d Directory, observedAt time.Time, now func() time.Time) (Directory, context.Context, context.CancelFunc) { |
| + return writeBound(ctx, d, membershipConfirmation{observedAt: observedAt}, now) |
| +} |
diff --git a/internal/hostactors/groupref.go b/internal/hostactors/groupref.go| new file mode 100644 |
| index 000000000..7cd3b115a |
| --- /dev/null |
| +++ b/internal/hostactors/groupref.go |
| @@ -0,0 +1,22 @@ |
| +package hostactors |
| + |
| +import ( |
| + "crypto/sha256" |
| + "encoding/hex" |
| +) |
| + |
| +// GroupBoxReference is the reference of a box its region's group launched |
| +// (census ruling region-ha-2b): "g-" and the first 20 hex digits of |
| +// sha256("carlos-box-ref-v1|" + account + "|" + instance). Status labels |
| +// come from 1,600 values and collide as groups replace their boxes; |
| +// instance ids never repeat in an account, and 80 bits of their digest |
| +// make a collision negligible. It is a pseudonym, so the public status |
| +// feed never shows an instance id. The box derives it to publish under it, |
| +// and the console derives it again from the instance it proved (the |
| +// attempt's, then the roster's pinned one) to read and admit it, never |
| +// trusting the key a box chose. Boxes an operator enrolled keep their |
| +// labels. |
| +func GroupBoxReference(account, instance string) string { |
| + sum := sha256.Sum256([]byte("carlos-box-ref-v1|" + account + "|" + instance)) |
| + return "g-" + hex.EncodeToString(sum[:])[:20] |
| +} |
diff --git a/internal/hostactors/membershipread.go b/internal/hostactors/membershipread.go| new file mode 100644 |
| index 000000000..442723acf |
| --- /dev/null |
| +++ b/internal/hostactors/membershipread.go |
| @@ -0,0 +1,280 @@ |
| +package hostactors |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "fmt" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// Group membership, confirmed by the console (region HA spec 2 §5 step 5, |
| +// H16, H28). Before it invites a box its region's group launched, and |
| +// again before it enrolls it, the console asks the cloud, read-only and |
| +// with its own credentials, whether the instance is a current member of |
| +// the region's group. No box writes or contributes membership evidence: |
| +// a confirmation exists only inside this package, built by |
| +// confirmMembership from one MembershipReader answer, and nothing |
| +// exported takes one in. |
| +// |
| +// What the cloud says is the reader's business; the AWS adapter is |
| +// internal/hostactors/awsmembership. This file names no provider (§25). |
| + |
| +// MembershipReader answers, read-only, from the cloud the roster's region |
| +// entries name. It is the console's own client. |
| +type MembershipReader interface { |
| + // Member describes instance in awsRegion: its group, if any, and its |
| + // state. An instance in no group is a Membership with no GroupName. An |
| + // error is a read that failed, never an answer. |
| + Member(ctx context.Context, awsRegion, instance string) (Membership, error) |
| +} |
| + |
| +// Membership is one instance's group membership, as the cloud reported it. |
| +type Membership struct { |
| + GroupName string |
| + AvailabilityZoneRegion string |
| + LifecycleState string |
| + InstanceState string |
| + Owner string |
| + // RequestIDs are the cloud's request ids for the reads, one per call. |
| + RequestIDs []string |
| +} |
| + |
| +// MembershipEvidence is what a confirmation read, for the console's log: |
| +// the cloud's request ids, and when the read began. |
| +type MembershipEvidence struct { |
| + RequestIDs []string |
| + ObservedAt time.Time |
| +} |
| + |
| +// membershipFreshness is the write deadline: a read's write is issued |
| +// less than membershipFreshness after the read's anchor, never at or after |
| +// it. |
| +const membershipFreshness = time.Minute |
| + |
| +var ( |
| + // ErrMembershipUnknown is a membership read that failed. It says |
| + // nothing about the instance: the console tries again with the same |
| + // attempt. |
| + ErrMembershipUnknown = errors.New("hostactors: group membership could not be read") |
| + // ErrNotMember is the cloud's positive answer that the instance is not |
| + // a current, running member of the region's group in the deployment's |
| + // account and AWS region. |
| + ErrNotMember = errors.New("hostactors: the instance is not a current member of the region's group") |
| + // ErrMembershipStale is a read that was already membershipFreshness old |
| + // or more when the describe returned: the cloud was slow, and the |
| + // answer counts against the attempt. |
| + ErrMembershipStale = errors.New("hostactors: the membership read is more than a minute old") |
| + // ErrWriteDeadline is a write the deadline stopped (writeBound): no |
| + // store call was issued at or after it, but one already in flight was |
| + // cancelled, so whether it committed is unknown. The next pass reads |
| + // back and describes again. |
| + ErrWriteDeadline = errors.New("hostactors: the write would be issued more than a minute after its membership read") |
| +) |
| + |
| +// membershipConfirmation is one read of an attempt's instance. It is built |
| +// only by confirmMembership, from the reader's answer, and never leaves |
| +// this package. A read the cloud answered carries its request ids and |
| +// time whether or not it found a member, so every read can be logged. |
| +type membershipConfirmation struct { |
| + requestIDs []string |
| + observedAt time.Time |
| +} |
| + |
| +// confirmMembership asks r about instance in region's AWS region, and |
| +// returns the confirmation only when the answer is a current member of |
| +// region's group: Pending or InService, pending or running, owned by the |
| +// region's account, in its AWS region, with a request id for each call. |
| +// The read's anchor, its time, is taken before the call: never later than |
| +// the describe's completion, so its age is never understated. |
| +func confirmMembership(ctx context.Context, r MembershipReader, region Region, instance string, now func() time.Time) (membershipConfirmation, error) { |
| + if r == nil || now == nil { |
| + return membershipConfirmation{}, fmt.Errorf("%w: no reader", ErrMembershipUnknown) |
| + } |
| + at := now().UTC() |
| + m, err := r.Member(ctx, region.AWSRegion, instance) |
| + if err != nil { |
| + return membershipConfirmation{observedAt: at}, fmt.Errorf("%w: %v", ErrMembershipUnknown, err) |
| + } |
| + c := membershipConfirmation{requestIDs: append([]string(nil), m.RequestIDs...), observedAt: at} |
| + switch { |
| + case m.GroupName != region.GroupName: |
| + return c, fmt.Errorf("%w: it is in group %q", ErrNotMember, m.GroupName) |
| + case m.LifecycleState != "Pending" && m.LifecycleState != "InService": |
| + return c, fmt.Errorf("%w: its lifecycle state is %q", ErrNotMember, m.LifecycleState) |
| + case m.InstanceState == "" || m.Owner == "": |
| + // Auto Scaling knows it but EC2 does not list it (yet, or any |
| + // more): no answer. |
| + return c, fmt.Errorf("%w: EC2 does not list it", ErrMembershipUnknown) |
| + case m.InstanceState != "pending" && m.InstanceState != "running": |
| + return c, fmt.Errorf("%w: it is %q", ErrNotMember, m.InstanceState) |
| + case m.Owner != region.Account || m.AvailabilityZoneRegion != region.AWSRegion: |
| + return c, fmt.Errorf("%w: it is in account %q, region %q", ErrNotMember, m.Owner, m.AvailabilityZoneRegion) |
| + case len(m.RequestIDs) < 2: |
| + return c, fmt.Errorf("%w: the cloud gave no request ids", ErrMembershipUnknown) |
| + } |
| + for _, id := range m.RequestIDs { |
| + if id == "" { |
| + return c, fmt.Errorf("%w: the cloud gave a blank request id", ErrMembershipUnknown) |
| + } |
| + } |
| + return c, nil |
| +} |
| + |
| +// fresh refuses a confirmation membershipFreshness old or more at now, or |
| +// one from the future: the describe itself was too slow. |
| +func (c membershipConfirmation) fresh(now time.Time) error { |
| + if c.observedAt.IsZero() || now.Sub(c.observedAt) >= membershipFreshness || c.observedAt.After(now.Add(5*time.Second)) { |
| + return fmt.Errorf("%w (read at %s, checked at %s)", ErrMembershipStale, c.observedAt.Format(time.RFC3339), now.UTC().Format(time.RFC3339)) |
| + } |
| + return nil |
| +} |
| + |
| +// deadline is the last moment a write this read allows may be issued. |
| +func (c membershipConfirmation) deadline() time.Time { return c.observedAt.Add(membershipFreshness) } |
| + |
| +// atWrite is the bound store's check before each call: at or after the |
| +// deadline it is ErrWriteDeadline, unknown, never a refusal. |
| +func (c membershipConfirmation) atWrite(now time.Time) error { |
| + if c.observedAt.IsZero() || !now.Before(c.deadline()) { |
| + return fmt.Errorf("%w (read at %s, write at %s)", ErrWriteDeadline, c.observedAt.Format(time.RFC3339), now.UTC().Format(time.RFC3339)) |
| + } |
| + return nil |
| +} |
| + |
| +func (c membershipConfirmation) evidence() MembershipEvidence { |
| + return MembershipEvidence{RequestIDs: append([]string(nil), c.requestIDs...), ObservedAt: c.observedAt} |
| +} |
| + |
| +// writeBound is d with every store call bound to c's deadline (C2, S1), |
| +// the single enforcement point of the write's freshness. Each store call |
| +// (a read, the write, the read-back after a lost write) is refused before |
| +// it is issued once the deadline is reached, by either clock, the |
| +// context's or now; the returned context ends at the deadline; and each |
| +// call it does issue gets at most storeCallTimeout (U2). The sealed |
| +// writers (Invite, Replace and the checkMembership it makes) run on it |
| +// unchanged, so no read, signature's Put or read-back of theirs is issued |
| +// 60 seconds or more after the membership read's anchor, and any outcome |
| +// not confirmed in time is ErrWriteDeadline, unknown. |
| +func writeBound(ctx context.Context, d Directory, c membershipConfirmation, now func() time.Time) (Directory, context.Context, context.CancelFunc) { |
| + bctx, cancel := context.WithTimeout(ctx, c.deadline().Sub(now())) |
| + bd := d |
| + bd.Store = deadlineStore{Store: d.Store, check: func(callCtx context.Context) error { |
| + if bctx.Err() != nil || callCtx.Err() != nil { |
| + return fmt.Errorf("%w: the deadline passed", ErrWriteDeadline) |
| + } |
| + return c.atWrite(now()) |
| + }} |
| + return bd, bctx, cancel |
| +} |
| + |
| +// storeCallTimeout is the hard limit on one store call of a bound write |
| +// (U2): a call the store has not answered within it is unknown, and its |
| +// context is cancelled, so an S3 request is abandoned at the HTTP layer. |
| +// It bounds how long the console waits, not when a write commits: a |
| +// call cut off may still commit later, and is settled from the roster on |
| +// a later pass. A var so tests can shorten it. |
| +var storeCallTimeout = 15 * time.Second |
| + |
| +// deadlineStore refuses every call once check says the deadline has been |
| +// reached, before the call reaches the store underneath, and gives each |
| +// call it issues at most storeCallTimeout. The bound cannot rest on the |
| +// store alone: release.DirStore ignores its context, so the wrapper |
| +// returns on its own when the call outlives the timeout or the deadline, |
| +// whatever the store does. Every outcome it cannot confirm in time is |
| +// ErrWriteDeadline, unknown: a call cut off in flight, one that never |
| +// answered, and one that answered, even successfully, only after the |
| +// deadline. |
| +type deadlineStore struct { |
| + release.Store |
| + check func(context.Context) error |
| +} |
| + |
| +// call runs one store call under the bound. |
| +func (s deadlineStore) call(ctx context.Context, f func(context.Context) error) error { |
| + if err := s.check(ctx); err != nil { |
| + return err |
| + } |
| + cctx, cancel := context.WithTimeout(ctx, storeCallTimeout) |
| + defer cancel() |
| + done := make(chan error, 1) |
| + go func() { done <- f(cctx) }() |
| + var err error |
| + select { |
| + case err = <-done: |
| + case <-cctx.Done(): |
| + } |
| + // Cut off, whether the store gave up on the context or not: by the |
| + // deadline, or by the hard timeout. Whatever it answered, if anything, |
| + // is not known to be the outcome. |
| + if cctx.Err() != nil { |
| + if ctx.Err() != nil { |
| + return fmt.Errorf("%w: in flight at the deadline", ErrWriteDeadline) |
| + } |
| + return fmt.Errorf("%w: the store did not answer in time", ErrWriteDeadline) |
| + } |
| + if err == nil { |
| + if late := s.check(ctx); late != nil { |
| + return fmt.Errorf("%w: it answered only after the deadline", ErrWriteDeadline) |
| + } |
| + } |
| + return err |
| +} |
| + |
| +func (s deadlineStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| + var body []byte |
| + var etag string |
| + err := s.call(ctx, func(cctx context.Context) error { |
| + var err error |
| + body, etag, err = s.Store.Get(cctx, key) |
| + return err |
| + }) |
| + if err != nil { |
| + return nil, "", err |
| + } |
| + return body, etag, nil |
| +} |
| + |
| +func (s deadlineStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { |
| + var etag string |
| + err := s.call(ctx, func(cctx context.Context) error { |
| + var err error |
| + etag, err = s.Store.Put(cctx, key, body, ifMatch) |
| + return err |
| + }) |
| + if err != nil { |
| + return "", err |
| + } |
| + return etag, nil |
| +} |
| + |
| +func (s deadlineStore) List(ctx context.Context, prefix string) ([]string, error) { |
| + var keys []string |
| + err := s.call(ctx, func(cctx context.Context) error { |
| + var err error |
| + keys, err = s.Store.List(cctx, prefix) |
| + return err |
| + }) |
| + if err != nil { |
| + return nil, err |
| + } |
| + return keys, nil |
| +} |
| + |
| +func (s deadlineStore) ListDirs(ctx context.Context, prefix string) ([]string, error) { |
| + var dirs []string |
| + err := s.call(ctx, func(cctx context.Context) error { |
| + var err error |
| + dirs, err = s.Store.ListDirs(cctx, prefix) |
| + return err |
| + }) |
| + if err != nil { |
| + return nil, err |
| + } |
| + return dirs, nil |
| +} |
| + |
| +func (s deadlineStore) Delete(ctx context.Context, key string) error { |
| + return s.call(ctx, func(cctx context.Context) error { return s.Store.Delete(cctx, key) }) |
| +} |
diff --git a/internal/hostactors/membershipread_test.go b/internal/hostactors/membershipread_test.go| new file mode 100644 |
| index 000000000..947111dea |
| --- /dev/null |
| +++ b/internal/hostactors/membershipread_test.go |
| @@ -0,0 +1,297 @@ |
| +package hostactors |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "fmt" |
| + "reflect" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/boxid" |
| + "github.com/carlosframework/platform/internal/boxid/boxidtest" |
| + "github.com/carlosframework/platform/internal/release" |
| + "github.com/carlosframework/platform/internal/status" |
| +) |
| + |
| +// fakeMembership is the cloud, as the console's reader sees it: each |
| +// instance's membership. err fails every call; during runs inside each |
| +// call (a slow describe, say). |
| +type fakeMembership struct { |
| + members map[string]Membership |
| + err error |
| + during func() |
| + calls []string |
| +} |
| + |
| +func (f *fakeMembership) Member(_ context.Context, awsRegion, instance string) (Membership, error) { |
| + f.calls = append(f.calls, awsRegion+" "+instance) |
| + if f.during != nil { |
| + f.during() |
| + } |
| + if f.err != nil { |
| + return Membership{}, f.err |
| + } |
| + return f.members[instance], nil |
| +} |
| + |
| +func member(group string) Membership { |
| + return Membership{GroupName: group, AvailabilityZoneRegion: "eu-west-1", LifecycleState: "InService", InstanceState: "running", Owner: "123", RequestIDs: []string{"asg-req", "ec2-req"}} |
| +} |
| + |
| +// memberWorld is fixture()'s roster with ie's group "routers-ie", and a |
| +// box (instance i-new, reference newRef) the group launched, invited, with an open |
| +// attempt. |
| +type memberWorld struct { |
| + d Directory |
| + sg release.Signer |
| + o Observed |
| + v boxid.Verifier |
| + raw []byte |
| + in EnrollmentRequest |
| + cloud *fakeMembership |
| + clock time.Time |
| +} |
| + |
| +func newMemberWorld(t *testing.T, m Membership) *memberWorld { |
| + t.Helper() |
| + ctx := context.Background() |
| + d, sg, o := fixture(t) |
| + aws := boxidtest.NewSigner(t) |
| + _, raw := aws.Record(t, newRef, "123", "eu-west-1", "i-new") |
| + next := Clone(o.Roster) |
| + next.Regions = map[string]Region{"ie": {Account: "123", AWSRegion: "eu-west-1", GroupName: "routers-ie", GroupARN: "arn:aws:autoscaling:eu-west-1:123:autoScalingGroup:x:autoScalingGroupName/routers-ie", Namespaces: []string{"platform"}, Pools: []string{"default"}}} |
| + o, err := d.Replace(ctx, sg, o, next) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + w := &memberWorld{d: d, sg: sg, v: boxid.Verifier{Account: "123", Certs: aws.Certs()}, raw: raw, clock: time.Now().UTC(), |
| + cloud: &fakeMembership{members: map[string]Membership{"i-new": m}}, |
| + in: EnrollmentRequest{Reference: newRef, Namespace: "platform", Label: newRef, Region: "ie", Pool: "default", AWSRegion: "eu-west-1"}} |
| + if _, err := d.Invite(ctx, sg, newRef, raw); err != nil { |
| + t.Fatal(err) |
| + } |
| + o, a, err := d.BeginEnrollmentAttempt(ctx, sg, o, w.in, raw, "i-new") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + w.o, w.in.AttemptID, w.in.ConfirmationID = o, a.AttemptID, a.ConfirmationID |
| + return w |
| +} |
| + |
| +func (w *memberWorld) now() time.Time { return w.clock } |
| + |
| +func (w *memberWorld) enroll() (Observed, MembershipEvidence, error) { |
| + return w.d.EnrollAutomatic(context.Background(), w.sg, w.o, w.v, w.in, w.raw, "i-new", w.cloud, w.now) |
| +} |
| + |
| +// The console reads membership itself, once, in the attempt's AWS region, |
| +// and enrolls on it; the read's request ids come back for its log. |
| +func TestEnrollAutomaticReadsMembershipItself(t *testing.T) { |
| + w := newMemberWorld(t, member("routers-ie")) |
| + next, ev, err := w.enroll() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if b := next.Roster.Boxes[newRef]; b.State != "active" || b.Group != "routers-ie" || b.Cloud.InstanceID != "i-new" { |
| + t.Fatalf("roster entry %+v", b) |
| + } |
| + if !reflect.DeepEqual(w.cloud.calls, []string{"eu-west-1 i-new"}) || !reflect.DeepEqual(ev.RequestIDs, []string{"asg-req", "ec2-req"}) || !ev.ObservedAt.Equal(w.clock) { |
| + t.Fatalf("calls %v, evidence %+v", w.cloud.calls, ev) |
| + } |
| + // A replay of the consumed attempt is refused before the cloud is |
| + // asked again. |
| + w.o = next |
| + if _, _, err := w.enroll(); !errors.Is(err, ErrRoster) || len(w.cloud.calls) != 1 { |
| + t.Fatalf("replay: %v, calls %v", err, w.cloud.calls) |
| + } |
| + if _, _, err := w.d.CheckConfirmed(context.Background(), next, w.v, w.in, w.raw, "i-new", w.cloud, w.now); !errors.Is(err, ErrRoster) { |
| + t.Fatalf("replayed check: %v", err) |
| + } |
| +} |
| + |
| +// Every positive answer that the instance is not a current member of the |
| +// region's group refuses, and writes nothing. |
| +func TestEnrollAutomaticRefusesANonMember(t *testing.T) { |
| + for name, m := range map[string]Membership{ |
| + "another group": member("routers-uk"), |
| + "no group": {Owner: "123", InstanceState: "running", AvailabilityZoneRegion: "eu-west-1", RequestIDs: []string{"a", "b"}}, |
| + "leaving the group": func() Membership { m := member("routers-ie"); m.LifecycleState = "Terminating"; return m }(), |
| + "stopped": func() Membership { m := member("routers-ie"); m.InstanceState = "stopped"; return m }(), |
| + "terminated": func() Membership { m := member("routers-ie"); m.InstanceState = "terminated"; return m }(), |
| + "another account": func() Membership { m := member("routers-ie"); m.Owner = "999"; return m }(), |
| + "another zone": func() Membership { m := member("routers-ie"); m.AvailabilityZoneRegion = "eu-west-2"; return m }(), |
| + } { |
| + t.Run(name, func(t *testing.T) { |
| + w := newMemberWorld(t, m) |
| + if _, _, err := w.d.CheckConfirmed(context.Background(), w.o, w.v, w.in, w.raw, "i-new", w.cloud, w.now); !errors.Is(err, ErrNotMember) { |
| + t.Fatalf("check: %v", err) |
| + } |
| + if _, _, err := w.enroll(); !errors.Is(err, ErrNotMember) { |
| + t.Fatalf("enroll: %v", err) |
| + } |
| + o, err := w.d.Read(context.Background()) |
| + if err != nil || o.Roster.Revision != w.o.Roster.Revision { |
| + t.Fatalf("a refused enrollment wrote the roster: %v", err) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// A read that fails, or is not an answer, is unknown: never a member and |
| +// never a non-member. |
| +func TestAMembershipReadThatIsNotAnAnswerIsUnknown(t *testing.T) { |
| + for name, edit := range map[string]func(*fakeMembership){ |
| + "the call fails": func(f *fakeMembership) { f.err = errors.New("Throttling") }, |
| + "EC2 does not list it": func(f *fakeMembership) { |
| + m := f.members["i-new"] |
| + m.InstanceState, m.Owner = "", "" |
| + f.members["i-new"] = m |
| + }, |
| + "one request id": func(f *fakeMembership) { m := f.members["i-new"]; m.RequestIDs = []string{"a"}; f.members["i-new"] = m }, |
| + "a blank request id": func(f *fakeMembership) { |
| + m := f.members["i-new"] |
| + m.RequestIDs = []string{"a", ""} |
| + f.members["i-new"] = m |
| + }, |
| + } { |
| + t.Run(name, func(t *testing.T) { |
| + w := newMemberWorld(t, member("routers-ie")) |
| + edit(w.cloud) |
| + _, _, err := w.enroll() |
| + if !errors.Is(err, ErrMembershipUnknown) || errors.Is(err, ErrNotMember) { |
| + t.Fatalf("%v", err) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// C2: the write deadline is 60 seconds from the read's anchor, taken |
| +// before the describe and so never later than its completion, and a read |
| +// sixty seconds old or more is refused. A describe that took exactly a |
| +// minute leaves the roster as it was. |
| +func TestEnrollAutomaticRefusesAReadOlderThanAMinuteAtTheWrite(t *testing.T) { |
| + w := newMemberWorld(t, member("routers-ie")) |
| + w.cloud.during = func() { w.clock = w.clock.Add(60 * time.Second) } |
| + if _, _, err := w.enroll(); !errors.Is(err, ErrMembershipStale) { |
| + t.Fatalf("slow read: %v", err) |
| + } |
| + o, err := w.d.Read(context.Background()) |
| + if err != nil || o.Roster.Revision != w.o.Roster.Revision || o.Roster.EnrollmentAttempts[newRef].Consumed { |
| + t.Fatalf("a stale read wrote the roster: %v", err) |
| + } |
| + // A read that was fresh when it was checked but is over a minute old |
| + // by the write: here every look at the clock takes 31 seconds, so the |
| + // read is 31 seconds old when checked and 62 when the roster write is |
| + // about to be made. The write bound refuses it there, as unknown. |
| + u := newMemberWorld(t, member("routers-ie")) |
| + ticking := func() time.Time { u.clock = u.clock.Add(31 * time.Second); return u.clock } |
| + if _, _, err := u.d.EnrollAutomatic(context.Background(), u.sg, u.o, u.v, u.in, u.raw, "i-new", u.cloud, ticking); !errors.Is(err, ErrWriteDeadline) || errors.Is(err, ErrMembershipStale) { |
| + t.Fatalf("a read gone stale before the write: %v", err) |
| + } |
| + if o, err := u.d.Read(context.Background()); err != nil || o.Roster.Revision != u.o.Roster.Revision { |
| + t.Fatalf("a read gone stale before the write wrote the roster: %v", err) |
| + } |
| + // Fifty-nine seconds is inside the minute. |
| + v := newMemberWorld(t, member("routers-ie")) |
| + v.cloud.during = func() { v.clock = v.clock.Add(59 * time.Second) } |
| + if _, _, err := v.enroll(); err != nil { |
| + t.Fatalf("a read inside the minute: %v", err) |
| + } |
| +} |
| + |
| +// No exported identifier of this package carries a membership |
| +// confirmation: the only one is built inside it, from a reader's answer. |
| +func TestNoExportedPathTakesAConfirmation(t *testing.T) { |
| + for _, name := range []string{"EnrollAutomatic", "CheckConfirmed"} { |
| + m, ok := reflect.TypeOf(Directory{}).MethodByName(name) |
| + if !ok { |
| + t.Fatalf("no %s", name) |
| + } |
| + for i := 0; i < m.Type.NumIn(); i++ { |
| + in := m.Type.In(i) |
| + if strings.Contains(strings.ToLower(in.Name()), "confirmation") || in == reflect.TypeOf(MembershipEvidence{}) || in == reflect.TypeOf(Membership{}) { |
| + t.Fatalf("%s takes a %s", name, in) |
| + } |
| + } |
| + } |
| +} |
| + |
| +// newRef is the joining box's reference: the group reference of |
| +// instance i-new in account 123. |
| +var newRef = GroupBoxReference("123", "i-new") |
| + |
| +// The ruling's first test: two group boxes never share a reference. The |
| +// reference is a pseudonym of account and instance, stable, and outside |
| +// the label space. |
| +func TestGroupBoxReferencesDoNotCollide(t *testing.T) { |
| + if GroupBoxReference("123", "i-0a61f580df1acfa65") != GroupBoxReference("123", "i-0a61f580df1acfa65") { |
| + t.Fatal("not stable") |
| + } |
| + if GroupBoxReference("123", "i-a") == GroupBoxReference("124", "i-a") { |
| + t.Fatal("the account is not in the reference") |
| + } |
| + seen := map[string]string{} |
| + for i := 0; i < 200000; i++ { |
| + inst := fmt.Sprintf("i-%017x", i) |
| + ref := GroupBoxReference("123456789012", inst) |
| + if len(ref) != 22 || !strings.HasPrefix(ref, "g-") || !segment(ref) { |
| + t.Fatalf("%s: %q", inst, ref) |
| + } |
| + if other, ok := seen[ref]; ok { |
| + t.Fatalf("%s and %s share %s", inst, other, ref) |
| + } |
| + seen[ref] = inst |
| + } |
| + if strings.Contains(GroupBoxReference("123", "i-0a61f580df1acfa65"), "0a61f580") { |
| + t.Fatal("the reference shows the instance id") |
| + } |
| +} |
| + |
| +// The console reads a group box's report under the reference it derives |
| +// from the roster's pinned instance, never the label in the entry; every |
| +// other box under its reference, exactly as before. |
| +func TestPrivateBoxReportDerivesAGroupBoxsKey(t *testing.T) { |
| + d, _, _ := fixture(t) |
| + ctx := context.Background() |
| + put := func(key, instance string) { |
| + raw, _ := json.Marshal(status.Box{GeneratedAt: time.Now(), Metrics: &status.BoxMetrics{InstanceID: instance}}) |
| + if _, err := d.Store.Put(ctx, key, raw, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + derived := GroupBoxReference("123", "i-g") |
| + put("status/boxes/"+derived+".json", "i-g") |
| + put("status/boxes/chosen.json", "i-chosen") |
| + put("status/boxes/annie-welding.json", "i-old") |
| + r, err := d.PrivateBoxReport(ctx, "chosen", Box{Label: "chosen", Group: "routers-ie", Cloud: Cloud{Account: "123", InstanceID: "i-g"}}) |
| + if err != nil || r.InstanceID != "i-g" { |
| + t.Fatalf("group box: %+v %v", r, err) |
| + } |
| + r, err = d.PrivateBoxReport(ctx, "annie-welding", Box{Label: "annie-welding", Cloud: Cloud{Account: "123", InstanceID: "i-old"}}) |
| + if err != nil || r.InstanceID != "i-old" { |
| + t.Fatalf("a box an operator enrolled: %+v %v", r, err) |
| + } |
| + if _, err := d.PrivateBoxReport(ctx, derived, Box{Group: "routers-ie"}); !errors.Is(err, ErrRoster) { |
| + t.Fatalf("a group box with no pinned instance: %v", err) |
| + } |
| +} |
| + |
| +// A group box is admitted only under the reference its instance derives: |
| +// an attempt under a reference the box chose is refused before the cloud |
| +// is asked, even for a member. |
| +func TestEnrollAutomaticRefusesAReferenceTheBoxChose(t *testing.T) { |
| + w := newMemberWorld(t, member("routers-ie")) |
| + in := w.in |
| + in.Reference, in.Label = "chosen", "chosen" |
| + o, a, err := w.d.BeginEnrollmentAttempt(context.Background(), w.sg, w.o, in, w.raw, "i-new") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + in.AttemptID, in.ConfirmationID = a.AttemptID, a.ConfirmationID |
| + if _, _, err := w.d.EnrollAutomatic(context.Background(), w.sg, o, w.v, in, w.raw, "i-new", w.cloud, w.now); !errors.Is(err, ErrRoster) || len(w.cloud.calls) != 0 { |
| + t.Fatalf("a chosen reference: %v, calls %v", err, w.cloud.calls) |
| + } |
| +} |
diff --git a/internal/hostactors/private.go b/internal/hostactors/private.go| index ed2e11ec5..6976e9edc 100644 |
| --- a/internal/hostactors/private.go |
| +++ b/internal/hostactors/private.go |
| @@ -7,9 +7,18 @@ import ( |
| "strings" |
| ) |
| +// PrivateBoxReport reads the private status report of box ref. A box its |
| +// region's group enrolled (b.Group set) is read under the reference the |
| +// console derives from the instance the roster pins, never under the key |
| +// the box chose (GroupBoxReference); every other box under ref, as before. |
| func (d Directory) PrivateBoxReport(ctx context.Context, ref string, b Box) (Report, error) { |
| key := "status/boxes/" + ref + ".json" |
| - if strings.HasPrefix(b.Namespace, "fleet:") { |
| + if b.Group != "" { |
| + if b.Cloud.Account == "" || !segment(b.Cloud.InstanceID) { |
| + return Report{}, ErrRoster |
| + } |
| + key = "status/boxes/" + GroupBoxReference(b.Cloud.Account, b.Cloud.InstanceID) + ".json" |
| + } else if strings.HasPrefix(b.Namespace, "fleet:") { |
| fleet := strings.TrimPrefix(b.Namespace, "fleet:") |
| if !segment(fleet) || !segment(b.Label) { |
| return Report{}, ErrRoster |
diff --git a/internal/hostactors/roles_test.go b/internal/hostactors/roles_test.go| index 2d0b0c507..a7714f2fd 100644 |
| --- a/internal/hostactors/roles_test.go |
| +++ b/internal/hostactors/roles_test.go |
| @@ -189,9 +189,9 @@ func TestAutomaticEnrollmentInheritsTheGroupRole(t *testing.T) { |
| if err != nil { |
| t.Fatal(err) |
| } |
| - raw, _ := json.Marshal(boxid.Record{V: 1, Box: "new", IdentityDocument: base64.StdEncoding.EncodeToString(doc), IdentitySignature: base64.StdEncoding.EncodeToString(sig)}) |
| + raw, _ := json.Marshal(boxid.Record{V: 1, Box: newRef, IdentityDocument: base64.StdEncoding.EncodeToString(doc), IdentitySignature: base64.StdEncoding.EncodeToString(sig)}) |
| v := boxid.Verifier{Account: "123", Certs: map[string]*x509.Certificate{"eu-west-1": {PublicKey: &key.PublicKey, PublicKeyAlgorithm: x509.RSA}}} |
| - in := EnrollmentRequest{Reference: "new", Namespace: "platform", Label: "new", Region: "ie", Pool: "default", AWSRegion: "eu-west-1"} |
| + in := EnrollmentRequest{Reference: newRef, Namespace: "platform", Label: newRef, Region: "ie", Pool: "default", AWSRegion: "eu-west-1"} |
| next := Clone(o.Roster) |
| next.Regions = map[string]Region{"ie": {Account: "123", AWSRegion: "eu-west-1", GroupName: "routers", GroupARN: "group-arn", Namespaces: []string{"platform"}, Pools: []string{"default"}, Role: role}} |
| if o, err = d.Replace(ctx, sg, o, next); err != nil { |
| @@ -204,19 +204,14 @@ func TestAutomaticEnrollmentInheritsTheGroupRole(t *testing.T) { |
| if err != nil { |
| t.Fatal(err) |
| } |
| - now := time.Now().UTC() |
| - c := MembershipConfirmation{PassID: "pass-1", AttemptID: a.AttemptID, ConfirmationID: a.ConfirmationID, EvidenceSHA256: a.EvidenceSHA256, TargetPlacement: "ie", AWSRegion: "eu-west-1", GroupName: "routers", GroupARN: "group-arn", InstanceID: "i-new", Account: "123", AvailabilityZoneRegion: "eu-west-1", LifecycleState: "InService", InstanceState: "running", ObservedAt: now, RequestIDs: []string{"asg", "ec2"}} |
| - c.Confirmer.Box, c.Confirmer.Enrollment, c.Confirmer.Region = "box", o.Roster.Boxes["box"].Enrollment, "ie" |
| - wire, _ := json.Marshal(c) |
| - if err := release.CreateExact(ctx, d.Store, ConfirmationKey(in.Reference, a.AttemptID, a.ConfirmationID), wire); err != nil { |
| - t.Fatal(err) |
| - } |
| + // The console's own read: a running member of the group. |
| + cloud := &fakeMembership{members: map[string]Membership{"i-new": member("routers")}} |
| in.AttemptID, in.ConfirmationID = a.AttemptID, a.ConfirmationID |
| - admitted, err := d.EnrollAutomatic(ctx, sg, o, v, in, raw, "i-new", now) |
| + admitted, _, err := d.EnrollAutomatic(ctx, sg, o, v, in, raw, "i-new", cloud, time.Now) |
| if err != nil { |
| t.Fatal(err) |
| } |
| - b := admitted.Roster.Boxes["new"] |
| + b := admitted.Roster.Boxes[newRef] |
| if b.Group != "routers" || b.Role != role { |
| t.Fatalf("admitted box group %q role %q, want routers %q", b.Group, b.Role, role) |
| } |
diff --git a/internal/provision/plan9_b8_census_test.go b/internal/provision/plan9_b8_census_test.go| index 207ab2d67..280792462 100644 |
| --- a/internal/provision/plan9_b8_census_test.go |
| +++ b/internal/provision/plan9_b8_census_test.go |
| @@ -814,6 +814,11 @@ var plan9B8CensusInventory = []censusEntry{ |
| {"internal/hostactors/censuskey.go", "Directory.BindCensusKey", "nonrecord", "sel:Put", "7d9d673435e8c8a6aac3105e3a754bde2adcbc5d603945dd599cef703f334a5b"}, |
| {"internal/hostactors/invitation.go", "Directory.Invite", "nonrecord", "sel:Put", "356618a75a23d0157d8e0402e20187a42956f67a4332d698bb22bf05133ac489"}, |
| {"internal/hostactors/roster.go", "Directory.Replace", "nonrecord", "sel:Put", "e6e8ac0070650dfacc4a438b080cb69738530b3d9013de55c1bcf06431b0a847"}, |
| + // Region HA plan 2b (Task 4): the store a membership read binds to its |
| + // deadline; it forwards Put, refuses it once the deadline passes, gives |
| + // it at most storeCallTimeout, and reports one that answers late as |
| + // unknown. |
| + {"internal/hostactors/membershipread.go", "deadlineStore.Put", "nonrecord", "def:Put,sel:Put", "848f2b529d87e34f128dd481f14213e1cfaf8c54555806e6314864ef9a2dbb14"}, |
| {"internal/hostbox/regionalworker.go", "RegionalWorker.Census", "key-site", "lit:instances", "477bcb896b5ae8ec80af592d68eceb7992ad8a23cd6cff6043ad03b3ef2a8085"}, |
| {"internal/hostbox/requeststore.go", "RequestStore.CopyVersion", "versioned-transport", "def:CopyVersion,sel:CopyVersion", "3a50049ce7ca7f2cbc770bef46218944cdc7000205a86c20e4932f0b510da28f"}, |
| {"internal/hostbox/requeststore.go", "RequestStore.PutIfAbsent", "versioned-transport", "def:PutIfAbsent,sel:PutIfAbsent", "3666cbe5d2b91c1c652ffb4459c10a374fbd92e325e4e5c574fa3525d0ce2ffe"}, |
diff --git a/internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv b/internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv| index 0855ccfe7..4747fe0eb 100644 |
| --- a/internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv |
| +++ b/internal/provision/testdata/plan14-root-reads-region-ha-2b.tsv |
| @@ -1,7 +1,8 @@ |
| # Plan 14 census overlay region-ha-2b (region HA spec 2, plan 2b: docs/superpowers/plans/2026-10-17-region-ha-2b-joining.md), loaded last. RULED by the Plan 14 controller 2026-10-16 as census-ruling-region-ha-2b. |
| # Task 2: host.env from the deployment's parameters (internal/hostenv: hostenv.go, the layout, assembly and rendering, with an allowlist of names, shared values written create-only and the group-box marker added on write; cli.go, carlos system host-env write and seed; write_linux.go, the write through directories held by descriptor, and write_other.go, its refusal off Linux; awsssm.go, the SSM adapter) and the SSM SDK in go.mod. Three new root-read units. runHostEnv: seed's --from file. Ruled authority A,J4 on conditions (no box write to the parameters; allowlisted names, safe values, atomic write, checked parent; authority-selecting values cross-checked against the roster at the agent's use). writeHostEnv and checkedDir (Linux): write's fixed path, /etc/carlos/host.env, reached only through /etc and /etc/carlos opened O_NOFOLLOW|O_DIRECTORY and checked by fstat (root's, writable by no group or other), the file written 0600 in that directory, synced, renamed and the directory synced; ruled A,J4 under census-ruling-region-ha-2b, as the checked parent runHostEnv's condition asks for. The legacy seal stands. |
| # Task 3: carlos system install-channel (internal/host installchannel.go, installchannel_linux.go): a fresh box installs its signed channel release into both protected slots through verifyPointer, hostfloor.CheckImage and stageCandidate, all unchanged, and on every run hands each directory on the agent's cache path (system, system/router and the version's) to the service user through bootstrap's own descriptor-anchored open, refusing a symlinked parent. One new root-read unit, channelInstall.run, whose one read site opens root's apply lock (systemApplier's lockPath). Ruled authority F,J4. stageCandidate, accepted-legacy, is called and not changed. The legacy seal stands. Fix rounds 1 and 2: more root-read units. slotState (Linux): each protected slot opened with O_NOFOLLOW and hashed from the descriptor against the signed artifact's sha256, to refuse a slot holding a build this command did not install. removeBeneath (Linux): removes the agent's cache entry only by openat2 and unlinkat on descriptors anchored at the router directory, opened with no symlink followed, never by path name, and refuses without openat2. var openat2: the unix.Openat2 seam for the test that it refuses. All ruled F,J4 on the same footing as channelInstall.run. |
| -@coverage . be7c96bb2ff00b5b018b436f976f0d33bbcc8765c17e357baa6cd0290f59f42d |
| +# Task 4: group membership, read by the console (internal/hostactors membershipread.go: MembershipReader, the unexported confirmation and the write bound, the single enforcement point of the 60-second deadline, each store call cut off at 15 seconds; enrollment_confirmation.go: EnrollAutomatic and CheckConfirmed make the read themselves and write on the bound store, and the box-written confirmation object is removed; groupref.go: GroupBoxReference; private.go: a group box's report read under the reference the console derives; awsmembership, the Auto Scaling and EC2 describe adapter) and the Auto Scaling SDK in go.mod. No root-read unit is added or changed: the read is the cloud's, through the reader. Envelope-only, and the seal stands. The record-write census gains deadlineStore.Put (nonrecord, ruled). |
| +@coverage . 3678cfe22582d35fe0d8e93d3fed898cdffa6fb2ce2d81e0252c8a8da77993b3 |
| @coverage internal/hostenv 45d9d4b90a5d441020f5ba4573821ca92701246c80b32d6f1823665fb0f65208 |
| internal/hostenv/cli.go runHostEnv authority A,J4 rev3=census-ruling-region-ha-2b;basis=carlos system host-env: write runs as root at a box's start-up and writes /etc/carlos/host.env, 0600, whole (temporary file and rename), from allowlisted parameters with safe values only, refusing a missing required value; its one read site is seed's --from file, an operator's host.env whose values go only to the parameter store; write's path is fixed and reached only through writeHostEnv file:ReadFile@1500 df4470176b229b311927ff0ac4f268d354c3a2357cac4300ac0992d2ebb90899 |
| internal/hostenv/write_linux.go checkedDir authority A,J4 rev3=census-ruling-region-ha-2b;basis=carlos system host-env write (root, Linux): opens one directory under one already held, O_NOFOLLOW|O_DIRECTORY, creating it 0755 first only when asked and missing, and refuses it unless fstat shows a directory owned by this process's user (root, as write requires) that no group or other can write file:Fstat@576,file:Openat@168,file:Openat@372 012257ac0866f3385044785d8abae9c31e61134928c394e22a2aeaef39ec9f3c |
| @@ -11,3 +12,5 @@ internal/host/installchannel.go channelInstall.run authority F,J4 rev3=census-ru |
| internal/host/installchannel_linux.go slotState authority F,J4 rev3=census-ruling-region-ha-2b;basis=carlos system install-channel (root, Linux): opens one protected slot with O_NOFOLLOW (a symlink is never followed), fstats the descriptor and hashes from it, to compare with the sha256 of the signed, verified artifact; a slot holding anything else, a symlink or a non-regular file is refused and left as it is, and nothing in it is executed file:Open@66,file:Stat@388 bc533a29f92ae95d31123b551354ab9f8960576e5ffef1bd6a74a06df0b57a24 |
| internal/host/installchannel_linux.go var openat2 authority F,J4 rev3=census-ruling-region-ha-2b;basis=carlos system install-channel (root, Linux): unix.Openat2 held in a variable so a test can make it fail; removeBeneath is its only user and has no path-based fallback when it fails file:Openat2@14 6081e86f758f4ae3948ec2d4f4c037888e1362f122a410c110e31290ba914816 |
| internal/host/installchannel_linux.go removeBeneath authority F,J4 rev3=census-ruling-region-ha-2b;basis=carlos system install-channel (root, Linux): removes the agent's cache entry for the version, after a run that made it, only through the router directory held by descriptor (opened with no symlink anywhere on its path); its version directory is opened beneath it with RESOLVE_BENEATH|RESOLVE_NO_SYMLINKS and everything under it is unlinked by descriptor; a symlink is unlinked, never entered; when openat2 is unavailable or refuses, the removal refuses and leaves the tree, with no path-based fallback; its one read site lists the held directory's names file:Readdirnames@381 42732bb7b400abdb7f18ddbf5aacdfadc9496d8941e774dc01a471bf15c35f69 |
| +@coverage internal/hostactors f6dbcea46bed878257c5b34f8caeef83e71cc995cae7b3a4ae59339ed0740c68 |
| +@coverage internal/hostactors/awsmembership a3defad7179245492a3c1dcfce5e20f0f85db2e240b474c654acf211da16df98 |
diff --git a/region_ha_boundary_test.go b/region_ha_boundary_test.go| index 344d509c4..5452a3357 100644 |
| --- a/region_ha_boundary_test.go |
| +++ b/region_ha_boundary_test.go |
| @@ -35,6 +35,8 @@ var regionHACoreFiles = []string{ |
| "internal/hostenv/write_linux.go", |
| "internal/hostenv/write_other.go", |
| "internal/host/installchannel.go", |
| + "internal/hostactors/membershipread.go", |
| + "internal/hostactors/groupref.go", |
| } |
| // regionHAAdapterFiles are spec 2's AWS adapters. Each is named for AWS so |
| @@ -43,6 +45,7 @@ var regionHAAdapterFiles = []string{ |
| "internal/host/awsaddr.go", |
| "internal/console/awsregionmetric.go", |
| "internal/hostenv/awsssm.go", |
| + "internal/hostactors/awsmembership/awsmembership.go", |
| } |
| // regionHAPackages are the packages spec 2 touches. Every S3 client built |
| @@ -55,6 +58,7 @@ var regionHAPackages = []string{ |
| "internal/edgesfile", |
| "internal/host", |
| "internal/hostactors", |
| + "internal/hostactors/awsmembership", |
| "internal/hostenv", |
| "internal/hostops", |
| "internal/lbhealth", |