amadan

carlos / platform Public

termfence: a fence binds to the roster entry's cloud and is read by a worker in the box's own region

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude c493d10d38348a2867ade21359ca1fa4518af840 parent 11bde18
3 files changed, +204 −62
  • internal/boxid/verify.go +0 −4
  • internal/termfence/fence.go +94 −30
  • internal/termfence/fence_test.go +110 −28
diff --git a/internal/boxid/verify.go b/internal/boxid/verify.go
index 033404a8d..59dcfe3ce 100644
--- a/internal/boxid/verify.go
+++ b/internal/boxid/verify.go
@@ -15,10 +15,6 @@ import (
// ErrUnverified wraps every reason a record is not evidence for enrollment.
var ErrUnverified = errors.New("boxid: record does not verify")
-// Identity is what an identity document proves. termfence binds to it
-// until it moves to Cloud; nothing else uses it.
-type Identity struct{ AccountID, Region, InstanceID string }
-
// Cloud is the roster entry's cloud field (spec §2, §5.1): the instance a
// box reference is enrolled as. Enrollment fills it from a verified record;
// every later check (the termination fence, volume records) compares
diff --git a/internal/termfence/fence.go b/internal/termfence/fence.go
index 01f7e32fd..b085407be 100644
--- a/internal/termfence/fence.go
+++ b/internal/termfence/fence.go
@@ -1,20 +1,28 @@
// Package termfence decides whether an observation proves that a box's
-// machine has been terminated (region HA spec 1 §5.9, invariant HA7).
+// machine has been terminated (region HA spec 1 §5.9, invariants HA7 and
+// H19).
//
-// It is pure: something else calls EC2 or reads the event queue and hands
-// the result here. The rule is narrow on purpose. Only the state
-// "terminated" counts, only for the instance, account and region that the
-// box's AWS-signed record names, and silence is never evidence.
+// Only the state "terminated" counts, only for the instance, account and
+// region the box's roster entry records under cloud, and only when the
+// worker that read it is in that same region. Silence is never evidence,
+// and no disk condition applies: volume records decide only whether the
+// dead box's local chores are discharged (internal/boxvolumes).
+//
+// This package reads and decides. Writing the fence record and citing it
+// in a claim CAS wait for the claim's acceptance of L2 to L5.
package termfence
import (
+ "context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
+ awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
"github.com/aws/aws-sdk-go-v2/service/ec2"
+ "github.com/aws/smithy-go"
"github.com/carlosframework/platform/internal/boxid"
)
@@ -41,18 +49,38 @@ type Observation struct {
Region string
State string
EventID string
+ RequestID string
+}
+
+// Box is a roster entry as the fence needs it: the reference, the
+// enrollment, and the cloud identity enrollment proved (boxid.Enroll).
+type Box struct {
+ Ref string
+ Enrollment string
+ Cloud boxid.Cloud
}
-// Fence is the record a relocation journals in place of stop evidence.
-type Fence struct {
+func (b Box) complete() bool { return b.Ref != "" && b.Enrollment != "" && b.Cloud.Complete() }
+
+// WrittenBy names the regional worker that made the read.
+type WrittenBy struct {
Box string `json:"box"`
- InstanceID string `json:"instance_id"`
- Account string `json:"account"`
- AWSRegion string `json:"aws_region"`
- State string `json:"state"`
- ObservedAt string `json:"observed_at"`
- Via string `json:"via"`
- EventID string `json:"event_id,omitempty"`
+ Enrollment string `json:"enrollment"`
+}
+
+// Record is §5.9's fence record. Its key (control/box-fences/<enrollment>.json)
+// and its writer belong to the claim's L4 and are not built here.
+type Record struct {
+ V int `json:"v"`
+ Box string `json:"box"`
+ Enrollment string `json:"enrollment"`
+ Cloud boxid.Cloud `json:"cloud"`
+ State string `json:"state"`
+ ObservedAt string `json:"observed_at"`
+ Via string `json:"via"`
+ RequestID string `json:"request_id,omitempty"`
+ EventID string `json:"event_id,omitempty"`
+ WrittenBy WrittenBy `json:"written_by"`
}
// FromDescribe reads one instance out of a DescribeInstances reply. region
@@ -65,6 +93,9 @@ func FromDescribe(out *ec2.DescribeInstancesOutput, instanceID, region string) O
if out == nil || instanceID == "" {
return o
}
+ if id, ok := awsmiddleware.GetRequestIDMetadata(out.ResultMetadata); ok {
+ o.RequestID = id
+ }
for _, r := range out.Reservations {
for _, inst := range r.Instances {
if aws.ToString(inst.InstanceId) != instanceID {
@@ -110,29 +141,62 @@ func FromEvent(body []byte) (Observation, error) {
Account: ev.Account, Region: ev.Region, State: ev.Detail.State, EventID: ev.ID}, nil
}
+// EC2 is the one call this package makes. *ec2.Client satisfies it; tests
+// use a fake. It is a read: the box role holds no EC2 write right.
+type EC2 interface {
+ DescribeInstances(ctx context.Context, in *ec2.DescribeInstancesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error)
+}
+
+// DescribeInstance asks EC2 about c.InstanceID alone, in c.Region whatever
+// region the client was built for. InvalidInstanceID.NotFound comes back
+// as an empty reply, which FromDescribe turns into Found false: absence,
+// never termination. Every other error is returned.
+func DescribeInstance(ctx context.Context, api EC2, c boxid.Cloud) (*ec2.DescribeInstancesOutput, error) {
+ if !c.Complete() {
+ return nil, errors.New("termfence: describe of an incomplete cloud identity")
+ }
+ region := c.Region
+ out, err := api.DescribeInstances(ctx, &ec2.DescribeInstancesInput{InstanceIds: []string{c.InstanceID}},
+ func(o *ec2.Options) { o.Region = region })
+ var apiErr smithy.APIError
+ if errors.As(err, &apiErr) && apiErr.ErrorCode() == "InvalidInstanceID.NotFound" {
+ return &ec2.DescribeInstancesOutput{}, nil
+ }
+ return out, err
+}
+
func refuse(format string, a ...any) error {
return fmt.Errorf("%w: %s", ErrNotEvidence, fmt.Sprintf(format, a...))
}
-// Verify returns a fence for box when o proves that the instance in id,
-// the box's verified identity, is terminated.
-func Verify(box string, id boxid.Identity, o Observation, now time.Time) (Fence, error) {
+// Verify returns box's fence record when o proves that the instance its
+// roster entry records is terminated, read by worker, a regional worker of
+// the same AWS region (spec §6: the fence refuses any observation not made
+// in the box's own region). Whether worker is active in the roster is the
+// caller's check, made before it reads.
+func Verify(box, worker Box, o Observation, now time.Time) (Record, error) {
+ c := box.Cloud
switch {
- case box == "" || id.InstanceID == "" || id.AccountID == "" || id.Region == "":
- return Fence{}, refuse("the box has no verified identity")
+ case !box.complete():
+ return Record{}, refuse("the box has no proven cloud identity")
+ case !worker.complete():
+ return Record{}, refuse("the worker has no proven cloud identity")
+ case worker.Cloud.Region != c.Region:
+ return Record{}, refuse("worker %s is in %q, the box in %q", worker.Ref, worker.Cloud.Region, c.Region)
case o.Via != ViaDescribe && o.Via != ViaEvent:
- return Fence{}, refuse("source %q is not one this binary accepts", o.Via)
+ return Record{}, refuse("source %q is not one this binary accepts", o.Via)
case !o.Found:
- return Fence{}, refuse("the source said nothing about %s; absence is not termination", id.InstanceID)
- case o.InstanceID != id.InstanceID:
- return Fence{}, refuse("observed %s, the box is %s", o.InstanceID, id.InstanceID)
- case o.Account != id.AccountID:
- return Fence{}, refuse("observed in account %q, the box is in %q", o.Account, id.AccountID)
- case o.Region != id.Region:
- return Fence{}, refuse("observed in region %q, the box is in %q", o.Region, id.Region)
+ return Record{}, refuse("the source said nothing about %s; absence is not termination", c.InstanceID)
+ case o.InstanceID != c.InstanceID:
+ return Record{}, refuse("observed %s, the box is %s", o.InstanceID, c.InstanceID)
+ case o.Account != c.Account:
+ return Record{}, refuse("observed in account %q, the box is in %q", o.Account, c.Account)
+ case o.Region != c.Region:
+ return Record{}, refuse("observed in region %q, the box is in %q", o.Region, c.Region)
case o.State != stateTerminated:
- return Fence{}, refuse("state is %q; only %q counts", o.State, stateTerminated)
+ return Record{}, refuse("state is %q; only %q counts", o.State, stateTerminated)
}
- return Fence{Box: box, InstanceID: o.InstanceID, Account: o.Account, AWSRegion: o.Region, State: o.State,
- ObservedAt: now.UTC().Format(time.RFC3339), Via: o.Via, EventID: o.EventID}, nil
+ return Record{V: 1, Box: box.Ref, Enrollment: box.Enrollment, Cloud: c, State: o.State,
+ ObservedAt: now.UTC().Format(time.RFC3339), Via: o.Via, RequestID: o.RequestID, EventID: o.EventID,
+ WrittenBy: WrittenBy{Box: worker.Ref, Enrollment: worker.Enrollment}}, nil
}
diff --git a/internal/termfence/fence_test.go b/internal/termfence/fence_test.go
index 57577dd5e..22c2fed0b 100644
--- a/internal/termfence/fence_test.go
+++ b/internal/termfence/fence_test.go
@@ -1,20 +1,27 @@
package termfence
import (
+ "context"
"errors"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
+ awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
"github.com/aws/aws-sdk-go-v2/service/ec2"
"github.com/aws/aws-sdk-go-v2/service/ec2/types"
+ "github.com/aws/smithy-go"
+ smithymiddleware "github.com/aws/smithy-go/middleware"
"github.com/carlosframework/platform/internal/boxid"
)
var (
- id = boxid.Identity{AccountID: "111122223333", Region: "eu-west-1", InstanceID: "i-0abc"}
- now = time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
+ box = Box{Ref: "annie-welding", Enrollment: "enr-annie-1",
+ Cloud: boxid.Cloud{Provider: "aws", Account: "111122223333", Region: "eu-west-1", InstanceID: "i-0abc"}}
+ worker = Box{Ref: "bert-sawing", Enrollment: "enr-bert-1",
+ Cloud: boxid.Cloud{Provider: "aws", Account: "111122223333", Region: "eu-west-1", InstanceID: "i-0bbb"}}
+ now = time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
)
func describe(owner, instance string, state types.InstanceStateName) *ec2.DescribeInstancesOutput {
@@ -24,20 +31,28 @@ func describe(owner, instance string, state types.InstanceStateName) *ec2.Descri
}}}
}
+func withRequestID(out *ec2.DescribeInstancesOutput, id string) *ec2.DescribeInstancesOutput {
+ var md smithymiddleware.Metadata
+ awsmiddleware.SetRequestIDMetadata(&md, id)
+ out.ResultMetadata = md
+ return out
+}
+
func TestATerminatedInstanceOfThisBoxIsAFence(t *testing.T) {
- o := FromDescribe(describe("111122223333", "i-0abc", types.InstanceStateNameTerminated), "i-0abc", "eu-west-1")
- f, err := Verify("i-0abc", id, o, now)
+ out := withRequestID(describe("111122223333", "i-0abc", types.InstanceStateNameTerminated), "req-1")
+ rec, err := Verify(box, worker, FromDescribe(out, "i-0abc", "eu-west-1"), now)
if err != nil {
t.Fatal(err)
}
- want := Fence{Box: "i-0abc", InstanceID: "i-0abc", Account: "111122223333", AWSRegion: "eu-west-1",
- State: "terminated", ObservedAt: "2026-10-05T12:00:00Z", Via: "describe"}
- if f != want {
- t.Fatalf("fence = %+v\nwant %+v", f, want)
+ want := Record{V: 1, Box: "annie-welding", Enrollment: "enr-annie-1", Cloud: box.Cloud, State: "terminated",
+ ObservedAt: "2026-10-06T12:00:00Z", Via: "describe", RequestID: "req-1",
+ WrittenBy: WrittenBy{Box: "bert-sawing", Enrollment: "enr-bert-1"}}
+ if rec != want {
+ t.Fatalf("record = %+v\nwant %+v", rec, want)
}
}
-// Review Focus 4: every lookalike is refused.
+// Every lookalike is refused.
func TestLookalikesAreNotEvidence(t *testing.T) {
cases := map[string]Observation{
"stopped": FromDescribe(describe("111122223333", "i-0abc", types.InstanceStateNameStopped), "i-0abc", "eu-west-1"),
@@ -56,39 +71,63 @@ func TestLookalikesAreNotEvidence(t *testing.T) {
}
for name, o := range cases {
t.Run(name, func(t *testing.T) {
- if f, err := Verify("i-0abc", id, o, now); !errors.Is(err, ErrNotEvidence) {
- t.Fatalf("fence = %+v, err = %v; want ErrNotEvidence", f, err)
+ if rec, err := Verify(box, worker, o, now); !errors.Is(err, ErrNotEvidence) {
+ t.Fatalf("record = %+v, err = %v; want ErrNotEvidence", rec, err)
}
})
}
}
-func TestAnUnverifiedIdentityIsNeverFenced(t *testing.T) {
+// Review Focus 5: the fence is a regional worker's own read in the box's
+// own region. A worker enrolled elsewhere is refused even when what it
+// read is a perfect terminated observation.
+func TestAWorkerInAnotherRegionNeverFences(t *testing.T) {
+ far := worker
+ far.Cloud.Region = "us-east-1"
+ o := FromDescribe(describe("111122223333", "i-0abc", types.InstanceStateNameTerminated), "i-0abc", "eu-west-1")
+ if _, err := Verify(box, far, o, now); !errors.Is(err, ErrNotEvidence) {
+ t.Fatalf("err = %v, want ErrNotEvidence", err)
+ }
+}
+
+func TestAnIncompleteRosterEntryIsNeverFenced(t *testing.T) {
// A found, terminated observation whose every coordinate is empty, so
- // it MATCHES an empty identity field for field. Only the guard on the
- // identity itself can refuse it.
+ // it MATCHES an empty cloud field for field. Only the guard on the
+ // entry itself can refuse it.
o := Observation{Via: ViaDescribe, Found: true, State: "terminated"}
- if _, err := Verify("i-0abc", boxid.Identity{}, o, now); !errors.Is(err, ErrNotEvidence) {
- t.Fatalf("err = %v: an empty identity matched an empty observation", err)
+ for name, b := range map[string]Box{
+ "empty cloud": {Ref: "annie-welding", Enrollment: "enr-annie-1"},
+ "no enrollment": {Ref: "annie-welding", Cloud: box.Cloud},
+ "no reference": {Enrollment: "enr-annie-1", Cloud: box.Cloud},
+ "not aws": {Ref: "annie-welding", Enrollment: "enr-annie-1", Cloud: boxid.Cloud{Provider: "gcp", Account: "111122223333", Region: "eu-west-1", InstanceID: "i-0abc"}},
+ } {
+ t.Run(name, func(t *testing.T) {
+ full := Observation{Via: ViaDescribe, Found: true, InstanceID: b.Cloud.InstanceID, Account: b.Cloud.Account, Region: b.Cloud.Region, State: "terminated"}
+ for _, obs := range []Observation{o, full} {
+ if _, err := Verify(b, worker, obs, now); !errors.Is(err, ErrNotEvidence) {
+ t.Fatalf("err = %v, want ErrNotEvidence", err)
+ }
+ }
+ })
}
- if _, err := Verify("", id, Observation{Via: ViaDescribe, Found: true, InstanceID: "i-0abc", Account: "111122223333", Region: "eu-west-1", State: "terminated"}, now); !errors.Is(err, ErrNotEvidence) {
- t.Fatalf("err = %v: a fence with no box name", err)
+ if _, err := Verify(box, Box{Ref: "bert-sawing"}, FromDescribe(describe("111122223333", "i-0abc", types.InstanceStateNameTerminated), "i-0abc", "eu-west-1"), now); !errors.Is(err, ErrNotEvidence) {
+ t.Fatalf("a worker with no proven identity: err = %v", err)
}
}
-const terminatedEvent = `{"version":"0","id":"ev-123","detail-type":"EC2 Instance State-change Notification","source":"aws.ec2","account":"111122223333","time":"2026-10-05T11:59:00Z","region":"eu-west-1","resources":["arn:aws:ec2:eu-west-1:111122223333:instance/i-0abc"],"detail":{"instance-id":"i-0abc","state":"terminated"}}`
+const terminatedEvent = `{"version":"0","id":"ev-123","detail-type":"EC2 Instance State-change Notification","source":"aws.ec2","account":"111122223333","time":"2026-10-06T11:59:00Z","region":"eu-west-1","resources":["arn:aws:ec2:eu-west-1:111122223333:instance/i-0abc"],"detail":{"instance-id":"i-0abc","state":"terminated"}}`
func TestATerminatedEventIsAFence(t *testing.T) {
o, err := FromEvent([]byte(terminatedEvent))
if err != nil {
t.Fatal(err)
}
- f, err := Verify("i-0abc", id, o, now)
+ rec, err := Verify(box, worker, o, now)
if err != nil {
t.Fatal(err)
}
- if f.Via != "event" || f.EventID != "ev-123" || f.State != "terminated" {
- t.Fatalf("fence = %+v", f)
+ if rec.Via != "event" || rec.EventID != "ev-123" || rec.State != "terminated" || rec.RequestID != "" {
+ t.Fatalf("record = %+v", rec)
}
}
@@ -113,15 +152,58 @@ func TestAShuttingDownEventIsNotEvidence(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if _, err := Verify("i-0abc", id, o, now); !errors.Is(err, ErrNotEvidence) {
+ if _, err := Verify(box, worker, o, now); !errors.Is(err, ErrNotEvidence) {
t.Fatalf("err = %v, want ErrNotEvidence", err)
}
}
-func TestALegacyLabelFencesByItsBoundInstance(t *testing.T) {
- o := FromDescribe(describe("111122223333", "i-0abc", types.InstanceStateNameTerminated), "i-0abc", "eu-west-1")
- f, err := Verify("annie-welding", id, o, now)
- if err != nil || f.Box != "annie-welding" || f.InstanceID != "i-0abc" {
- t.Fatalf("fence = %+v, err = %v", f, err)
+// fakeEC2 answers DescribeInstances and records the region each call was
+// forced into.
+type fakeEC2 struct {
+ out *ec2.DescribeInstancesOutput
+ err error
+ regions []string
+ ids [][]string
+}
+
+func (f *fakeEC2) DescribeInstances(_ context.Context, in *ec2.DescribeInstancesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) {
+ o := ec2.Options{Region: "client-default"}
+ for _, fn := range optFns {
+ fn(&o)
+ }
+ f.regions = append(f.regions, o.Region)
+ f.ids = append(f.ids, in.InstanceIds)
+ return f.out, f.err
+}
+
+func TestDescribeInstanceAsksInTheBoxsOwnRegion(t *testing.T) {
+ f := &fakeEC2{out: describe("111122223333", "i-0abc", types.InstanceStateNameTerminated)}
+ out, err := DescribeInstance(context.Background(), f, box.Cloud)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(f.regions) != 1 || f.regions[0] != "eu-west-1" || len(f.ids[0]) != 1 || f.ids[0][0] != "i-0abc" {
+ t.Fatalf("called in %v for %v", f.regions, f.ids)
+ }
+ if _, err := Verify(box, worker, FromDescribe(out, "i-0abc", "eu-west-1"), now); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestDescribeInstanceNotFoundIsAbsenceAndOtherErrorsAreErrors(t *testing.T) {
+ notFound := &fakeEC2{err: &smithy.GenericAPIError{Code: "InvalidInstanceID.NotFound", Message: "gone"}}
+ out, err := DescribeInstance(context.Background(), notFound, box.Cloud)
+ if err != nil {
+ t.Fatalf("NotFound: err = %v", err)
+ }
+ if _, err := Verify(box, worker, FromDescribe(out, "i-0abc", "eu-west-1"), now); !errors.Is(err, ErrNotEvidence) {
+ t.Fatalf("NotFound fenced the box: err = %v", err)
+ }
+ denied := &fakeEC2{err: &smithy.GenericAPIError{Code: "UnauthorizedOperation", Message: "no"}}
+ if _, err := DescribeInstance(context.Background(), denied, box.Cloud); err == nil {
+ t.Fatal("an access error read as an answer")
+ }
+ if _, err := DescribeInstance(context.Background(), &fakeEC2{}, boxid.Cloud{InstanceID: "i-0abc"}); err == nil {
+ t.Fatal("described an incomplete cloud")
}
}