amadan

carlos / platform Public

termfence: only a terminated state, for the box's own verified instance, is a fence

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude 4de08af3f449691d994335493fc08c50f7299ed3 parent 6f4ac27
2 files changed, +265 −0
  • internal/termfence/fence.go +138 −0
  • internal/termfence/fence_test.go +127 −0
diff --git a/internal/termfence/fence.go b/internal/termfence/fence.go
new file mode 100644
index 000000000..01f7e32fd
--- /dev/null
+++ b/internal/termfence/fence.go
@@ -0,0 +1,138 @@
+// Package termfence decides whether an observation proves that a box's
+// machine has been terminated (region HA spec 1 §5.9, invariant HA7).
+//
+// It is pure: something else calls EC2 or reads the event queue and hands
+// the result here. The rule is narrow on purpose. Only the state
+// "terminated" counts, only for the instance, account and region that the
+// box's AWS-signed record names, and silence is never evidence.
+package termfence
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/service/ec2"
+
+ "github.com/carlosframework/platform/internal/boxid"
+)
+
+const (
+ ViaDescribe = "describe"
+ ViaEvent = "event"
+
+ stateTerminated = "terminated"
+ eventSource = "aws.ec2"
+ eventDetailType = "EC2 Instance State-change Notification"
+)
+
+// ErrNotEvidence wraps every reason an observation does not fence a box.
+var ErrNotEvidence = errors.New("termfence: not evidence of termination")
+
+// Observation is what one source said about one instance. Found is false
+// when the source said nothing about it: an empty reply, another instance.
+type Observation struct {
+ Via string
+ Found bool
+ InstanceID string
+ Account string
+ Region string
+ State string
+ EventID string
+}
+
+// Fence is the record a relocation journals in place of stop evidence.
+type Fence struct {
+ Box string `json:"box"`
+ InstanceID string `json:"instance_id"`
+ Account string `json:"account"`
+ AWSRegion string `json:"aws_region"`
+ State string `json:"state"`
+ ObservedAt string `json:"observed_at"`
+ Via string `json:"via"`
+ EventID string `json:"event_id,omitempty"`
+}
+
+// FromDescribe reads one instance out of a DescribeInstances reply. region
+// is the region the call was made in: the reply does not carry it. A reply
+// that does not mention instanceID yields Found false, which Verify
+// refuses: AWS forgets a terminated instance within about an hour, so
+// absence proves nothing.
+func FromDescribe(out *ec2.DescribeInstancesOutput, instanceID, region string) Observation {
+ o := Observation{Via: ViaDescribe, Region: region}
+ if out == nil || instanceID == "" {
+ return o
+ }
+ for _, r := range out.Reservations {
+ for _, inst := range r.Instances {
+ if aws.ToString(inst.InstanceId) != instanceID {
+ continue
+ }
+ o.Found, o.InstanceID, o.Account = true, instanceID, aws.ToString(r.OwnerId)
+ if inst.State != nil {
+ o.State = string(inst.State.Name)
+ }
+ return o
+ }
+ }
+ return o
+}
+
+// FromEvent reads an EventBridge "EC2 Instance State-change Notification".
+// It refuses anything else: the queue's policy admits only the rule that
+// forwards these, and this is the second check.
+func FromEvent(body []byte) (Observation, error) {
+ var ev struct {
+ ID string `json:"id"`
+ DetailType string `json:"detail-type"`
+ Source string `json:"source"`
+ Account string `json:"account"`
+ Region string `json:"region"`
+ Detail struct {
+ InstanceID string `json:"instance-id"`
+ State string `json:"state"`
+ } `json:"detail"`
+ }
+ if err := json.Unmarshal(body, &ev); err != nil {
+ return Observation{}, fmt.Errorf("termfence: event is not JSON: %w", err)
+ }
+ switch {
+ case ev.Source != eventSource:
+ return Observation{}, fmt.Errorf("termfence: event source %q is not %s", ev.Source, eventSource)
+ case ev.DetailType != eventDetailType:
+ return Observation{}, fmt.Errorf("termfence: event detail-type %q is not a state change", ev.DetailType)
+ case ev.ID == "":
+ return Observation{}, errors.New("termfence: event has no id")
+ }
+ return Observation{Via: ViaEvent, Found: ev.Detail.InstanceID != "", InstanceID: ev.Detail.InstanceID,
+ Account: ev.Account, Region: ev.Region, State: ev.Detail.State, EventID: ev.ID}, nil
+}
+
+func refuse(format string, a ...any) error {
+ return fmt.Errorf("%w: %s", ErrNotEvidence, fmt.Sprintf(format, a...))
+}
+
+// Verify returns a fence for box when o proves that the instance in id,
+// the box's verified identity, is terminated.
+func Verify(box string, id boxid.Identity, o Observation, now time.Time) (Fence, error) {
+ switch {
+ case box == "" || id.InstanceID == "" || id.AccountID == "" || id.Region == "":
+ return Fence{}, refuse("the box has no verified identity")
+ case o.Via != ViaDescribe && o.Via != ViaEvent:
+ return Fence{}, refuse("source %q is not one this binary accepts", o.Via)
+ case !o.Found:
+ return Fence{}, refuse("the source said nothing about %s; absence is not termination", id.InstanceID)
+ case o.InstanceID != id.InstanceID:
+ return Fence{}, refuse("observed %s, the box is %s", o.InstanceID, id.InstanceID)
+ case o.Account != id.AccountID:
+ return Fence{}, refuse("observed in account %q, the box is in %q", o.Account, id.AccountID)
+ case o.Region != id.Region:
+ return Fence{}, refuse("observed in region %q, the box is in %q", o.Region, id.Region)
+ case o.State != stateTerminated:
+ return Fence{}, refuse("state is %q; only %q counts", o.State, stateTerminated)
+ }
+ return Fence{Box: box, InstanceID: o.InstanceID, Account: o.Account, AWSRegion: o.Region, State: o.State,
+ ObservedAt: now.UTC().Format(time.RFC3339), Via: o.Via, EventID: o.EventID}, nil
+}
diff --git a/internal/termfence/fence_test.go b/internal/termfence/fence_test.go
new file mode 100644
index 000000000..57577dd5e
--- /dev/null
+++ b/internal/termfence/fence_test.go
@@ -0,0 +1,127 @@
+package termfence
+
+import (
+ "errors"
+ "testing"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/service/ec2"
+ "github.com/aws/aws-sdk-go-v2/service/ec2/types"
+
+ "github.com/carlosframework/platform/internal/boxid"
+)
+
+var (
+ id = boxid.Identity{AccountID: "111122223333", Region: "eu-west-1", InstanceID: "i-0abc"}
+ now = time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
+)
+
+func describe(owner, instance string, state types.InstanceStateName) *ec2.DescribeInstancesOutput {
+ return &ec2.DescribeInstancesOutput{Reservations: []types.Reservation{{
+ OwnerId: aws.String(owner),
+ Instances: []types.Instance{{InstanceId: aws.String(instance), State: &types.InstanceState{Name: state}}},
+ }}}
+}
+
+func TestATerminatedInstanceOfThisBoxIsAFence(t *testing.T) {
+ o := FromDescribe(describe("111122223333", "i-0abc", types.InstanceStateNameTerminated), "i-0abc", "eu-west-1")
+ f, err := Verify("i-0abc", id, o, now)
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := Fence{Box: "i-0abc", InstanceID: "i-0abc", Account: "111122223333", AWSRegion: "eu-west-1",
+ State: "terminated", ObservedAt: "2026-10-05T12:00:00Z", Via: "describe"}
+ if f != want {
+ t.Fatalf("fence = %+v\nwant %+v", f, want)
+ }
+}
+
+// Review Focus 4: every lookalike is refused.
+func TestLookalikesAreNotEvidence(t *testing.T) {
+ cases := map[string]Observation{
+ "stopped": FromDescribe(describe("111122223333", "i-0abc", types.InstanceStateNameStopped), "i-0abc", "eu-west-1"),
+ "stopping": FromDescribe(describe("111122223333", "i-0abc", types.InstanceStateNameStopping), "i-0abc", "eu-west-1"),
+ "shutting-down": FromDescribe(describe("111122223333", "i-0abc", types.InstanceStateNameShuttingDown), "i-0abc", "eu-west-1"),
+ "running": FromDescribe(describe("111122223333", "i-0abc", types.InstanceStateNameRunning), "i-0abc", "eu-west-1"),
+ "another account": FromDescribe(describe("999999999999", "i-0abc", types.InstanceStateNameTerminated), "i-0abc", "eu-west-1"),
+ "another region": FromDescribe(describe("111122223333", "i-0abc", types.InstanceStateNameTerminated), "i-0abc", "us-east-1"),
+ "another instance": FromDescribe(describe("111122223333", "i-0zzz", types.InstanceStateNameTerminated), "i-0zzz", "eu-west-1"),
+ "not in the reply": FromDescribe(describe("111122223333", "i-0zzz", types.InstanceStateNameTerminated), "i-0abc", "eu-west-1"),
+ "empty reply": FromDescribe(&ec2.DescribeInstancesOutput{}, "i-0abc", "eu-west-1"),
+ "nil reply": FromDescribe(nil, "i-0abc", "eu-west-1"),
+ "no state": FromDescribe(&ec2.DescribeInstancesOutput{Reservations: []types.Reservation{{OwnerId: aws.String("111122223333"), Instances: []types.Instance{{InstanceId: aws.String("i-0abc")}}}}}, "i-0abc", "eu-west-1"),
+ "unknown source": {Via: "cloudtrail", Found: true, InstanceID: "i-0abc", Account: "111122223333", Region: "eu-west-1", State: "terminated"},
+ "zero observation": {},
+ }
+ for name, o := range cases {
+ t.Run(name, func(t *testing.T) {
+ if f, err := Verify("i-0abc", id, o, now); !errors.Is(err, ErrNotEvidence) {
+ t.Fatalf("fence = %+v, err = %v; want ErrNotEvidence", f, err)
+ }
+ })
+ }
+}
+
+func TestAnUnverifiedIdentityIsNeverFenced(t *testing.T) {
+ // A found, terminated observation whose every coordinate is empty, so
+ // it MATCHES an empty identity field for field. Only the guard on the
+ // identity itself can refuse it.
+ o := Observation{Via: ViaDescribe, Found: true, State: "terminated"}
+ if _, err := Verify("i-0abc", boxid.Identity{}, o, now); !errors.Is(err, ErrNotEvidence) {
+ t.Fatalf("err = %v: an empty identity matched an empty observation", err)
+ }
+ if _, err := Verify("", id, Observation{Via: ViaDescribe, Found: true, InstanceID: "i-0abc", Account: "111122223333", Region: "eu-west-1", State: "terminated"}, now); !errors.Is(err, ErrNotEvidence) {
+ t.Fatalf("err = %v: a fence with no box name", err)
+ }
+}
+
+const terminatedEvent = `{"version":"0","id":"ev-123","detail-type":"EC2 Instance State-change Notification","source":"aws.ec2","account":"111122223333","time":"2026-10-05T11:59:00Z","region":"eu-west-1","resources":["arn:aws:ec2:eu-west-1:111122223333:instance/i-0abc"],"detail":{"instance-id":"i-0abc","state":"terminated"}}`
+
+func TestATerminatedEventIsAFence(t *testing.T) {
+ o, err := FromEvent([]byte(terminatedEvent))
+ if err != nil {
+ t.Fatal(err)
+ }
+ f, err := Verify("i-0abc", id, o, now)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if f.Via != "event" || f.EventID != "ev-123" || f.State != "terminated" {
+ t.Fatalf("fence = %+v", f)
+ }
+}
+
+func TestEventsThatAreNotEC2StateChangesAreRefused(t *testing.T) {
+ for name, body := range map[string]string{
+ "not JSON": `terminated`,
+ "another source": `{"id":"e","detail-type":"EC2 Instance State-change Notification","source":"my.app","account":"111122223333","region":"eu-west-1","detail":{"instance-id":"i-0abc","state":"terminated"}}`,
+ "another detail-type": `{"id":"e","detail-type":"EC2 Spot Instance Interruption Warning","source":"aws.ec2","account":"111122223333","region":"eu-west-1","detail":{"instance-id":"i-0abc","state":"terminated"}}`,
+ "no event id": `{"detail-type":"EC2 Instance State-change Notification","source":"aws.ec2","account":"111122223333","region":"eu-west-1","detail":{"instance-id":"i-0abc","state":"terminated"}}`,
+ } {
+ t.Run(name, func(t *testing.T) {
+ if _, err := FromEvent([]byte(body)); err == nil {
+ t.Fatal("accepted")
+ }
+ })
+ }
+}
+
+func TestAShuttingDownEventIsNotEvidence(t *testing.T) {
+ body := `{"id":"e","detail-type":"EC2 Instance State-change Notification","source":"aws.ec2","account":"111122223333","region":"eu-west-1","detail":{"instance-id":"i-0abc","state":"shutting-down"}}`
+ o, err := FromEvent([]byte(body))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := Verify("i-0abc", id, o, now); !errors.Is(err, ErrNotEvidence) {
+ t.Fatalf("err = %v, want ErrNotEvidence", err)
+ }
+}
+
+func TestALegacyLabelFencesByItsBoundInstance(t *testing.T) {
+ o := FromDescribe(describe("111122223333", "i-0abc", types.InstanceStateNameTerminated), "i-0abc", "eu-west-1")
+ f, err := Verify("annie-welding", id, o, now)
+ if err != nil || f.Box != "annie-welding" || f.InstanceID != "i-0abc" {
+ t.Fatalf("fence = %+v, err = %v", f, err)
+ }
+}