amadan

carlos / platform Public

boxvolumes: a running box's volumes are recorded create-only, and only an unbroken all-true record discharges its chores

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude f802bb82796a0369162e73c09575589fa6ad8e7f parent c493d10
2 files changed, +640 −0
  • internal/boxvolumes/volumes.go +369 −0
  • internal/boxvolumes/volumes_test.go +271 −0
diff --git a/internal/boxvolumes/volumes.go b/internal/boxvolumes/volumes.go
new file mode 100644
index 000000000..ec05e0d18
--- /dev/null
+++ b/internal/boxvolumes/volumes.go
@@ -0,0 +1,369 @@
+// Package boxvolumes records a platform box's block device mappings while
+// it runs (region HA spec 1 §5.9, ruling H19), and decides from those
+// records whether a terminated box's park-local chores may be discharged
+// without retirement (claim D5's destroyed-local exception, L3).
+//
+// Nothing here is a termination fence, and nothing about execution,
+// admission, relocation or recovery reads it (spec §8, B4-1). A record that
+// is wrong or missing can leave a disk unretired for an operator; it can
+// never admit a writer.
+//
+// This package builds, writes and reads records. The loop that describes
+// every enrolled box in the region needs the roster's enrollments, and the
+// discharge CAS needs L3: both wait for the claim.
+package boxvolumes
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "slices"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
+ "github.com/aws/aws-sdk-go-v2/service/ec2"
+
+ "github.com/carlosframework/platform/internal/boxid"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+const (
+ Prefix = "control/box-volumes/"
+
+ // RefreshEvery is the longest a running box goes without a new record
+ // when nothing changes (spec §5.9: "at least once an hour").
+ RefreshEvery = time.Hour
+ // EnrollmentWindow is how soon after enrollment the first record must
+ // be taken for it to stand for the launch-time mappings.
+ EnrollmentWindow = time.Hour
+ // LastRecordWindow is the most the newest record may predate the
+ // termination observation (controller ruling, 2026-10-06). An older one
+ // cannot speak for a volume attached after it.
+ LastRecordWindow = 2 * time.Hour
+
+ keyTime = "2006-01-02T15-04-05.000000000Z"
+)
+
+// Device is one EBS volume in the instance's block device mappings.
+type Device struct {
+ Device string `json:"device"`
+ VolumeID string `json:"volume_id"`
+ DeleteOnTermination bool `json:"delete_on_termination"`
+ Status string `json:"status"`
+}
+
+// WrittenBy names the regional worker that made the read.
+type WrittenBy struct {
+ Box string `json:"box"`
+ Enrollment string `json:"enrollment"`
+}
+
+// Record is §5.9's volume record,
+// control/box-volumes/<enrollment>/<observed_at>.json.
+type Record struct {
+ V int `json:"v"`
+ Box string `json:"box"`
+ Enrollment string `json:"enrollment"`
+ Cloud boxid.Cloud `json:"cloud"`
+ InstanceState string `json:"instance_state"`
+ BlockDevices []Device `json:"block_devices"`
+ RemovedVolumes []string `json:"removed_volumes"`
+ ObservedAt string `json:"observed_at"`
+ RequestID string `json:"request_id"`
+ WrittenBy WrittenBy `json:"written_by"`
+}
+
+// Entry is the roster entry a record describes.
+type Entry struct {
+ Box string
+ Enrollment string
+ Cloud boxid.Cloud
+}
+
+// Observation is one DescribeInstances read of one enrolled box.
+type Observation struct {
+ State string
+ Devices []Device
+ RequestID string
+}
+
+var ErrNotRecordable = errors.New("boxvolumes: not a recordable observation")
+
+// ValidEnrollment reports whether e can be a key segment: 1 to 128 of
+// [A-Za-z0-9._-], never "." or "..".
+func ValidEnrollment(e string) bool {
+ if e == "" || len(e) > 128 || e == "." || e == ".." {
+ return false
+ }
+ for _, r := range e {
+ ok := r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '.' || r == '_' || r == '-'
+ if !ok {
+ return false
+ }
+ }
+ return true
+}
+
+// Key is where a record observed at t lives. The time is fixed-width UTC,
+// so keys sort in observation order.
+func Key(enrollment string, t time.Time) string {
+ return Prefix + enrollment + "/" + t.UTC().Format(keyTime) + ".json"
+}
+
+// recordable is every state in which EC2 still reports the instance's
+// mappings. "terminated" is not one: its BlockDeviceMappings is empty (the
+// dev test of 2026-10-06), which would read as every volume removed.
+var recordable = map[string]bool{"pending": true, "running": true, "shutting-down": true, "stopping": true, "stopped": true}
+
+// Observe reads the entry's instance out of a DescribeInstances reply (see
+// termfence.DescribeInstance for the call). It refuses a reply that does
+// not name the instance, names it under another owner, or reports a state
+// whose mappings are not the instance's.
+func Observe(out *ec2.DescribeInstancesOutput, c boxid.Cloud) (Observation, error) {
+ if out == nil || !c.Complete() {
+ return Observation{}, fmt.Errorf("%w: no reply, or no proven cloud", ErrNotRecordable)
+ }
+ var o Observation
+ if id, ok := awsmiddleware.GetRequestIDMetadata(out.ResultMetadata); ok {
+ o.RequestID = id
+ }
+ for _, r := range out.Reservations {
+ for _, inst := range r.Instances {
+ if aws.ToString(inst.InstanceId) != c.InstanceID {
+ continue
+ }
+ if aws.ToString(r.OwnerId) != c.Account {
+ return Observation{}, fmt.Errorf("%w: %s is owned by %q, the roster says %q", ErrNotRecordable, c.InstanceID, aws.ToString(r.OwnerId), c.Account)
+ }
+ if inst.State != nil {
+ o.State = string(inst.State.Name)
+ }
+ if !recordable[o.State] {
+ return Observation{}, fmt.Errorf("%w: state %q", ErrNotRecordable, o.State)
+ }
+ for _, m := range inst.BlockDeviceMappings {
+ if m.Ebs == nil {
+ continue // instance store dies with the instance
+ }
+ o.Devices = append(o.Devices, Device{Device: aws.ToString(m.DeviceName), VolumeID: aws.ToString(m.Ebs.VolumeId),
+ DeleteOnTermination: aws.ToBool(m.Ebs.DeleteOnTermination), Status: string(m.Ebs.Status)})
+ }
+ sort.Slice(o.Devices, func(i, j int) bool { return o.Devices[i].VolumeID < o.Devices[j].VolumeID })
+ return o, nil
+ }
+ }
+ return Observation{}, fmt.Errorf("%w: the reply does not name %s", ErrNotRecordable, c.InstanceID)
+}
+
+func volumeIDs(ds []Device) []string {
+ out := make([]string, 0, len(ds))
+ for _, d := range ds {
+ out = append(out, d.VolumeID)
+ }
+ return out
+}
+
+func removed(prev, cur []Device) []string {
+ have := volumeIDs(cur)
+ var out []string
+ for _, id := range volumeIDs(prev) {
+ if !slices.Contains(have, id) {
+ out = append(out, id)
+ }
+ }
+ sort.Strings(out)
+ return out
+}
+
+// Next decides whether o, made by writer at now, is a new record after
+// prev (nil when the enrollment has none). It is one when there is no
+// record yet, when the mapping or the instance state differs from prev,
+// and otherwise once RefreshEvery has passed.
+func Next(prev *Record, e Entry, writer WrittenBy, o Observation, now time.Time) (Record, bool) {
+ rec := Record{V: 1, Box: e.Box, Enrollment: e.Enrollment, Cloud: e.Cloud, InstanceState: o.State,
+ BlockDevices: o.Devices, RemovedVolumes: []string{}, ObservedAt: now.UTC().Format(time.RFC3339Nano),
+ RequestID: o.RequestID, WrittenBy: writer}
+ if rec.BlockDevices == nil {
+ rec.BlockDevices = []Device{}
+ }
+ if prev == nil {
+ return rec, true
+ }
+ if r := removed(prev.BlockDevices, o.Devices); len(r) > 0 {
+ rec.RemovedVolumes = r
+ }
+ if !slices.Equal(prev.BlockDevices, rec.BlockDevices) || prev.InstanceState != o.State {
+ return rec, true
+ }
+ at, err := time.Parse(time.RFC3339Nano, prev.ObservedAt)
+ return rec, err != nil || now.Sub(at) >= RefreshEvery
+}
+
+// Write stores rec create-only at its key. Records are never replaced or
+// deleted (spec §6); the same bytes written twice are one record.
+func Write(ctx context.Context, s release.Store, rec Record) (string, error) {
+ if !ValidEnrollment(rec.Enrollment) {
+ return "", fmt.Errorf("boxvolumes: %q is not an enrollment", rec.Enrollment)
+ }
+ at, err := time.Parse(time.RFC3339Nano, rec.ObservedAt)
+ if err != nil {
+ return "", fmt.Errorf("boxvolumes: observed_at: %w", err)
+ }
+ body, err := json.Marshal(rec)
+ if err != nil {
+ return "", err
+ }
+ key := Key(rec.Enrollment, at)
+ _, err = s.Put(ctx, key, body, "")
+ if errors.Is(err, release.ErrCASConflict) {
+ cur, _, gerr := s.Get(ctx, key)
+ if gerr == nil && bytes.Equal(cur, body) {
+ return key, nil
+ }
+ return "", fmt.Errorf("boxvolumes: %s already holds another record: %w", key, err)
+ }
+ return key, err
+}
+
+func decode(body []byte) (Record, error) {
+ dec := json.NewDecoder(bytes.NewReader(body))
+ dec.DisallowUnknownFields()
+ var rec Record
+ if err := dec.Decode(&rec); err != nil {
+ return Record{}, err
+ }
+ if _, err := dec.Token(); err != io.EOF {
+ return Record{}, errors.New("trailing data")
+ }
+ return rec, nil
+}
+
+// List returns every record stored for an enrollment, in key order. Any
+// record that cannot be read or parsed is an error, never a gap: a gap
+// could hide the record that shows a removal.
+func List(ctx context.Context, s release.Store, enrollment string) ([]Record, error) {
+ if !ValidEnrollment(enrollment) {
+ return nil, fmt.Errorf("boxvolumes: %q is not an enrollment", enrollment)
+ }
+ keys, err := s.List(ctx, Prefix+enrollment+"/")
+ if err != nil {
+ return nil, err
+ }
+ sort.Strings(keys)
+ out := make([]Record, 0, len(keys))
+ for _, k := range keys {
+ if !strings.HasSuffix(k, ".json") {
+ continue
+ }
+ body, _, err := s.Get(ctx, k)
+ if err != nil {
+ return nil, fmt.Errorf("boxvolumes: %s: %w", k, err)
+ }
+ rec, err := decode(body)
+ if err != nil {
+ return nil, fmt.Errorf("boxvolumes: %s: %w", k, err)
+ }
+ out = append(out, rec)
+ }
+ return out, nil
+}
+
+// incomplete says why a stored record is not evidence of a running
+// instance's mappings, or "" when it is. Write never produces one of
+// these; a record that is one anyway (an older writer, a hand edit, a
+// bug) keeps the chores open rather than being read as "nothing attached".
+func incomplete(r Record) string {
+ if !recordable[r.InstanceState] {
+ return fmt.Sprintf("has state %q, which carries no mapping", r.InstanceState)
+ }
+ if len(r.BlockDevices) == 0 {
+ return "lists no EBS volume" // a platform box always has its root volume
+ }
+ for _, d := range r.BlockDevices {
+ if d.VolumeID == "" || d.Device == "" {
+ return "lists a volume with no id or device"
+ }
+ }
+ return ""
+}
+
+// Verdict is whether a terminated box's park-local chores may be
+// discharged without retirement, and if not, why.
+type Verdict struct {
+ Discharge bool
+ Reason string
+}
+
+// Discharge decides from records (as List returns them) whether every EBS
+// volume e's instance had attached, from the first record taken within
+// EnrollmentWindow of enrolledAt to the last, taken within LastRecordWindow
+// of terminatedAt (the fence's observed_at), was DeleteOnTermination, with
+// none ever removed (spec §5.9, H19). Records for another enrollment or
+// another instance are ignored; a record of this enrollment's that is
+// incomplete, carries no mapping, or has a version this binary does not
+// read keeps the chores open. Anything short of that keeps the chores
+// open for an operator, which costs cleanup and never recovery.
+func Discharge(records []Record, e Entry, enrolledAt, terminatedAt time.Time) Verdict {
+ no := func(format string, a ...any) Verdict { return Verdict{Reason: fmt.Sprintf(format, a...)} }
+ if !e.Cloud.Complete() || e.Enrollment == "" {
+ return no("the roster entry has no proven cloud")
+ }
+ type stamped struct {
+ at time.Time
+ rec Record
+ }
+ var mine []stamped
+ for _, r := range records {
+ if r.Enrollment != e.Enrollment {
+ continue
+ }
+ // A version this binary does not read could hold the adverse
+ // record; it is never skipped.
+ if r.V != 1 {
+ return no("a record for %s has version %d", e.Enrollment, r.V)
+ }
+ if r.Cloud != e.Cloud {
+ continue // another instance's (spec §5.9: ignored)
+ }
+ if reason := incomplete(r); reason != "" {
+ return no("the record observed at %q %s", r.ObservedAt, reason)
+ }
+ at, err := time.Parse(time.RFC3339Nano, r.ObservedAt)
+ if err != nil {
+ return no("record with unreadable observed_at %q", r.ObservedAt)
+ }
+ mine = append(mine, stamped{at, r})
+ }
+ if len(mine) == 0 {
+ return no("no volume record for %s", e.Enrollment)
+ }
+ sort.SliceStable(mine, func(i, j int) bool { return mine[i].at.Before(mine[j].at) })
+ if first := mine[0].at; first.After(enrolledAt.Add(EnrollmentWindow)) {
+ return no("the first record (%s) is more than %s after enrollment", first.Format(time.RFC3339), EnrollmentWindow)
+ }
+ if last := mine[len(mine)-1].at; terminatedAt.Sub(last) > LastRecordWindow {
+ return no("the newest record (%s) is more than %s before the termination", last.Format(time.RFC3339), LastRecordWindow)
+ }
+ for i, s := range mine {
+ for _, d := range s.rec.BlockDevices {
+ if !d.DeleteOnTermination {
+ return no("volume %s was not DeleteOnTermination at %s", d.VolumeID, s.rec.ObservedAt)
+ }
+ }
+ if len(s.rec.RemovedVolumes) > 0 {
+ return no("volumes %v were removed by %s", s.rec.RemovedVolumes, s.rec.ObservedAt)
+ }
+ if i > 0 {
+ if r := removed(mine[i-1].rec.BlockDevices, s.rec.BlockDevices); len(r) > 0 {
+ return no("volumes %v left the mapping by %s", r, s.rec.ObservedAt)
+ }
+ }
+ }
+ return Verdict{Discharge: true}
+}
diff --git a/internal/boxvolumes/volumes_test.go b/internal/boxvolumes/volumes_test.go
new file mode 100644
index 000000000..b3dd7678d
--- /dev/null
+++ b/internal/boxvolumes/volumes_test.go
@@ -0,0 +1,271 @@
+package boxvolumes
+
+import (
+ "context"
+ "errors"
+ "testing"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
+ "github.com/aws/aws-sdk-go-v2/service/ec2"
+ "github.com/aws/aws-sdk-go-v2/service/ec2/types"
+ smithymiddleware "github.com/aws/smithy-go/middleware"
+
+ "github.com/carlosframework/platform/internal/boxid"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+var (
+ cloud = boxid.Cloud{Provider: "aws", Account: "111122223333", Region: "eu-west-1", InstanceID: "i-0abc"}
+ entry = Entry{Box: "annie-welding", Enrollment: "enr-annie-1", Cloud: cloud}
+ writer = WrittenBy{Box: "bert-sawing", Enrollment: "enr-bert-1"}
+ t0 = time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
+)
+
+type vol struct {
+ dev, id string
+ dot bool
+}
+
+func reply(owner string, state types.InstanceStateName, vols ...vol) *ec2.DescribeInstancesOutput {
+ var maps []types.InstanceBlockDeviceMapping
+ for _, v := range vols {
+ maps = append(maps, types.InstanceBlockDeviceMapping{DeviceName: aws.String(v.dev),
+ Ebs: &types.EbsInstanceBlockDevice{VolumeId: aws.String(v.id), DeleteOnTermination: aws.Bool(v.dot), Status: types.AttachmentStatusAttached}})
+ }
+ var md smithymiddleware.Metadata
+ awsmiddleware.SetRequestIDMetadata(&md, "req-1")
+ return &ec2.DescribeInstancesOutput{ResultMetadata: md, Reservations: []types.Reservation{{OwnerId: aws.String(owner),
+ Instances: []types.Instance{{InstanceId: aws.String("i-0abc"), State: &types.InstanceState{Name: state}, BlockDeviceMappings: maps}}}}}
+}
+
+var root, data, scratch = vol{"/dev/xvda", "vol-root", true}, vol{"/dev/xvdb", "vol-data", true}, vol{"/dev/xvdc", "vol-keep", false}
+
+func observe(t *testing.T, out *ec2.DescribeInstancesOutput) Observation {
+ t.Helper()
+ o, err := Observe(out, cloud)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return o
+}
+
+func TestObserveReadsEBSMappingsInVolumeOrder(t *testing.T) {
+ out := reply("111122223333", types.InstanceStateNameRunning, root, data)
+ out.Reservations[0].Instances[0].BlockDeviceMappings = append(out.Reservations[0].Instances[0].BlockDeviceMappings,
+ types.InstanceBlockDeviceMapping{DeviceName: aws.String("/dev/sdz")}) // instance store: no Ebs
+ o := observe(t, out)
+ want := []Device{{"/dev/xvdb", "vol-data", true, "attached"}, {"/dev/xvda", "vol-root", true, "attached"}}
+ if o.State != "running" || o.RequestID != "req-1" || len(o.Devices) != 2 || o.Devices[0] != want[0] || o.Devices[1] != want[1] {
+ t.Fatalf("observation = %+v", o)
+ }
+}
+
+// Review Focus 3: a terminated instance's reply has no mappings. Recorded,
+// it would claim every volume was removed; it must not be recordable.
+func TestObserveRefusesWhatIsNotTheInstancesMapping(t *testing.T) {
+ for name, out := range map[string]*ec2.DescribeInstancesOutput{
+ "terminated": reply("111122223333", types.InstanceStateNameTerminated),
+ "another owner": reply("999999999999", types.InstanceStateNameRunning, root),
+ "another instance": {Reservations: []types.Reservation{{OwnerId: aws.String("111122223333"), Instances: []types.Instance{{InstanceId: aws.String("i-0zzz")}}}}},
+ "empty reply": {},
+ "nil reply": nil,
+ } {
+ t.Run(name, func(t *testing.T) {
+ if o, err := Observe(out, cloud); !errors.Is(err, ErrNotRecordable) {
+ t.Fatalf("observation = %+v, err = %v; want ErrNotRecordable", o, err)
+ }
+ })
+ }
+ // A shutting-down read still lists the mappings and may be recorded.
+ if o := observe(t, reply("111122223333", types.InstanceStateNameShuttingDown, root)); len(o.Devices) != 1 {
+ t.Fatalf("shutting-down: %+v", o)
+ }
+}
+
+func TestNextWritesOnChangeAndAtLeastHourly(t *testing.T) {
+ first, ok := Next(nil, entry, writer, observe(t, reply("111122223333", types.InstanceStateNameRunning, root)), t0)
+ if !ok || first.ObservedAt != "2026-10-06T09:00:00Z" || len(first.RemovedVolumes) != 0 || first.RemovedVolumes == nil {
+ t.Fatalf("first = %+v ok=%v", first, ok)
+ }
+ same := observe(t, reply("111122223333", types.InstanceStateNameRunning, root))
+ if _, ok := Next(&first, entry, writer, same, t0.Add(59*time.Minute)); ok {
+ t.Fatal("wrote an unchanged mapping inside the hour")
+ }
+ if _, ok := Next(&first, entry, writer, same, t0.Add(time.Hour)); !ok {
+ t.Fatal("no record after an hour")
+ }
+ grown, ok := Next(&first, entry, writer, observe(t, reply("111122223333", types.InstanceStateNameRunning, root, scratch)), t0.Add(time.Minute))
+ if !ok || len(grown.BlockDevices) != 2 {
+ t.Fatalf("an attached volume was not recorded: %+v", grown)
+ }
+ shrunk, ok := Next(&grown, entry, writer, observe(t, reply("111122223333", types.InstanceStateNameRunning, root)), t0.Add(2*time.Minute))
+ if !ok || len(shrunk.RemovedVolumes) != 1 || shrunk.RemovedVolumes[0] != "vol-keep" {
+ t.Fatalf("a detached volume is not in removed_volumes: %+v", shrunk)
+ }
+ if _, ok := Next(&first, entry, writer, observe(t, reply("111122223333", types.InstanceStateNameShuttingDown, root)), t0.Add(time.Minute)); !ok {
+ t.Fatal("a state change was not recorded")
+ }
+}
+
+func TestWriteIsCreateOnlyAndListReturnsObservationOrder(t *testing.T) {
+ ctx, st := context.Background(), release.NewDirStore(t.TempDir())
+ a, _ := Next(nil, entry, writer, observe(t, reply("111122223333", types.InstanceStateNameRunning, root)), t0)
+ b, _ := Next(&a, entry, writer, observe(t, reply("111122223333", types.InstanceStateNameRunning, root, data)), t0.Add(10*time.Minute))
+ for _, r := range []Record{b, a} {
+ if _, err := Write(ctx, st, r); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if _, err := Write(ctx, st, a); err != nil {
+ t.Fatalf("the same record twice: %v", err)
+ }
+ other := a
+ other.RequestID = "req-other"
+ if _, err := Write(ctx, st, other); !errors.Is(err, release.ErrCASConflict) {
+ t.Fatalf("a different record under the same key: err = %v", err)
+ }
+ got, err := List(ctx, st, entry.Enrollment)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(got) != 2 || got[0].ObservedAt != a.ObservedAt || got[1].ObservedAt != b.ObservedAt {
+ t.Fatalf("list = %+v", got)
+ }
+ if _, err := Write(ctx, st, Record{Enrollment: "../edges", ObservedAt: a.ObservedAt}); err == nil {
+ t.Fatal("wrote outside the prefix")
+ }
+}
+
+func TestListFailsOnARecordItCannotRead(t *testing.T) {
+ ctx, st := context.Background(), release.NewDirStore(t.TempDir())
+ if _, err := st.Put(ctx, Key(entry.Enrollment, t0), []byte(`{"v":1,"surprise":true}`), ""); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := List(ctx, st, entry.Enrollment); err == nil {
+ t.Fatal("an unreadable record read as a gap")
+ }
+}
+
+func records(t *testing.T, steps ...[]vol) []Record {
+ t.Helper()
+ var out []Record
+ var prev *Record
+ for i, vs := range steps {
+ rec, _ := Next(prev, entry, writer, observe(t, reply("111122223333", types.InstanceStateNameRunning, vs...)), t0.Add(time.Duration(i)*10*time.Minute))
+ out = append(out, rec)
+ prev = &out[len(out)-1]
+ }
+ return out
+}
+
+func TestDischarge(t *testing.T) {
+ enrolled := t0.Add(-5 * time.Minute)
+ terminated := t0.Add(30 * time.Minute)
+ if v := Discharge(records(t, []vol{root}, []vol{root, data}, []vol{root, data}), entry, enrolled, terminated); !v.Discharge {
+ t.Fatalf("all true, nothing removed: %+v", v)
+ }
+ cases := map[string]struct {
+ recs []Record
+ at time.Time
+ }{
+ "no record at all": {nil, enrolled},
+ "a false flag": {records(t, []vol{root}, []vol{root, scratch}), enrolled},
+ "a removed volume": {records(t, []vol{root, data}, []vol{root}), enrolled},
+ "a replaced root volume": {records(t, []vol{root}, []vol{{"/dev/xvda", "vol-root2", true}}), enrolled},
+ "first record long after": {records(t, []vol{root}), t0.Add(-2 * time.Hour)},
+ "only another enrollment's": {func() []Record { r := records(t, []vol{root}); r[0].Enrollment = "enr-other"; return r }(), enrolled},
+ "only another instance's": {func() []Record { r := records(t, []vol{root}); r[0].Cloud.InstanceID = "i-0zzz"; return r }(), enrolled},
+ }
+ for name, tc := range cases {
+ t.Run(name, func(t *testing.T) {
+ if v := Discharge(tc.recs, entry, tc.at, terminated); v.Discharge || v.Reason == "" {
+ t.Fatalf("verdict = %+v, want the chores left open with a reason", v)
+ }
+ })
+ }
+}
+
+// A removal is caught from the sequence even when the record that should
+// have listed it was never written (two workers, one missed write).
+func TestDischargeSeesARemovalNoRecordListed(t *testing.T) {
+ recs := records(t, []vol{root, data}, []vol{root})
+ recs[1].RemovedVolumes = []string{}
+ if v := Discharge(recs, entry, t0, t0.Add(time.Hour)); v.Discharge {
+ t.Fatal("discharged across a silent removal")
+ }
+}
+
+// Controller ruling, 2026-10-06: the newest record must be no more than
+// LastRecordWindow older than the termination observation. A recorder that
+// stopped long before the box died cannot speak for its last hours.
+func TestDischargeNeedsARecentLastRecord(t *testing.T) {
+ recs := records(t, []vol{root}, []vol{root}) // newest at t0+10m
+ newest := t0.Add(10 * time.Minute)
+ if v := Discharge(recs, entry, t0, newest.Add(LastRecordWindow)); !v.Discharge {
+ t.Fatalf("exactly at the window: %+v", v)
+ }
+ if v := Discharge(recs, entry, t0, newest.Add(LastRecordWindow+time.Second)); v.Discharge || v.Reason == "" {
+ t.Fatalf("a newest record older than the window discharged: %+v", v)
+ }
+}
+
+// Astra plan R1-6: what List returns is whatever is stored. A stored record
+// that is not evidence of a running instance's mappings keeps the chores
+// open, however it got there. Each bad record sits beside a good one so
+// that no removal is seen between them: only the record's own check can
+// refuse it.
+func TestDischargeRefusesStoredRecordsThatAreNotEvidence(t *testing.T) {
+ ctx := context.Background()
+ good := records(t, []vol{root})
+ const before = `"enrollment":"enr-annie-1","cloud":{"provider":"aws","account":"111122223333","region":"eu-west-1","instance_id":"i-0abc"},"observed_at":"2026-10-06T08:55:00Z"`
+ const after = `"enrollment":"enr-annie-1","cloud":{"provider":"aws","account":"111122223333","region":"eu-west-1","instance_id":"i-0abc"},"observed_at":"2026-10-06T09:05:00Z"`
+ type bad struct {
+ body string
+ at time.Duration // from t0; the good record is at t0
+ }
+ cases := map[string]bad{
+ // Before the good record: an empty mapping followed by one volume is
+ // an attach, not a removal.
+ "only the binding fields": {`{"v":1,` + before + `}`, -5 * time.Minute},
+ "a terminated read": {`{"v":1,` + before + `,"instance_state":"terminated","block_devices":[],"removed_volumes":[]}`, -5 * time.Minute},
+ // After it, keeping vol-root, so nothing is removed.
+ "a volume with no id": {`{"v":1,` + after + `,"instance_state":"running","block_devices":[{"device":"/dev/xvda","volume_id":"vol-root","delete_on_termination":true,"status":"attached"},{"device":"/dev/xvdb","volume_id":"","delete_on_termination":true,"status":"attached"}],"removed_volumes":[]}`, 5 * time.Minute},
+ "a version 2 record": {`{"v":2,` + after + `,"instance_state":"running","block_devices":[{"device":"/dev/xvda","volume_id":"vol-root","delete_on_termination":true,"status":"attached"}],"removed_volumes":[]}`, 5 * time.Minute},
+ // Astra plan R2-2: one case per check, each failing only that check.
+ "a running read with no volume": {`{"v":1,` + before + `,"instance_state":"running","block_devices":[],"removed_volumes":[]}`, -5 * time.Minute},
+ "a terminated read with its root mapping": {`{"v":1,` + after + `,"instance_state":"terminated","block_devices":[{"device":"/dev/xvda","volume_id":"vol-root","delete_on_termination":true,"status":"attached"}],"removed_volumes":[]}`, 5 * time.Minute},
+ "a volume with no device": {`{"v":1,` + after + `,"instance_state":"running","block_devices":[{"device":"/dev/xvda","volume_id":"vol-root","delete_on_termination":true,"status":"attached"},{"device":"","volume_id":"vol-data","delete_on_termination":true,"status":"attached"}],"removed_volumes":[]}`, 5 * time.Minute},
+ }
+ for name, tc := range cases {
+ t.Run(name, func(t *testing.T) {
+ st := release.NewDirStore(t.TempDir())
+ if _, err := Write(ctx, st, good[0]); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := st.Put(ctx, Key(entry.Enrollment, t0.Add(tc.at)), []byte(tc.body), ""); err != nil {
+ t.Fatal(err)
+ }
+ recs, err := List(ctx, st, entry.Enrollment)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(recs) != 2 {
+ t.Fatalf("listed %d records, want 2", len(recs))
+ }
+ if v := Discharge(recs, entry, t0, t0.Add(time.Hour)); v.Discharge || v.Reason == "" {
+ t.Fatalf("verdict = %+v, want the chores left open", v)
+ }
+ })
+ }
+ // The good record alone, stored and listed, still discharges.
+ st := release.NewDirStore(t.TempDir())
+ if _, err := Write(ctx, st, good[0]); err != nil {
+ t.Fatal(err)
+ }
+ recs, _ := List(ctx, st, entry.Enrollment)
+ if v := Discharge(recs, entry, t0, t0.Add(time.Hour)); !v.Discharge {
+ t.Fatalf("a stored good record: %+v", v)
+ }
+}