amadan

carlos / platform Public

offlinecleanup: the cleanup's wire types, validators, root-only layout checks and seam audit; leasetest keeps null ids and suspension

internal/offlinecleanup (standard library only) holds every wire type of
plan 4 (Retired, CorruptStub, the inventory, Report and root's request
and acknowledgement as declarations for Tasks 5a and 6b), CleanupID,
ValidateRetired and ValidateCorruptStub, the restated host shape, plain
segment, report key and UsableVersionID rules (each pinned to its
original by an equivalence test), RetrySchedule (1 s, 2 s, 4 s, then
every 10 s), Identify, RootOnlyChain, RootOnlyDir and RootOwned
(fstatat under the opened parent), and SetTrustForE2E with its no-op
twin off Linux. TestEveryE2ESeamHasOneSetter audits every test seam's
setters over internal/ (SetTrustForE2E for host.NewE2E only, from Task
9a; RetrySchedule never assigned; moveCfg.rootOnlyChain in host tests).

leasetest.VersionedBucket gains null versions on demand (PutNull) and a
suspended bucket (SetSuspended), each replacing the key's null version
in place, versions with chosen times for ties (PutAt), a listing hook
and the prefixes listed (Listings).

moves gains OfflineOpDir, RetiredFile and CleanupReportKey.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
claude 0a98db1ef65f7b0378ba388cb4f636755c5da907 parent 84917d0
15 files changed, +2193 −1
  • internal/lease/leasetest/bucket.go +76 −1
  • internal/lease/leasetest/s3server.go +3 −0
  • internal/lease/leasetest/versions_test.go +198 −0
  • internal/moves/offline.go +34 −0
  • internal/moves/offline_opdir_test.go +33 −0
  • internal/offlinecleanup/equivalence_test.go +74 −0
  • internal/offlinecleanup/fstatat_linux_amd64.go +22 −0
  • internal/offlinecleanup/fstatat_linux_arm64.go +10 −0
  • internal/offlinecleanup/fstatat_linux_generic.go +10 −0
  • internal/offlinecleanup/layout_linux.go +207 −0
  • internal/offlinecleanup/layout_linux_test.go +296 −0
  • internal/offlinecleanup/layout_other.go +19 −0
  • internal/offlinecleanup/seams_test.go +356 −0
  • internal/offlinecleanup/wire.go +562 −0
  • internal/offlinecleanup/wire_test.go +293 −0
diff --git a/internal/lease/leasetest/bucket.go b/internal/lease/leasetest/bucket.go
index cc1f4a824..596650562 100644
--- a/internal/lease/leasetest/bucket.go
+++ b/internal/lease/leasetest/bucket.go
@@ -36,6 +36,13 @@ type VersionedBucket struct {
PageSize int // listing page size; default 1000
FailListAfterPages int // fail a listing once this many pages have been served; 0 never
FailOn func(op, key string) error // op: list|get|copy|delete|put
+ // ListHook, when set, is given ListVersions' whole answer for prefix
+ // and returns the answer the caller gets: a test reorders ties, or
+ // marks a second entry IsLatest, as a listing it must refuse.
+ ListHook func(prefix string, vs []lease.ObjectVersion) []lease.ObjectVersion
+
+ suspended map[string]bool // bucket -> versioning suspended (SetSuspended)
+ listed []string // every listing's prefix, in order (Listings)
}
func NewVersionedBucket() *VersionedBucket {
@@ -70,10 +77,29 @@ func (f *VersionedBucket) known(bucket string) error {
}
func (f *VersionedBucket) push(bucket, key string, v version) string {
+ return f.pushAs(bucket, key, v, f.suspended[bucket])
+}
+
+// pushAs stores v as the key's newest version. A null version (a write
+// while the bucket is not versioning, or suspended) has the id "null" and
+// replaces the key's existing null version in place, as S3 does; any other
+// gets the next real id.
+func (f *VersionedBucket) pushAs(bucket, key string, v version, null bool) string {
f.next++
f.clock = f.clock.Add(time.Second)
v.id, v.modified = fmt.Sprintf("v%06d", f.next), f.clock
- f.buckets[bucket][key] = append([]version{v}, f.buckets[bucket][key]...)
+ vs := f.buckets[bucket][key]
+ if null {
+ v.id = "null"
+ kept := vs[:0:0]
+ for _, o := range vs {
+ if o.id != "null" {
+ kept = append(kept, o)
+ }
+ }
+ vs = kept
+ }
+ f.buckets[bucket][key] = append([]version{v}, vs...)
return v.id
}
@@ -156,6 +182,51 @@ func (f *VersionedBucket) BareDelete(bucket, key string) string {
return f.push(bucket, key, version{deleteMarker: true})
}
+// PutNull writes body as the key's null version: an object written while
+// the bucket was not versioning (or was suspended), which lists with the
+// version id "null" and which the next such write overwrites in place.
+func (f *VersionedBucket) PutNull(bucket, key string, body []byte) string {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ return f.pushAs(bucket, key, version{body: append([]byte(nil), body...), etag: md5hex(body)}, true)
+}
+
+// PutAt is Put with the version's LastModified set to at, for listings
+// with ties (S3 lists times to the second, and ties are unordered).
+func (f *VersionedBucket) PutAt(bucket, key string, body []byte, at time.Time) string {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ id := f.push(bucket, key, version{body: append([]byte(nil), body...), etag: md5hex(body)})
+ f.buckets[bucket][key][0].modified = at
+ return id
+}
+
+// SetSuspended suspends (or resumes) versioning on bucket: what is there
+// keeps its ids, and every write after is the key's null version.
+func (f *VersionedBucket) SetSuspended(bucket string, on bool) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ if f.suspended == nil {
+ f.suspended = map[string]bool{}
+ }
+ f.suspended[bucket] = on
+}
+
+// Suspended reports SetSuspended's state of bucket.
+func (f *VersionedBucket) Suspended(bucket string) bool {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ return f.suspended[bucket]
+}
+
+// Listings is the prefix of every versions listing asked of the model, in
+// order (a paginated listing over the S3 server counts once).
+func (f *VersionedBucket) Listings() []string {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ return append([]string(nil), f.listed...)
+}
+
func (f *VersionedBucket) LatestID(bucket, key string) string {
f.mu.Lock()
defer f.mu.Unlock()
@@ -204,6 +275,7 @@ func (f *VersionedBucket) ListVersions(ctx context.Context, bucket, prefix strin
if err := f.fail("list", prefix); err != nil {
return nil, err
}
+ f.listed = append(f.listed, prefix)
all := f.listAll(bucket, prefix)
var out []lease.ObjectVersion
for page := 0; ; page++ {
@@ -213,6 +285,9 @@ func (f *VersionedBucket) ListVersions(ctx context.Context, bucket, prefix strin
}
out = append(out, vs...)
if !more {
+ if f.ListHook != nil {
+ out = f.ListHook(prefix, out)
+ }
return out, nil
}
}
diff --git a/internal/lease/leasetest/s3server.go b/internal/lease/leasetest/s3server.go
index ee7819d3b..391aed24b 100644
--- a/internal/lease/leasetest/s3server.go
+++ b/internal/lease/leasetest/s3server.go
@@ -254,6 +254,9 @@ func (f *VersionedBucket) serveList(w http.ResponseWriter, bucket string, q url.
}
all := f.listAll(bucket, q.Get("prefix"))
km, vm := q.Get("key-marker"), q.Get("version-id-marker")
+ if km == "" {
+ f.listed = append(f.listed, q.Get("prefix")) // a listing counts once, at its first page
+ }
start := 0
if km != "" {
start = len(all) // a marker not found resumes at the end
diff --git a/internal/lease/leasetest/versions_test.go b/internal/lease/leasetest/versions_test.go
new file mode 100644
index 000000000..ab64e9d01
--- /dev/null
+++ b/internal/lease/leasetest/versions_test.go
@@ -0,0 +1,198 @@
+package leasetest
+
+import (
+ "context"
+ "encoding/xml"
+ "net/http/httptest"
+ "net/url"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/lease"
+)
+
+// The multi-version model the cleanup's inventory is tested against
+// (self-serve moves plan 4 Task 1): every version id of a key, delete
+// markers, a listing paginated with both markers, deletion by version id,
+// "null" ids on demand and a suspended bucket, ties in LastModified, and
+// the prefixes a listing was asked for.
+
+func ids(vs []lease.ObjectVersion) []string {
+ var out []string
+ for _, v := range vs {
+ out = append(out, v.Key+"@"+v.VersionID)
+ }
+ return out
+}
+
+func TestVersionedBucketKeepsEveryVersionOfAKey(t *testing.T) {
+ ctx := context.Background()
+ f := NewVersionedBucket()
+ f.AddBucket("data")
+ v1 := f.Put("data", "p/db", []byte("1"))
+ v2 := f.Put("data", "p/db", []byte("22"))
+ m := f.BareDelete("data", "p/db")
+ v4 := f.Put("data", "p/db", []byte("4444"))
+ all, err := f.ListVersions(ctx, "data", "p/")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(all) != 4 || all[0].VersionID != v4 || !all[0].IsLatest || all[1].VersionID != m || !all[1].DeleteMarker || all[1].ETag != "" ||
+ all[2].VersionID != v2 || all[2].Size != 2 || all[2].ETag == "" || all[3].VersionID != v1 || all[1].IsLatest || all[3].IsLatest {
+ t.Fatalf("listing %+v", all)
+ }
+ // Deleting a version by id removes that version and nothing else; a
+ // delete of a gone version is success.
+ if err := f.DeleteVersion(ctx, "data", "p/db", v2); err != nil {
+ t.Fatal(err)
+ }
+ if err := f.DeleteVersion(ctx, "data", "p/db", v2); err != nil {
+ t.Fatal(err)
+ }
+ all, _ = f.ListVersions(ctx, "data", "p/")
+ if got := ids(all); len(got) != 3 || got[0] != "p/db@"+v4 || got[1] != "p/db@"+m || got[2] != "p/db@"+v1 {
+ t.Fatalf("after deleting %s: %v", v2, got)
+ }
+ if err := f.DeleteVersion(ctx, "data", "p/db", ""); err != lease.ErrNoVersionID {
+ t.Fatalf("an empty id: %v", err)
+ }
+}
+
+func TestVersionedBucketPaginatesWithBothMarkers(t *testing.T) {
+ f := NewVersionedBucket()
+ f.AddBucket("data")
+ for i := 0; i < 3; i++ {
+ f.Put("data", "p/a", []byte{byte(i)})
+ f.Put("data", "p/b", []byte{byte(i)})
+ }
+ f.PageSize = 2
+ h, _ := NewS3Server(f)
+ var got []string
+ km, vm := "", ""
+ for page := 0; ; page++ {
+ q := url.Values{"versions": {""}, "prefix": {"p/"}}
+ if km != "" {
+ q.Set("key-marker", km)
+ q.Set("version-id-marker", vm)
+ }
+ rec := httptest.NewRecorder()
+ h.ServeHTTP(rec, httptest.NewRequest("GET", "/data?"+q.Encode(), nil))
+ var res struct {
+ IsTruncated bool
+ NextKeyMarker string
+ NextVersionIdMarker string
+ Version []struct{ Key, VersionId string }
+ }
+ if err := xml.Unmarshal(rec.Body.Bytes(), &res); err != nil {
+ t.Fatal(err)
+ }
+ for _, v := range res.Version {
+ got = append(got, v.Key+"@"+v.VersionId)
+ }
+ if !res.IsTruncated {
+ break
+ }
+ if res.NextKeyMarker == "" || res.NextVersionIdMarker == "" || page > 5 {
+ t.Fatalf("page %d: markers %q %q", page, res.NextKeyMarker, res.NextVersionIdMarker)
+ }
+ km, vm = res.NextKeyMarker, res.NextVersionIdMarker
+ }
+ all, _ := f.ListVersions(context.Background(), "data", "p/")
+ if want := ids(all); len(got) != 6 || len(want) != 6 {
+ t.Fatalf("pages %v, whole %v", got, want)
+ } else {
+ for i := range want {
+ if got[i] != want[i] {
+ t.Fatalf("pages %v, whole %v", got, want)
+ }
+ }
+ }
+ if l := f.Listings(); len(l) != 2 || l[0] != "p/" || l[1] != "p/" {
+ t.Fatalf("listings %q: the server records a listing once, at its first page", l)
+ }
+}
+
+func TestVersionedBucketNullIDsOnDemand(t *testing.T) {
+ ctx := context.Background()
+ f := NewVersionedBucket()
+ f.AddBucket("data")
+ // An object written while the bucket was not versioning lists as
+ // "null", and a later such write replaces it in place.
+ if id := f.PutNull("data", "p/old", []byte("a")); id != "null" {
+ t.Fatalf("PutNull answered %q", id)
+ }
+ f.PutNull("data", "p/old", []byte("bb"))
+ all, _ := f.ListVersions(ctx, "data", "p/")
+ if len(all) != 1 || all[0].VersionID != "null" || all[0].Size != 2 || !all[0].IsLatest {
+ t.Fatalf("listing %+v", all)
+ }
+ // A real version over it keeps the null version beneath.
+ real := f.Put("data", "p/old", []byte("ccc"))
+ all, _ = f.ListVersions(ctx, "data", "p/")
+ if len(all) != 2 || all[0].VersionID != real || all[1].VersionID != "null" {
+ t.Fatalf("listing %+v", all)
+ }
+
+ // Suspended: what was written before keeps its real id; every write
+ // after is the key's null version, replaced in place.
+ before := f.Put("data", "p/new", []byte("1"))
+ f.SetSuspended("data", true)
+ if !f.Suspended("data") {
+ t.Fatal("not suspended")
+ }
+ all, _ = f.ListVersions(ctx, "data", "p/new")
+ if len(all) != 1 || all[0].VersionID != before {
+ t.Fatalf("a suspension rewrote the listing: %+v", all)
+ }
+ if id := f.Put("data", "p/new", []byte("22")); id != "null" {
+ t.Fatalf("a suspended write answered %q", id)
+ }
+ if id := f.Put("data", "p/new", []byte("333")); id != "null" {
+ t.Fatalf("a suspended write answered %q", id)
+ }
+ if id := f.BareDelete("data", "p/other"); id != "null" {
+ t.Fatalf("a suspended marker answered %q", id)
+ }
+ all, _ = f.ListVersions(ctx, "data", "p/new")
+ if len(all) != 2 || all[0].VersionID != "null" || all[0].Size != 3 || all[1].VersionID != before {
+ t.Fatalf("listing %+v", all)
+ }
+ f.SetSuspended("data", false)
+ if id := f.Put("data", "p/new", []byte("4")); id == "null" {
+ t.Fatal("versioning on again still writes null")
+ }
+ if err := f.DeleteVersion(ctx, "data", "p/new", "null"); err != nil {
+ t.Fatal(err)
+ }
+ all, _ = f.ListVersions(ctx, "data", "p/new")
+ if len(all) != 2 || all[1].VersionID != before {
+ t.Fatalf("after deleting the null version: %+v", all)
+ }
+}
+
+func TestVersionedBucketTiesAndListHook(t *testing.T) {
+ ctx := context.Background()
+ f := NewVersionedBucket()
+ f.AddBucket("data")
+ at := time.Date(2026, 10, 24, 9, 0, 0, 0, time.UTC)
+ a := f.PutAt("data", "p/k", []byte("a"), at)
+ b := f.PutAt("data", "p/k", []byte("b"), at)
+ all, _ := f.ListVersions(ctx, "data", "p/")
+ if len(all) != 2 || all[0].VersionID != b || all[1].VersionID != a || !all[0].LastModified.Equal(at) || !all[1].LastModified.Equal(at) {
+ t.Fatalf("tied listing %+v", all)
+ }
+ f.ListHook = func(prefix string, vs []lease.ObjectVersion) []lease.ObjectVersion {
+ if prefix != "p/" {
+ t.Errorf("hook saw prefix %q", prefix)
+ }
+ vs[0], vs[1] = vs[1], vs[0]
+ return vs
+ }
+ all, _ = f.ListVersions(ctx, "data", "p/")
+ if all[0].VersionID != a {
+ t.Fatalf("the hook's answer was not returned: %+v", all)
+ }
+ if l := f.Listings(); len(l) != 2 {
+ t.Fatalf("listings %q", l)
+ }
+}
diff --git a/internal/moves/offline.go b/internal/moves/offline.go
index d98664e2c..733c38610 100644
--- a/internal/moves/offline.go
+++ b/internal/moves/offline.go
@@ -695,6 +695,40 @@ func OfflineParkDir(dataDir, host, op, role string) string {
return filepath.Join(dataDir, ".offline-moves", host, op, role)
}
+// OfflineOpDir is a move's own directory on a box,
+// <dataDir>/.offline-moves/<host>/<op>, which holds its park directories
+// and, on the sealing source box, the cleanup obligation (RetiredFile). ""
+// on the names OfflineParkDir refuses: an empty dataDir, a host that fails
+// provision.ValidHostShape, an op that is not a plain name.
+func OfflineOpDir(dataDir, host, op string) string {
+ if dataDir == "" || !provision.ValidHostShape(host) || !offlineSegment(op) {
+ return ""
+ }
+ return filepath.Join(dataDir, ".offline-moves", host, op)
+}
+
+// RetiredFile is the cleanup obligation's file (offlinecleanup.Retired, or
+// its corrupt stub): OfflineOpDir's retired.json, "" when OfflineOpDir is.
+func RetiredFile(dataDir, host, op string) string {
+ dir := OfflineOpDir(dataDir, host, op)
+ if dir == "" {
+ return ""
+ }
+ return filepath.Join(dir, "retired.json")
+}
+
+// CleanupReportKey is where box reports its cleanup of move op on host:
+// OfflineReportPrefix(host, op) + "cleanup/" + box + ".json". "" when the
+// prefix is, or box is not a plain name. Like a report, the box rewrites
+// it.
+func CleanupReportKey(host, op, box string) string {
+ prefix := OfflineReportPrefix(host, op)
+ if prefix == "" || !offlineSegment(box) {
+ return ""
+ }
+ return prefix + "cleanup/" + box + ".json"
+}
+
// OfflineNamesOK is the validity the key helpers share, for a caller that
// wants to ask before it builds anything.
func OfflineNamesOK(host, op, role, box string) bool {
diff --git a/internal/moves/offline_opdir_test.go b/internal/moves/offline_opdir_test.go
new file mode 100644
index 000000000..a54bb91a4
--- /dev/null
+++ b/internal/moves/offline_opdir_test.go
@@ -0,0 +1,33 @@
+package moves
+
+import (
+ "path/filepath"
+ "testing"
+)
+
+// The cleanup's local names (self-serve moves plan 4 Task 1): the op
+// directory holds the park directories, and retired.json beside them; both
+// refuse what OfflineParkDir refuses.
+func TestOfflineOpDirAndRetiredFile(t *testing.T) {
+ const op = "mv_zyxwvutsrqponmlkjihgfedcba"
+ if got := OfflineOpDir("/data", "shop.example.com", op); got != "/data/.offline-moves/shop.example.com/"+op {
+ t.Fatalf("op dir %q", got)
+ }
+ if got := RetiredFile("/data", "shop.example.com", op); got != "/data/.offline-moves/shop.example.com/"+op+"/retired.json" {
+ t.Fatalf("retired file %q", got)
+ }
+ if park := OfflineParkDir("/data", "shop.example.com", op, OfflineRoleSource); filepath.Dir(park) != OfflineOpDir("/data", "shop.example.com", op) {
+ t.Fatalf("the park directory %q is not under the op directory", park)
+ }
+ for _, c := range [][3]string{{"", "shop.example.com", op}, {"/data", "Shop.example.com", op}, {"/data", "../x", op}, {"/data", "shop.example.com", ".."}, {"/data", "shop.example.com", "a/b"}, {"/data", "shop.example.com", ""}} {
+ if OfflineOpDir(c[0], c[1], c[2]) != "" || RetiredFile(c[0], c[1], c[2]) != "" {
+ t.Errorf("%q accepted", c)
+ }
+ }
+ if CleanupReportKey("shop.example.com", op, "ie-a") != OfflineReportPrefix("shop.example.com", op)+"cleanup/ie-a.json" {
+ t.Fatal("the cleanup report key is not under the move's report prefix")
+ }
+ if CleanupReportKey("shop.example.com", op, "a/b") != "" || CleanupReportKey("shop.example.com", "", "ie-a") != "" {
+ t.Fatal("a bad box or op was given a key")
+ }
+}
diff --git a/internal/offlinecleanup/equivalence_test.go b/internal/offlinecleanup/equivalence_test.go
new file mode 100644
index 000000000..79aec8f84
--- /dev/null
+++ b/internal/offlinecleanup/equivalence_test.go
@@ -0,0 +1,74 @@
+package offlinecleanup_test
+
+// The validators internal/offlinecleanup restates, because it imports the
+// standard library only, each pinned to the original it restates. A change
+// to either side that makes them disagree fails here.
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/lease"
+ "github.com/carlosframework/platform/internal/moves"
+ "github.com/carlosframework/platform/internal/offlinecleanup"
+ "github.com/carlosframework/platform/internal/provision"
+)
+
+func TestUsableVersionIDMatchesTheLease(t *testing.T) {
+ for _, v := range []string{"", "null", "Null", "NULL", " null", "3HL4kqtJlcpXroDTDmJ+rmSpXd3dIbrHY", "v000001"} {
+ if got, want := offlinecleanup.UsableVersionID(v), lease.UsableVersionID(v); got != want {
+ t.Errorf("UsableVersionID(%q) = %v, the lease says %v", v, got, want)
+ }
+ }
+}
+
+// hostCases covers every branch of controlname's label rule and the length
+// limit.
+func hostCases() []string {
+ long := strings.Repeat("a", 63)
+ return []string{
+ "", "shop.example.com", "shop", "a.b", "Shop.example.com", "shop..example.com", ".shop.example.com", "shop.example.com.",
+ "-shop.example.com", "shop-.example.com", "sh-op.example.com", "0.example.com", "shop_x.example.com", "shop.example.com/",
+ "../x", "shop example.com", long + ".com", long + "a.com",
+ strings.Repeat(long+".", 3) + strings.Repeat("a", 61), // 253
+ strings.Repeat(long+".", 3) + strings.Repeat("a", 62), // 254
+ "xn--bcher-kva.example", "shop.exämple.com",
+ }
+}
+
+func TestValidHostShapeMatchesProvision(t *testing.T) {
+ for _, h := range hostCases() {
+ if got, want := offlinecleanup.ValidHostShape(h), provision.ValidHostShape(h); got != want {
+ t.Errorf("ValidHostShape(%q) = %v, provision says %v", h, got, want)
+ }
+ }
+}
+
+func segmentCases() []string {
+ return []string{"", ".", "..", "...", "mv_abc", "ie-a", "a.b", "A_Z-0.9", "a/b", `a\b`, "a b", "café", "x\x00", "-", "_"}
+}
+
+// moves.offlineSegment is unexported; OfflineReportPrefix applies it to the
+// op and nothing else once the host is valid.
+func TestPlainSegmentMatchesMoves(t *testing.T) {
+ for _, s := range segmentCases() {
+ if got, want := offlinecleanup.PlainSegment(s), moves.OfflineReportPrefix("shop.example.com", s) != ""; got != want {
+ t.Errorf("PlainSegment(%q) = %v, moves says %v", s, got, want)
+ }
+ }
+}
+
+func TestReportKeyMatchesMoves(t *testing.T) {
+ for _, h := range append(hostCases(), "shop.example.com") {
+ for _, op := range segmentCases() {
+ for _, box := range []string{"ie-a", "", "..", "a/b", "uk-box"} {
+ if got, want := offlinecleanup.ReportKey(h, op, box), moves.CleanupReportKey(h, op, box); got != want {
+ t.Fatalf("ReportKey(%q, %q, %q) = %q, moves says %q", h, op, box, got, want)
+ }
+ }
+ }
+ }
+ if k := moves.CleanupReportKey("shop.example.com", "mv_x", "ie-a"); k != "control/offline-moves/shop.example.com/mv_x/cleanup/ie-a.json" {
+ t.Fatalf("the key form: %q", k)
+ }
+}
diff --git a/internal/offlinecleanup/fstatat_linux_amd64.go b/internal/offlinecleanup/fstatat_linux_amd64.go
new file mode 100644
index 000000000..912b5553e
--- /dev/null
+++ b/internal/offlinecleanup/fstatat_linux_amd64.go
@@ -0,0 +1,22 @@
+//go:build linux && amd64
+
+package offlinecleanup
+
+import (
+ "syscall"
+ "unsafe"
+)
+
+// fstatat is fstatat(2), which the syscall package exports on arm64 but not
+// on amd64, where it is newfstatat.
+func fstatat(dirfd int, name string, st *syscall.Stat_t, flags int) error {
+ p, err := syscall.BytePtrFromString(name)
+ if err != nil {
+ return err
+ }
+ _, _, e := syscall.Syscall6(syscall.SYS_NEWFSTATAT, uintptr(dirfd), uintptr(unsafe.Pointer(p)), uintptr(unsafe.Pointer(st)), uintptr(flags), 0, 0)
+ if e != 0 {
+ return e
+ }
+ return nil
+}
diff --git a/internal/offlinecleanup/fstatat_linux_arm64.go b/internal/offlinecleanup/fstatat_linux_arm64.go
new file mode 100644
index 000000000..c7afda1ae
--- /dev/null
+++ b/internal/offlinecleanup/fstatat_linux_arm64.go
@@ -0,0 +1,10 @@
+//go:build linux && arm64
+
+package offlinecleanup
+
+import "syscall"
+
+// fstatat is fstatat(2).
+func fstatat(dirfd int, name string, st *syscall.Stat_t, flags int) error {
+ return syscall.Fstatat(dirfd, name, st, flags)
+}
diff --git a/internal/offlinecleanup/fstatat_linux_generic.go b/internal/offlinecleanup/fstatat_linux_generic.go
new file mode 100644
index 000000000..9d625929c
--- /dev/null
+++ b/internal/offlinecleanup/fstatat_linux_generic.go
@@ -0,0 +1,10 @@
+//go:build linux && !amd64 && !arm64
+
+package offlinecleanup
+
+import "syscall"
+
+// fstatat is not wired on this architecture: the boxes run arm64 and the
+// development machines amd64. RootOwned then answers false with the error,
+// which fails closed.
+func fstatat(int, string, *syscall.Stat_t, int) error { return syscall.ENOSYS }
diff --git a/internal/offlinecleanup/layout_linux.go b/internal/offlinecleanup/layout_linux.go
new file mode 100644
index 000000000..dde1addc0
--- /dev/null
+++ b/internal/offlinecleanup/layout_linux.go
@@ -0,0 +1,207 @@
+//go:build linux
+
+package offlinecleanup
+
+import (
+ "errors"
+ "fmt"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "syscall"
+)
+
+// Identify is what path is now, read by Lstat (a link is never followed):
+// its device, inode and type, and a link's target. An absent path answers
+// false; any other failure, or a file that is neither a symlink nor a
+// regular file, is an error and fails the caller's step.
+func Identify(path string) (FileIdentity, bool, error) {
+ fi, err := os.Lstat(path)
+ if errors.Is(err, fs.ErrNotExist) {
+ return FileIdentity{}, false, nil
+ }
+ if err != nil {
+ return FileIdentity{}, false, err
+ }
+ st, ok := fi.Sys().(*syscall.Stat_t)
+ if !ok {
+ return FileIdentity{}, false, fmt.Errorf("%s: no stat", path)
+ }
+ id := FileIdentity{Dev: uint64(st.Dev), Inode: uint64(st.Ino)}
+ switch {
+ case fi.Mode()&fs.ModeSymlink != 0:
+ id.Mode = ModeSymlink
+ if id.LinkTarget, err = os.Readlink(path); err != nil {
+ return FileIdentity{}, false, err
+ }
+ if id.LinkTarget == "" {
+ return FileIdentity{}, false, fmt.Errorf("%s: a link with an empty target", path)
+ }
+ case fi.Mode().IsRegular():
+ id.Mode = ModeRegular
+ default:
+ return FileIdentity{}, false, fmt.Errorf("%s is %v, neither a symlink nor a regular file", path, fi.Mode().Type())
+ }
+ return id, true, nil
+}
+
+// RootOnlyChain reports whether every component from / to dir is a
+// directory (a symlink component answers false) owned by uid 0 with no
+// group or other write bit. dir must be absolute. An absent component
+// answers false; any other Lstat failure is an error.
+func RootOnlyChain(dir string) (bool, error) {
+ if !filepath.IsAbs(dir) {
+ return false, fmt.Errorf("root-only chain: %q is not absolute", dir)
+ }
+ dir = filepath.Clean(dir)
+ from := "/"
+ tr := currentTrust()
+ if base, ok := tr.base(dir); ok {
+ from = base // the trust covers its base and below, and nothing above
+ }
+ for _, p := range chainFrom(from, dir) {
+ fi, err := os.Lstat(p)
+ if errors.Is(err, fs.ErrNotExist) {
+ return false, nil
+ }
+ if err != nil {
+ return false, err
+ }
+ if !fi.IsDir() || !tr.rootOnly(fi, p) {
+ return false, nil
+ }
+ }
+ return true, nil
+}
+
+// chainFrom is from, then each component of dir below it, ending at dir.
+func chainFrom(from, dir string) []string {
+ out := []string{from}
+ rest := strings.TrimPrefix(strings.TrimPrefix(dir, from), "/")
+ p := from
+ for _, part := range strings.Split(rest, "/") {
+ if part == "" {
+ continue
+ }
+ p = filepath.Join(p, part)
+ out = append(out, p)
+ }
+ return out
+}
+
+// RootOnlyDir is the opened-parent check: parent, a directory the caller
+// opened O_DIRECTORY|O_NOFOLLOW after RootOnlyChain passed for its path,
+// is a directory owned by uid 0 with no group or other write bit. An fstat
+// failure answers false with it. The check is judged where parent was
+// opened, filepath.Clean(parent.Name()).
+func RootOnlyDir(parent *os.File) (bool, error) {
+ fi, err := parent.Stat()
+ if err != nil {
+ return false, err
+ }
+ return fi.IsDir() && currentTrust().rootOnly(fi, filepath.Clean(parent.Name())), nil
+}
+
+// RootOwned judges the entry name under its opened parent: RootOnlyDir
+// first (false or an error is the answer), then fstatat(parent, name,
+// AT_SYMLINK_NOFOLLOW): a regular file owned by uid 0 with no group or
+// other write bit is true; absent, another file type (a symlink included),
+// another owner or a write bit is false, nil; any other failure is false
+// with it. name is one path element.
+func RootOwned(parent *os.File, name string) (bool, error) {
+ if name == "" || name == "." || name == ".." || strings.ContainsRune(name, '/') {
+ return false, fmt.Errorf("root-owned: %q is not one path element", name)
+ }
+ if ok, err := RootOnlyDir(parent); !ok || err != nil {
+ return false, err
+ }
+ var st syscall.Stat_t
+ err := fstatat(int(parent.Fd()), name, &st, atSymlinkNofollow)
+ if errors.Is(err, syscall.ENOENT) {
+ return false, nil
+ }
+ if err != nil {
+ return false, &fs.PathError{Op: "fstatat", Path: filepath.Join(parent.Name(), name), Err: err}
+ }
+ if st.Mode&syscall.S_IFMT != syscall.S_IFREG {
+ return false, nil
+ }
+ return currentTrust().rootUID(st.Uid, filepath.Clean(parent.Name())) && st.Mode&0o022 == 0, nil
+}
+
+const atSymlinkNofollow = 0x100 // AT_SYMLINK_NOFOLLOW on every Linux architecture
+
+// rootOnly: fi, found at path, is owned by root (or, under the trust, by
+// its uid inside its bases) with no group or other write bit.
+func (t *trust) rootOnly(fi fs.FileInfo, path string) bool {
+ st, ok := fi.Sys().(*syscall.Stat_t)
+ return ok && t.rootUID(st.Uid, path) && fi.Mode().Perm()&0o022 == 0
+}
+
+// rootUID: uid 0, or the trusted uid for a path inside a trusted base.
+func (t *trust) rootUID(uid uint32, path string) bool {
+ return uid == 0 || (t != nil && t.owns(path) && int64(uid) == int64(t.uid))
+}
+
+// trust is SetTrustForE2E's: inside bases, uid stands for root.
+type trust struct {
+ bases []string
+ uid int
+}
+
+var (
+ trustMu sync.Mutex
+ trustOn *trust
+)
+
+func currentTrust() *trust {
+ trustMu.Lock()
+ defer trustMu.Unlock()
+ return trustOn
+}
+
+// base is the trusted base p is in (p itself or below it), if any.
+func (t *trust) base(p string) (string, bool) {
+ if t == nil {
+ return "", false
+ }
+ for _, b := range t.bases {
+ if p == b || strings.HasPrefix(p, b+"/") {
+ return b, true
+ }
+ }
+ return "", false
+}
+
+func (t *trust) owns(p string) bool {
+ _, ok := t.base(p)
+ return ok
+}
+
+// SetTrustForE2E makes the root-only checks treat uid as uid 0, for paths
+// inside bases only (a base itself and below it; a sibling sharing a
+// base's name as a string prefix, a base's parent and everything else keep
+// the real rule), and RootOnlyChain checks a path inside a base from that
+// base down. It is the e2e bridge's trust root (host.NewE2E), so a test
+// world that cannot run as root can stand a directory in for root's; the
+// returned restore puts back what was there before. Nothing else calls it
+// (TestEveryE2ESeamHasOneSetter).
+func SetTrustForE2E(bases []string, uid int) (restore func()) {
+ t := &trust{uid: uid}
+ for _, b := range bases {
+ if b = filepath.Clean(b); filepath.IsAbs(b) && b != "/" {
+ t.bases = append(t.bases, b)
+ }
+ }
+ trustMu.Lock()
+ prev := trustOn
+ trustOn = t
+ trustMu.Unlock()
+ return func() {
+ trustMu.Lock()
+ trustOn = prev
+ trustMu.Unlock()
+ }
+}
diff --git a/internal/offlinecleanup/layout_linux_test.go b/internal/offlinecleanup/layout_linux_test.go
new file mode 100644
index 000000000..b1118fe3d
--- /dev/null
+++ b/internal/offlinecleanup/layout_linux_test.go
@@ -0,0 +1,296 @@
+//go:build linux
+
+package offlinecleanup
+
+import (
+ "os"
+ "path/filepath"
+ "syscall"
+ "testing"
+)
+
+// rootOwnedOnThisSystem is the fixture's precondition for a check against
+// the real system: path is owned by uid 0 with no group or other write bit.
+// A sandbox maps the host's root to nobody, so these halves skip there.
+func rootOwnedOnThisSystem(t *testing.T, paths ...string) {
+ t.Helper()
+ for _, p := range paths {
+ fi, err := os.Lstat(p)
+ if err != nil {
+ t.Skipf("%s: %v", p, err)
+ }
+ st := fi.Sys().(*syscall.Stat_t)
+ if st.Uid != 0 || fi.Mode().Perm()&0o022 != 0 {
+ t.Skipf("%s is uid %d mode %v here, not root-only", p, st.Uid, fi.Mode().Perm())
+ }
+ }
+}
+
+// trusted is a temporary directory the root-only checks treat as root's:
+// the test's own uid stands for uid 0 inside it and nowhere else.
+func trusted(t *testing.T) string {
+ t.Helper()
+ base := t.TempDir()
+ if err := os.Chmod(base, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(SetTrustForE2E([]string{base}, os.Getuid()))
+ return base
+}
+
+func mkdir(t *testing.T, p string, mode os.FileMode) {
+ t.Helper()
+ if err := os.Mkdir(p, mode); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Chmod(p, mode); err != nil { // past the umask
+ t.Fatal(err)
+ }
+}
+
+func write(t *testing.T, p string, mode os.FileMode) {
+ t.Helper()
+ if err := os.WriteFile(p, []byte("x"), mode); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Chmod(p, mode); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestIdentifyUsesLstat(t *testing.T) {
+ dir := t.TempDir()
+ link := filepath.Join(dir, "live")
+ if err := os.Symlink(filepath.Join(dir, "absent-version"), link); err != nil {
+ t.Fatal(err)
+ }
+ id, ok, err := Identify(link)
+ if err != nil || !ok {
+ t.Fatalf("a dangling link: %v %v", ok, err)
+ }
+ fi, _ := os.Lstat(link)
+ st := fi.Sys().(*syscall.Stat_t)
+ if id.Mode != ModeSymlink || id.LinkTarget != filepath.Join(dir, "absent-version") || id.Inode != uint64(st.Ino) || id.Dev != uint64(st.Dev) {
+ t.Fatalf("the link's identity: %+v", id)
+ }
+
+ file := filepath.Join(dir, "programs.json")
+ write(t, file, 0o644)
+ id, ok, err = Identify(file)
+ fi, _ = os.Lstat(file)
+ st = fi.Sys().(*syscall.Stat_t)
+ if err != nil || !ok || id.Mode != ModeRegular || id.LinkTarget != "" || id.Inode != uint64(st.Ino) {
+ t.Fatalf("a regular file: %+v %v %v", id, ok, err)
+ }
+ // A link to the file is the link, never the file.
+ l2 := filepath.Join(dir, "to-file")
+ if err := os.Symlink(file, l2); err != nil {
+ t.Fatal(err)
+ }
+ if id, _, _ := Identify(l2); id.Mode != ModeSymlink || id.Inode == uint64(st.Ino) {
+ t.Fatalf("Identify followed a link: %+v", id)
+ }
+
+ if _, ok, err := Identify(filepath.Join(dir, "nothing")); ok || err != nil {
+ t.Fatalf("absent: %v %v", ok, err)
+ }
+ // Any other failure fails the step: a path through a regular file.
+ if _, ok, err := Identify(filepath.Join(file, "under")); ok || err == nil {
+ t.Fatalf("ENOTDIR answered %v %v", ok, err)
+ }
+ // Another file type is not an identity this plan records.
+ if _, _, err := Identify(dir); err == nil {
+ t.Fatal("a directory has an identity")
+ }
+}
+
+func TestRootOnlyChain(t *testing.T) {
+ // The fixture: a chain the trust makes root's.
+ base := trusted(t)
+ a := filepath.Join(base, "a")
+ mkdir(t, a, 0o755)
+ b := filepath.Join(a, "b")
+ mkdir(t, b, 0o700)
+ if ok, err := RootOnlyChain(b); !ok || err != nil {
+ t.Fatalf("a root-only chain: %v %v", ok, err)
+ }
+ if ok, err := RootOnlyChain(base); !ok || err != nil {
+ t.Fatalf("the base itself: %v %v", ok, err)
+ }
+ // A group or other write bit anywhere on the chain.
+ for _, mode := range []os.FileMode{0o775, 0o757, 0o777} {
+ if err := os.Chmod(a, mode); err != nil {
+ t.Fatal(err)
+ }
+ if ok, err := RootOnlyChain(b); ok || err != nil {
+ t.Fatalf("a %v component answered %v %v", mode, ok, err)
+ }
+ }
+ if err := os.Chmod(a, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ // A symlink component answers false, wherever it points.
+ link := filepath.Join(base, "via")
+ if err := os.Symlink(a, link); err != nil {
+ t.Fatal(err)
+ }
+ if ok, err := RootOnlyChain(filepath.Join(link, "b")); ok || err != nil {
+ t.Fatalf("a symlink component answered %v %v", ok, err)
+ }
+ if ok, err := RootOnlyChain(link); ok || err != nil {
+ t.Fatalf("a symlink as the last component answered %v %v", ok, err)
+ }
+ // A component that is not a directory.
+ f := filepath.Join(base, "file")
+ write(t, f, 0o644)
+ if ok, _ := RootOnlyChain(f); ok {
+ t.Fatal("a regular file is a root-only directory")
+ }
+ // An absent component is not root-only, and not an error.
+ if ok, err := RootOnlyChain(filepath.Join(base, "nothing", "here")); ok || err != nil {
+ t.Fatalf("an absent chain answered %v %v", ok, err)
+ }
+ // A temporary directory outside the trust: owned by the test's uid.
+ if ok, err := RootOnlyChain(t.TempDir()); ok || err != nil {
+ t.Fatalf("a temp directory answered %v %v", ok, err)
+ }
+ if _, err := RootOnlyChain("relative/dir"); err == nil {
+ t.Fatal("a relative path was judged")
+ }
+ // The real system, where it is root's.
+ t.Run("usr", func(t *testing.T) {
+ rootOwnedOnThisSystem(t, "/", "/usr")
+ if ok, err := RootOnlyChain("/usr"); !ok || err != nil {
+ t.Fatalf("/usr: %v %v", ok, err)
+ }
+ })
+}
+
+func open(t *testing.T, dir string) *os.File {
+ t.Helper()
+ f, err := os.OpenFile(dir, os.O_RDONLY|syscall.O_DIRECTORY|syscall.O_NOFOLLOW, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { f.Close() })
+ return f
+}
+
+func TestRootOwnedJudgesTheEntryUnderItsOpenedParent(t *testing.T) {
+ base := trusted(t)
+ dir := filepath.Join(base, "root-cleanup")
+ mkdir(t, dir, 0o755)
+ write(t, filepath.Join(dir, "gated"), 0o644)
+ write(t, filepath.Join(dir, "writable"), 0o664)
+ if err := os.Symlink("gated", filepath.Join(dir, "link")); err != nil {
+ t.Fatal(err)
+ }
+ mkdir(t, filepath.Join(dir, "sub"), 0o755)
+ parent := open(t, dir)
+ if ok, err := RootOnlyDir(parent); !ok || err != nil {
+ t.Fatalf("RootOnlyDir: %v %v", ok, err)
+ }
+ cases := []struct {
+ name string
+ want bool
+ }{{"gated", true}, {"writable", false}, {"link", false}, {"sub", false}, {"absent", false}}
+ for _, c := range cases {
+ if ok, err := RootOwned(parent, c.name); ok != c.want || err != nil {
+ t.Errorf("RootOwned(%s) = %v %v, want %v", c.name, ok, err, c.want)
+ }
+ }
+ if _, err := RootOwned(parent, "../gated"); err == nil {
+ t.Error("a name with a separator was judged")
+ }
+ // The parent is judged first: a writable parent answers false for
+ // every entry, root-owned or not.
+ if err := os.Chmod(dir, 0o775); err != nil {
+ t.Fatal(err)
+ }
+ if ok, err := RootOnlyDir(parent); ok || err != nil {
+ t.Fatalf("a writable parent: %v %v", ok, err)
+ }
+ if ok, err := RootOwned(parent, "gated"); ok || err != nil {
+ t.Fatalf("under a writable parent: %v %v", ok, err)
+ }
+
+ // A temporary directory outside the trust: the test owns it.
+ tmp := t.TempDir()
+ write(t, filepath.Join(tmp, "mine"), 0o644)
+ if err := os.Symlink("mine", filepath.Join(tmp, "link")); err != nil {
+ t.Fatal(err)
+ }
+ tp := open(t, tmp)
+ if ok, err := RootOnlyDir(tp); ok || err != nil {
+ t.Fatalf("a temp directory: %v %v", ok, err)
+ }
+ for _, n := range []string{"mine", "link", "absent"} {
+ if ok, err := RootOwned(tp, n); ok || err != nil {
+ t.Errorf("RootOwned(tmp, %s) = %v %v", n, ok, err)
+ }
+ }
+ // A closed parent: the fstat fails, and says so.
+ closed := open(t, base)
+ closed.Close()
+ if ok, err := RootOnlyDir(closed); ok || err == nil {
+ t.Fatalf("a closed parent: %v %v", ok, err)
+ }
+
+ t.Run("etc", func(t *testing.T) {
+ rootOwnedOnThisSystem(t, "/etc", "/etc/passwd")
+ etc := open(t, "/etc")
+ if ok, err := RootOnlyDir(etc); !ok || err != nil {
+ t.Fatalf("/etc: %v %v", ok, err)
+ }
+ if ok, err := RootOwned(etc, "passwd"); !ok || err != nil {
+ t.Fatalf("/etc/passwd: %v %v", ok, err)
+ }
+ })
+}
+
+// The trust is a uid standing for 0 inside its bases only: a sibling that
+// shares the base's name as a string prefix, the base's parent and any
+// other directory keep the real rule, and restore ends it.
+func TestTrustForE2EAppliesOnlyInsideItsBases(t *testing.T) {
+ parent := t.TempDir()
+ base := filepath.Join(parent, "root")
+ mkdir(t, base, 0o755)
+ sibling := filepath.Join(parent, "root2")
+ mkdir(t, sibling, 0o755)
+ write(t, filepath.Join(base, "gated"), 0o644)
+ write(t, filepath.Join(sibling, "gated"), 0o644)
+ write(t, filepath.Join(parent, "gated"), 0o644)
+
+ restore := SetTrustForE2E([]string{base}, os.Getuid())
+ if ok, _ := RootOnlyChain(base); !ok {
+ t.Fatal("the base is not trusted")
+ }
+ if ok, _ := RootOwned(open(t, base), "gated"); !ok {
+ t.Fatal("a file in the base is not trusted")
+ }
+ if ok, _ := RootOnlyChain(sibling); ok {
+ t.Fatal("a sibling sharing the base's name is trusted")
+ }
+ if ok, _ := RootOwned(open(t, sibling), "gated"); ok {
+ t.Fatal("a sibling's file is trusted")
+ }
+ if ok, _ := RootOwned(open(t, parent), "gated"); ok {
+ t.Fatal("a file beside the base is trusted")
+ }
+ if ok, _ := RootOnlyChain(parent); ok {
+ t.Fatal("the base's parent is trusted")
+ }
+ // Another uid is not the trusted one.
+ other := SetTrustForE2E([]string{base}, os.Getuid()+1)
+ if ok, _ := RootOnlyChain(base); ok {
+ t.Fatal("a uid that is not the trusted one passed")
+ }
+ other()
+ restore()
+ if ok, _ := RootOnlyChain(base); ok {
+ t.Fatal("the trust outlived its restore")
+ }
+ if ok, _ := RootOwned(open(t, base), "gated"); ok {
+ t.Fatal("a file in the base is trusted after the restore")
+ }
+}
diff --git a/internal/offlinecleanup/layout_other.go b/internal/offlinecleanup/layout_other.go
new file mode 100644
index 000000000..3a4e9711c
--- /dev/null
+++ b/internal/offlinecleanup/layout_other.go
@@ -0,0 +1,19 @@
+//go:build !linux
+
+package offlinecleanup
+
+import "os"
+
+// Off Linux the cleanup does not run (the box is Linux): Identify answers
+// absent, and the root-only checks answer false.
+
+func Identify(path string) (FileIdentity, bool, error) { return FileIdentity{}, false, nil }
+
+func RootOnlyChain(dir string) (bool, error) { return false, nil }
+
+func RootOnlyDir(parent *os.File) (bool, error) { return false, nil }
+
+func RootOwned(parent *os.File, name string) (bool, error) { return false, nil }
+
+// SetTrustForE2E changes nothing off Linux: the checks stay false.
+func SetTrustForE2E(bases []string, uid int) (restore func()) { return func() {} }
diff --git a/internal/offlinecleanup/seams_test.go b/internal/offlinecleanup/seams_test.go
new file mode 100644
index 000000000..97e60612a
--- /dev/null
+++ b/internal/offlinecleanup/seams_test.go
@@ -0,0 +1,356 @@
+package offlinecleanup
+
+import (
+ "go/ast"
+ "go/parser"
+ "go/token"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "sort"
+ "strconv"
+ "strings"
+ "testing"
+)
+
+// The test-only seams of the cleanup (plan 4): production code a test uses
+// to stand in for root, a clock, a fault or a schedule. Each is listed with
+// the only places allowed to set it, and the audit below walks every Go
+// file under internal/ and fails on any other setter. A later task that
+// adds a seam adds its entry here.
+//
+// A seam is one of:
+// - call: a function whose every reference (a call or a value) must sit
+// inside one of Setters, named "<dir under internal>.<func>" (a method
+// as "<dir>.<Type>.<method>");
+// - field: a struct field (Owner the struct's package directory) set by a
+// composite literal key or an assignment, only in the _test.go files of
+// the directories named in TestDirs;
+// - var: a package variable never assigned, nor its address taken,
+// outside its declaration.
+type seam struct {
+ Kind string // call, field, var
+ Owner string // the declaring package's directory under internal/
+ Name string
+ Setters []string // call: the functions allowed to reference it
+ TestDirs []string // field: the directories whose _test.go files may set it
+ // OwnTests lets the declaring package's own _test.go files use it.
+ OwnTests bool
+ // MinRefs is how many permitted references must exist (0 while the
+ // setter is not built yet).
+ MinRefs int
+ Why string
+}
+
+var e2eSeams = []seam{
+ {Kind: "call", Owner: "offlinecleanup", Name: "SetTrustForE2E", Setters: []string{"host.NewE2E"}, OwnTests: true, MinRefs: 0,
+ Why: "the root-only checks' trust root for the e2e bridge; the bridge calls it from Task 9a, and nothing else ever may"},
+ {Kind: "var", Owner: "offlinecleanup", Name: "RetrySchedule",
+ Why: "the resend schedule lease and moves share; a test schedule is built with NewBackoff, never by assigning this"},
+ {Kind: "field", Owner: "host", Name: "rootOnlyChain", TestDirs: []string{"host"},
+ Why: "moveCfg's injection of offlinecleanup.RootOnlyChain for the gone evidence's local paths (Task 1); host tests only"},
+}
+
+const modulePath = "github.com/carlosframework/platform/internal/"
+
+type seamRef struct {
+ file, fn string
+ line int
+ test bool
+}
+
+// auditSeams parses every Go file under root (a directory standing for
+// internal/) and returns, per seam name, the references it found that the
+// seam does not permit, and the count of permitted references.
+func auditSeams(t *testing.T, root string, seams []seam) (bad map[string][]string, permitted map[string]int) {
+ t.Helper()
+ bad, permitted = map[string][]string{}, map[string]int{}
+ fset := token.NewFileSet()
+ err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if d.IsDir() {
+ if n := d.Name(); p != root && (n == "testdata" || strings.HasPrefix(n, ".") || strings.HasPrefix(n, "_")) {
+ return filepath.SkipDir
+ }
+ return nil
+ }
+ if !strings.HasSuffix(p, ".go") {
+ return nil
+ }
+ f, err := parser.ParseFile(fset, p, nil, parser.SkipObjectResolution)
+ if err != nil {
+ return err
+ }
+ rel, _ := filepath.Rel(root, p)
+ dir := filepath.ToSlash(filepath.Dir(rel))
+ test := strings.HasSuffix(p, "_test.go")
+ ownPkg := !strings.HasSuffix(f.Name.Name, "_test") // an external test package is not the declaring package
+ imports := map[string]string{} // local name -> dir under internal/
+ for _, im := range f.Imports {
+ path, _ := strconv.Unquote(im.Path.Value)
+ if !strings.HasPrefix(path, modulePath) {
+ continue
+ }
+ d := strings.TrimPrefix(path, modulePath)
+ name := filepath.Base(d)
+ if im.Name != nil {
+ name = im.Name.Name
+ }
+ imports[name] = d
+ }
+ for _, s := range seams {
+ for _, r := range seamRefs(fset, f, s, dir, ownPkg, imports) {
+ r.file, r.test = rel, test
+ where := r.file + ":" + strconv.Itoa(r.line) + " in " + r.fn
+ if seamPermits(s, dir, r) {
+ permitted[s.Name]++
+ continue
+ }
+ bad[s.Name] = append(bad[s.Name], where)
+ }
+ }
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ return bad, permitted
+}
+
+func seamPermits(s seam, dir string, r seamRef) bool {
+ if s.OwnTests && r.test && dir == s.Owner {
+ return true
+ }
+ switch s.Kind {
+ case "call":
+ for _, set := range s.Setters {
+ if dir+"."+r.fn == set {
+ return true
+ }
+ }
+ case "field":
+ for _, d := range s.TestDirs {
+ if r.test && dir == d {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+// seamRefs finds s's references in f, each with its enclosing function.
+func seamRefs(fset *token.FileSet, f *ast.File, s seam, dir string, ownPkg bool, imports map[string]string) []seamRef {
+ var out []seamRef
+ inOwner := dir == s.Owner && ownPkg
+ qualified := func(e ast.Expr) bool { // offlinecleanup.Name, or Name inside its own package
+ switch x := e.(type) {
+ case *ast.SelectorExpr:
+ id, ok := x.X.(*ast.Ident)
+ return ok && x.Sel.Name == s.Name && imports[id.Name] == s.Owner
+ case *ast.Ident:
+ return inOwner && x.Name == s.Name
+ }
+ return false
+ }
+ var fn string
+ var visit func(n ast.Node) bool
+ visit = func(n ast.Node) bool {
+ switch x := n.(type) {
+ case *ast.FuncDecl:
+ fn = x.Name.Name
+ if x.Recv != nil && len(x.Recv.List) > 0 {
+ fn = recvName(x.Recv.List[0].Type) + "." + fn
+ }
+ if s.Kind == "call" && inOwner && x.Recv == nil && x.Name.Name == s.Name {
+ if x.Body != nil {
+ ast.Inspect(x.Body, visit)
+ }
+ fn = ""
+ return false // the declaration itself
+ }
+ if x.Body != nil {
+ ast.Inspect(x.Body, visit)
+ }
+ fn = ""
+ return false
+ case *ast.ValueSpec:
+ if s.Kind == "var" && inOwner && fn == "" {
+ for _, nm := range x.Names {
+ if nm.Name == s.Name {
+ for _, v := range x.Values {
+ ast.Inspect(v, visit)
+ }
+ return false // the declaration itself
+ }
+ }
+ }
+ }
+ where := fn
+ if where == "" {
+ where = "<package scope>"
+ }
+ switch s.Kind {
+ case "call":
+ if e, ok := n.(ast.Expr); ok && qualified(e) {
+ out = append(out, seamRef{fn: where, line: fset.Position(n.Pos()).Line})
+ return false
+ }
+ case "var":
+ switch x := n.(type) {
+ case *ast.AssignStmt:
+ for _, l := range x.Lhs {
+ if writesThrough(l, qualified) {
+ out = append(out, seamRef{fn: where, line: fset.Position(l.Pos()).Line})
+ }
+ }
+ case *ast.IncDecStmt:
+ if writesThrough(x.X, qualified) {
+ out = append(out, seamRef{fn: where, line: fset.Position(x.Pos()).Line})
+ }
+ case *ast.UnaryExpr:
+ if x.Op == token.AND && writesThrough(x.X, qualified) {
+ out = append(out, seamRef{fn: where, line: fset.Position(x.Pos()).Line})
+ }
+ }
+ case "field":
+ if dir != s.Owner {
+ return true // an unexported field is set only inside its package
+ }
+ switch x := n.(type) {
+ case *ast.KeyValueExpr:
+ if id, ok := x.Key.(*ast.Ident); ok && id.Name == s.Name {
+ out = append(out, seamRef{fn: where, line: fset.Position(x.Pos()).Line})
+ }
+ case *ast.AssignStmt:
+ for _, l := range x.Lhs {
+ if sel, ok := l.(*ast.SelectorExpr); ok && sel.Sel.Name == s.Name {
+ out = append(out, seamRef{fn: where, line: fset.Position(l.Pos()).Line})
+ }
+ }
+ }
+ }
+ return true
+ }
+ for _, d := range f.Decls {
+ ast.Inspect(d, visit)
+ }
+ return out
+}
+
+// writesThrough reports whether an assignment target (or an operand whose
+// address is taken) writes through the seam: RetrySchedule itself, or a
+// field or element of it, as in RetrySchedule.steps[0] = y.
+func writesThrough(e ast.Expr, qualified func(ast.Expr) bool) bool {
+ for {
+ if qualified(e) {
+ return true
+ }
+ switch x := e.(type) {
+ case *ast.SelectorExpr:
+ e = x.X
+ case *ast.IndexExpr:
+ e = x.X
+ case *ast.StarExpr:
+ e = x.X
+ case *ast.ParenExpr:
+ e = x.X
+ default:
+ return false
+ }
+ }
+}
+
+func recvName(e ast.Expr) string {
+ switch x := e.(type) {
+ case *ast.StarExpr:
+ return recvName(x.X)
+ case *ast.Ident:
+ return x.Name
+ case *ast.IndexExpr:
+ return recvName(x.X)
+ case *ast.IndexListExpr:
+ return recvName(x.X)
+ }
+ return "?"
+}
+
+func TestEveryE2ESeamHasOneSetter(t *testing.T) {
+ root, err := filepath.Abs("..")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Stat(filepath.Join(root, "offlinecleanup", "seams_test.go")); err != nil {
+ t.Fatalf("the audit does not stand in internal/: %v", err)
+ }
+ bad, permitted := auditSeams(t, root, e2eSeams)
+ names := make([]string, 0, len(bad))
+ for n := range bad {
+ names = append(names, n)
+ }
+ sort.Strings(names)
+ for _, n := range names {
+ for _, where := range bad[n] {
+ t.Errorf("%s is set at %s, which its entry does not permit", n, where)
+ }
+ }
+ for _, s := range e2eSeams {
+ if permitted[s.Name] < s.MinRefs {
+ t.Errorf("%s has %d permitted setters, want at least %d", s.Name, permitted[s.Name], s.MinRefs)
+ }
+ }
+}
+
+// The audit finds what it should: a synthetic tree with one permitted and
+// one forbidden reference of each kind.
+func TestSeamAuditFindsForbiddenSetters(t *testing.T) {
+ root := t.TempDir()
+ files := map[string]string{
+ "offlinecleanup/x.go": `package offlinecleanup
+var RetrySchedule = 1
+func SetTrustForE2E() func() { return nil }
+func own() { RetrySchedule = 2 }
+`,
+ "host/e2e_bridge.go": `package host
+import oc "github.com/carlosframework/platform/internal/offlinecleanup"
+type moveCfg struct{ rootOnlyChain func() }
+func NewE2E() { defer oc.SetTrustForE2E()() }
+func (w *worker) run() { oc.SetTrustForE2E() }
+func prod() { _ = moveCfg{rootOnlyChain: nil} }
+`,
+ "host/x_test.go": `package host
+func fixture(c *moveCfg) { c.rootOnlyChain = nil }
+`,
+ "moves/y.go": `package moves
+import "github.com/carlosframework/platform/internal/offlinecleanup"
+var f = offlinecleanup.SetTrustForE2E
+func g() { offlinecleanup.RetrySchedule = 3; p := &offlinecleanup.RetrySchedule; _ = p; x := offlinecleanup.RetrySchedule; _ = x }
+`,
+ }
+ for name, body := range files {
+ p := filepath.Join(root, name)
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ bad, permitted := auditSeams(t, root, e2eSeams)
+ want := map[string][]string{
+ "SetTrustForE2E": {"host/e2e_bridge.go:5 in worker.run", "moves/y.go:3 in <package scope>"},
+ "RetrySchedule": {"moves/y.go:4 in g", "moves/y.go:4 in g", "offlinecleanup/x.go:4 in own"},
+ "rootOnlyChain": {"host/e2e_bridge.go:6 in prod"},
+ }
+ for n, w := range want {
+ got := append([]string(nil), bad[n]...)
+ sort.Strings(got)
+ if strings.Join(got, "|") != strings.Join(w, "|") {
+ t.Errorf("%s: found %q, want %q", n, got, w)
+ }
+ }
+ if permitted["SetTrustForE2E"] != 1 || permitted["rootOnlyChain"] != 1 {
+ t.Errorf("permitted: %v", permitted)
+ }
+}
diff --git a/internal/offlinecleanup/wire.go b/internal/offlinecleanup/wire.go
new file mode 100644
index 000000000..f7ced1495
--- /dev/null
+++ b/internal/offlinecleanup/wire.go
@@ -0,0 +1,562 @@
+// Package offlinecleanup holds the wire types of a finished offline move's
+// cleanup (self-serve moves plan 4): the obligation a sealing source box
+// records at gone (retired.json), its corrupt stub, the box's report and
+// root's request and acknowledgement, with the validators and the
+// root-only layout checks every side shares.
+//
+// It imports the standard library only, so the agent, root and the
+// console can all read it without an import cycle. The few rules it needs
+// from elsewhere (the host shape, a plain path segment, the report key
+// form, lease.UsableVersionID) are restated here, each pinned to its
+// original by an equivalence test in a package that imports both
+// (equivalence_test.go).
+package offlinecleanup
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "strings"
+ "time"
+)
+
+// Version is Retired.V.
+const Version = 1
+
+// MarkPrefix begins the owner of a cleaning mark on the lifted tombstone
+// (lease.CleaningOwner).
+const MarkPrefix = "cleaning:"
+
+// CleanupID names one obligation: "cu_" and the first 16 hex digits of
+// sha256(box "\n" host "\n" moveID). It is recomputable from the op
+// directory and the box, so a corrupt file, root and the agent all name
+// the same obligation.
+func CleanupID(box, host, moveID string) string {
+ sum := sha256.Sum256([]byte(box + "\n" + host + "\n" + moveID))
+ return "cu_" + hex.EncodeToString(sum[:])[:16]
+}
+
+// The obligation's states. State is the only lifecycle field.
+const (
+ StateOpen = "open"
+ StateFinished = "finished"
+ StateCleaning = "cleaning"
+ StatePublishing = "publishing"
+ StateRootRequested = "root-requested"
+ StateCancelPending = "cancel-pending"
+ StateReleasing = "releasing"
+ StateEnded = "ended"
+ // StateCorrupt is how a CorruptStub is listed; no retired.json holds it.
+ StateCorrupt = "corrupt"
+)
+
+var states = map[string]bool{
+ StateOpen: true, StateFinished: true, StateCleaning: true, StatePublishing: true, StateRootRequested: true,
+ StateCancelPending: true, StateReleasing: true, StateEnded: true,
+}
+
+// The outcomes, set from releasing on (Task 4a's outcome table).
+const (
+ OutcomeCleaned = "cleaned"
+ OutcomeSuperseded = "superseded"
+ OutcomeCancelled = "cancelled"
+)
+
+var outcomes = map[string]bool{OutcomeCleaned: true, OutcomeSuperseded: true, OutcomeCancelled: true}
+
+// The supersession reasons Superseding names while a delete in flight is
+// answered before the obligation leaves cleaning: the conditions (1) to (3)
+// of the plan's "conditions at every key" failing definitely.
+const (
+ SupersededRecord = "record" // (1): the judged record is no longer away at this lifetime
+ SupersededMark = "mark" // (2): lock.json's owner is not this obligation's mark
+ SupersededRow = "row" // (3): the box's registry has a row for the host
+)
+
+var supersessions = map[string]bool{SupersededRecord: true, SupersededMark: true, SupersededRow: true}
+
+// A key's outcome states.
+const (
+ KeyPending = "pending"
+ KeyDeleted = "deleted"
+ KeySuperseded = "superseded"
+ KeyUnsettled = "unsettled"
+)
+
+var keyStates = map[string]bool{KeyPending: true, KeyDeleted: true, KeySuperseded: true, KeyUnsettled: true}
+
+// The local paths an obligation can name. Root derives each path itself
+// from the host; the agent's file names none.
+const (
+ WhichLiveLink = "live-link"
+ WhichProgramsFile = "programs-file"
+)
+
+var whiches = map[string]bool{WhichLiveLink: true, WhichProgramsFile: true}
+
+// ReasonWritableLayout keeps a path whose directory chain is not
+// root-only: no exclusion against a writer sharing the agent's uid is
+// possible there.
+const ReasonWritableLayout = "writable layout"
+
+// The file types an identity records.
+const (
+ ModeSymlink = "symlink"
+ ModeRegular = "regular"
+)
+
+// The two keys under the prefix that are never inventoried, never deleted.
+const (
+ LockKey = "lock.json"
+ BuildStateKey = "build-state.json"
+)
+
+// Retired is retired.json: one finished move's cleanup obligation on the
+// sealing source box, recorded at gone. Its directory is
+// moves.OfflineOpDir(dataDir, host, move).
+type Retired struct {
+ V int `json:"v"`
+ ID string `json:"id"`
+ MoveID string `json:"move_id"`
+ Host string `json:"host"`
+ Account string `json:"account"`
+ App string `json:"app"`
+ Box string `json:"box"`
+ Incarnation uint64 `json:"incarnation"`
+ From string `json:"from"`
+ To string `json:"to"`
+ OpenAt time.Time `json:"open_at,omitzero"`
+ FinalTXID uint64 `json:"final_txid"`
+ TombstoneOwner string `json:"tombstone_owner"`
+ Bucket string `json:"bucket"`
+ BucketRegion string `json:"bucket_region"`
+ // Prefix is the normalised hibernate prefix, ending in "/".
+ Prefix string `json:"prefix"`
+ BucketInventory []InventoryKey `json:"bucket_inventory"`
+ LocalInventory []LocalPath `json:"local_inventory"`
+ LocalLeft []KeptPath `json:"local_left"`
+ WrittenAt time.Time `json:"written_at,omitzero"`
+
+ State string `json:"state"`
+ Outcome string `json:"outcome,omitempty"`
+ Superseding string `json:"superseding,omitempty"`
+
+ FinishedAt time.Time `json:"finished_at,omitzero"`
+ ReservedAt time.Time `json:"reserved_at,omitzero"`
+ MarkedAt time.Time `json:"marked_at,omitzero"`
+ RootRequestedAt time.Time `json:"root_requested_at,omitzero"`
+ CancelRequestedAt time.Time `json:"cancel_requested_at,omitzero"`
+ EndedAt time.Time `json:"ended_at,omitzero"`
+
+ ReserveUncertain bool `json:"reserve_uncertain,omitempty"`
+ InFlight *InFlightDelete `json:"in_flight,omitempty"`
+ Keys map[string]KeyOutcome `json:"keys,omitempty"`
+ Events []Event `json:"events,omitempty"`
+ PagesPending []PendingPage `json:"pages_pending,omitempty"`
+ Root *RootCleanupAck `json:"root,omitempty"`
+
+ // Rev counts the file's writes; PublishedRev is the Rev the box's
+ // report last carried (set alone by Task 5a's markPublished).
+ Rev uint64 `json:"rev"`
+ PublishedRev uint64 `json:"published_rev,omitempty"`
+}
+
+// InventoryKey is one key under the prefix with every version the bucket
+// listed at gone, in deletion order: the versions that were not current,
+// oldest first, then Latest, the one current at gone.
+type InventoryKey struct {
+ Key string `json:"key"`
+ Latest string `json:"latest"`
+ Versions []InventoryVersion `json:"versions"`
+}
+
+// InventoryVersion is one listed entry, an object version or a delete
+// marker. Seq is its index in ListVersions' answer for the whole prefix;
+// ETag is unquoted, "" for a marker.
+type InventoryVersion struct {
+ Seq int `json:"seq"`
+ VersionID string `json:"version_id"`
+ ETag string `json:"etag,omitempty"`
+ Size int64 `json:"size"`
+ DeleteMarker bool `json:"delete_marker,omitempty"`
+ Latest bool `json:"latest,omitempty"`
+ LastModified time.Time `json:"last_modified,omitzero"`
+}
+
+// InFlightDelete is a delete recorded before it is sent; At is the first
+// send, kept on every resend.
+type InFlightDelete struct {
+ Key string `json:"key"`
+ VersionID string `json:"version_id"`
+ At time.Time `json:"at,omitzero"`
+}
+
+// KeyOutcome is one key's progress: Done versions, in deletion order, are
+// known done.
+type KeyOutcome struct {
+ State string `json:"state"`
+ Done int `json:"done"`
+ At time.Time `json:"at,omitzero"`
+}
+
+// Event is one entry of the obligation's log. Task 5a gives it its fields.
+type Event struct{}
+
+// PendingPage is a page the obligation owes. Task 5a gives it its fields.
+type PendingPage struct{}
+
+// Report is the box's cleanup report at moves.CleanupReportKey. Task 5a
+// gives it its fields.
+type Report struct{}
+
+// FileIdentity is what a local path was at gone: Lstat's device and inode,
+// the file type (ModeSymlink or ModeRegular) and a link's target.
+type FileIdentity struct {
+ Dev uint64 `json:"dev"`
+ Inode uint64 `json:"inode"`
+ Mode string `json:"mode"`
+ LinkTarget string `json:"link_target,omitempty"`
+}
+
+// LocalPath is a local path root may remove, by which one it is.
+type LocalPath struct {
+ Which string `json:"which"`
+ Identity FileIdentity `json:"identity"`
+}
+
+// KeptPath is a local path the obligation leaves, and why.
+type KeptPath struct {
+ Which string `json:"which"`
+ Reason string `json:"reason"`
+}
+
+// RootCleanupRequest is the agent's request to root. Task 6b gives it its
+// fields.
+type RootCleanupRequest struct{}
+
+// RootCleanupPath is one path of a request. Task 6b gives it its fields.
+type RootCleanupPath struct{}
+
+// RootCleanupAck is root's acknowledgement. Task 6b gives it its fields.
+type RootCleanupAck struct{}
+
+// CorruptStub stands in place of a retired.json that was neither a valid
+// Retired nor a valid stub: {corrupt: true, id, host, found_at}. FoundAt is
+// the first damaged file's (retired.json.corrupt-1) modification time.
+type CorruptStub struct {
+ Corrupt bool `json:"corrupt"`
+ ID string `json:"id"`
+ Host string `json:"host"`
+ FoundAt time.Time `json:"found_at,omitzero"`
+}
+
+// ValidateCorruptStub reports why c is not the stub of the obligation of
+// box for move on host.
+func ValidateCorruptStub(c CorruptStub, box, host, move string) error {
+ switch {
+ case !c.Corrupt:
+ return errors.New("corrupt stub: corrupt is not set")
+ case c.Host != host:
+ return fmt.Errorf("corrupt stub: host %q, want %q", c.Host, host)
+ case c.ID != CleanupID(box, host, move):
+ return fmt.Errorf("corrupt stub: id %q, want %q", c.ID, CleanupID(box, host, move))
+ case c.FoundAt.IsZero():
+ return errors.New("corrupt stub: no found_at")
+ }
+ return nil
+}
+
+// ValidateRetired reports the first rule r breaks.
+func ValidateRetired(r Retired) error {
+ if r.V != Version {
+ return fmt.Errorf("retired: version %d, want %d", r.V, Version)
+ }
+ if r.ID != CleanupID(r.Box, r.Host, r.MoveID) {
+ return fmt.Errorf("retired: id %q is not this obligation's", r.ID)
+ }
+ if !PlainSegment(r.MoveID) {
+ return fmt.Errorf("retired: move id %q is not a plain segment", r.MoveID)
+ }
+ if !ValidHostShape(r.Host) {
+ return fmt.Errorf("retired: host %q fails the host shape", r.Host)
+ }
+ switch {
+ case r.Account == "":
+ return errors.New("retired: no account")
+ case r.App == "":
+ return errors.New("retired: no app")
+ case r.Box == "":
+ return errors.New("retired: no box")
+ case r.Incarnation == 0:
+ return errors.New("retired: incarnation 0 is not a tracked lifetime")
+ case r.From == r.To:
+ return fmt.Errorf("retired: from and to are both %q", r.From)
+ }
+ if !states[r.State] {
+ return fmt.Errorf("retired: state %q is not a state of the file", r.State)
+ }
+ settled := r.State == StateReleasing || r.State == StateEnded
+ if !settled && r.Outcome != "" {
+ return fmt.Errorf("retired: outcome %q before releasing", r.Outcome)
+ }
+ if settled && !outcomes[r.Outcome] {
+ return fmt.Errorf("retired: outcome %q at %s", r.Outcome, r.State)
+ }
+ if r.Superseding != "" {
+ if !supersessions[r.Superseding] {
+ return fmt.Errorf("retired: %q is not a supersession reason", r.Superseding)
+ }
+ if r.State != StateCleaning || r.InFlight == nil {
+ return fmt.Errorf("retired: superseding is set only in cleaning with a delete in flight (state %s)", r.State)
+ }
+ }
+ if r.WrittenAt.IsZero() {
+ return errors.New("retired: no written_at")
+ }
+ if !strings.HasSuffix(r.Prefix, "/") {
+ return fmt.Errorf("retired: prefix %q does not end in /", r.Prefix)
+ }
+ byKey, err := validateInventory(r.Prefix, r.BucketInventory)
+ if err != nil {
+ return err
+ }
+ seen := map[string]bool{}
+ for _, p := range r.LocalInventory {
+ if err := validWhich(p.Which, seen); err != nil {
+ return err
+ }
+ if err := validIdentity(p.Identity); err != nil {
+ return fmt.Errorf("retired: %s: %w", p.Which, err)
+ }
+ }
+ for _, p := range r.LocalLeft {
+ if err := validWhich(p.Which, seen); err != nil {
+ return err
+ }
+ if p.Reason == "" {
+ return fmt.Errorf("retired: %s is left with no reason", p.Which)
+ }
+ }
+ for key, o := range r.Keys {
+ k, ok := byKey[key]
+ if !ok {
+ return fmt.Errorf("retired: keys names %q, which is not in the inventory", key)
+ }
+ if !keyStates[o.State] {
+ return fmt.Errorf("retired: key %q: state %q", key, o.State)
+ }
+ if o.Done < 0 || o.Done > len(k.Versions) {
+ return fmt.Errorf("retired: key %q: done %d of %d versions", key, o.Done, len(k.Versions))
+ }
+ }
+ if f := r.InFlight; f != nil {
+ k, ok := byKey[f.Key]
+ if !ok || !k.has(f.VersionID) {
+ return fmt.Errorf("retired: in flight %s@%s is not an inventoried version", f.Key, f.VersionID)
+ }
+ if o, ok := r.Keys[f.Key]; !ok || o.State != KeyPending {
+ return fmt.Errorf("retired: in flight %s@%s is on a key that is not pending", f.Key, f.VersionID)
+ }
+ }
+ if r.Rev == 0 {
+ return errors.New("retired: rev 0 (every write counts from 1)")
+ }
+ if r.PublishedRev > r.Rev {
+ return fmt.Errorf("retired: published rev %d past rev %d", r.PublishedRev, r.Rev)
+ }
+ return nil
+}
+
+func (k InventoryKey) has(id string) bool {
+ for _, v := range k.Versions {
+ if v.VersionID == id {
+ return true
+ }
+ }
+ return false
+}
+
+func validateInventory(prefix string, inv []InventoryKey) (map[string]InventoryKey, error) {
+ byKey := map[string]InventoryKey{}
+ seqs := map[int]bool{}
+ for _, k := range inv {
+ if _, dup := byKey[k.Key]; dup {
+ return nil, fmt.Errorf("retired: key %q is inventoried twice", k.Key)
+ }
+ byKey[k.Key] = k
+ if !strings.HasPrefix(k.Key, prefix) {
+ return nil, fmt.Errorf("retired: key %q is not under the prefix %q", k.Key, prefix)
+ }
+ if k.Key == prefix+LockKey || k.Key == prefix+BuildStateKey {
+ return nil, fmt.Errorf("retired: key %q is excluded from the inventory", k.Key)
+ }
+ if len(k.Versions) == 0 {
+ return nil, fmt.Errorf("retired: key %q has no version", k.Key)
+ }
+ ids := map[string]bool{}
+ latest := 0
+ for i, v := range k.Versions {
+ if !UsableVersionID(v.VersionID) {
+ return nil, fmt.Errorf("retired: key %q: version id %q does not name one version", k.Key, v.VersionID)
+ }
+ if ids[v.VersionID] {
+ return nil, fmt.Errorf("retired: key %q: version %q is listed twice", k.Key, v.VersionID)
+ }
+ ids[v.VersionID] = true
+ if v.Seq < 0 || seqs[v.Seq] {
+ return nil, fmt.Errorf("retired: key %q: seq %d is negative or not unique", k.Key, v.Seq)
+ }
+ seqs[v.Seq] = true
+ if (v.ETag != "") == v.DeleteMarker {
+ return nil, fmt.Errorf("retired: key %q: version %q: an etag belongs to exactly the object versions", k.Key, v.VersionID)
+ }
+ if v.Latest {
+ latest++
+ if i != len(k.Versions)-1 {
+ return nil, fmt.Errorf("retired: key %q: the latest version is not last", k.Key)
+ }
+ }
+ if i > 0 && i < len(k.Versions)-1 && !DeletionLess(k.Versions[i-1], v) {
+ return nil, fmt.Errorf("retired: key %q: versions %q and %q are out of deletion order", k.Key, k.Versions[i-1].VersionID, v.VersionID)
+ }
+ }
+ if latest != 1 {
+ return nil, fmt.Errorf("retired: key %q has %d latest versions, want 1", k.Key, latest)
+ }
+ if last := k.Versions[len(k.Versions)-1]; k.Latest != last.VersionID {
+ return nil, fmt.Errorf("retired: key %q names latest %q, but its latest version is %q", k.Key, k.Latest, last.VersionID)
+ }
+ }
+ return byKey, nil
+}
+
+// DeletionLess orders a key's noncurrent versions for deletion:
+// LastModified ascending, ties by Seq descending (the listing is newest
+// first, so the later-listed of two equal times is the older).
+func DeletionLess(a, b InventoryVersion) bool {
+ if !a.LastModified.Equal(b.LastModified) {
+ return a.LastModified.Before(b.LastModified)
+ }
+ return a.Seq > b.Seq
+}
+
+func validWhich(which string, seen map[string]bool) error {
+ if !whiches[which] {
+ return fmt.Errorf("retired: which %q is not a known path", which)
+ }
+ if seen[which] {
+ return fmt.Errorf("retired: %s is named twice", which)
+ }
+ seen[which] = true
+ return nil
+}
+
+func validIdentity(id FileIdentity) error {
+ switch id.Mode {
+ case ModeSymlink:
+ if id.LinkTarget == "" {
+ return errors.New("a symlink identity with no link target")
+ }
+ case ModeRegular:
+ if id.LinkTarget != "" {
+ return errors.New("a regular file identity with a link target")
+ }
+ default:
+ return fmt.Errorf("identity mode %q", id.Mode)
+ }
+ return nil
+}
+
+// UsableVersionID reports whether v names one immutable object version:
+// lease.UsableVersionID's rule (internal/lease/staging.go), restated here.
+// "" is no version; "null" is what a bucket that is not versioning answers,
+// and the next write to the key overwrites that very version.
+func UsableVersionID(v string) bool { return v != "" && v != "null" }
+
+// ValidHostShape is provision.ValidHostShape (controlname's rule),
+// restated: at most 253 bytes of dot-separated labels, each 1 to 63 of
+// a-z, 0-9 and '-', starting and ending with a letter or digit.
+func ValidHostShape(host string) bool {
+ if host == "" || len(host) > 253 {
+ return false
+ }
+ for _, part := range strings.Split(host, ".") {
+ if !validLabel(part) {
+ return false
+ }
+ }
+ return true
+}
+
+func validLabel(s string) bool {
+ if len(s) == 0 || len(s) > 63 {
+ return false
+ }
+ for i := 0; i < len(s); i++ {
+ c := s[i]
+ alnum := c >= 'a' && c <= 'z' || c >= '0' && c <= '9'
+ if !alnum && (c != '-' || i == 0 || i == len(s)-1) {
+ return false
+ }
+ }
+ return true
+}
+
+// PlainSegment is moves' offline segment rule, restated: a string fit to
+// be one path or key segment, not empty, not a dot name, made only of
+// letters, digits and _ . -.
+func PlainSegment(s string) bool {
+ if s == "" || s == "." || s == ".." {
+ return false
+ }
+ for i := 0; i < len(s); i++ {
+ c := s[i]
+ switch {
+ case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9', c == '_', c == '.', c == '-':
+ default:
+ return false
+ }
+ }
+ return true
+}
+
+// ReportKey is moves.CleanupReportKey, restated: where box reports the
+// cleanup of move op on host,
+// control/offline-moves/<host>/<op>/cleanup/<box>.json, "" for an unusable
+// name.
+func ReportKey(host, op, box string) string {
+ if !ValidHostShape(host) || !PlainSegment(op) || !PlainSegment(box) {
+ return ""
+ }
+ return "control/offline-moves/" + host + "/" + op + "/cleanup/" + box + ".json"
+}
+
+// Backoff is a resend schedule: a wait before each of the first resends,
+// then Every for each one after. Build a test schedule with NewBackoff.
+type Backoff struct {
+ steps []time.Duration
+ every time.Duration
+}
+
+// NewBackoff is a schedule of steps, then every.
+func NewBackoff(every time.Duration, steps ...time.Duration) Backoff {
+ return Backoff{steps: append([]time.Duration(nil), steps...), every: every}
+}
+
+// Wait is the wait before resend n+1 (n counts from 0).
+func (b Backoff) Wait(n int) time.Duration {
+ if n < 0 {
+ n = 0
+ }
+ if n < len(b.steps) {
+ return b.steps[n]
+ }
+ return b.every
+}
+
+// RetrySchedule is the resend schedule of every single-attempt write the
+// cleanup's protocol sends until it is answered (the table's PutTable and
+// the lease's lock writes): 1 s, 2 s, 4 s, then every 10 s. Nothing assigns
+// it (TestEveryE2ESeamHasOneSetter).
+var RetrySchedule = NewBackoff(10*time.Second, time.Second, 2*time.Second, 4*time.Second)
diff --git a/internal/offlinecleanup/wire_test.go b/internal/offlinecleanup/wire_test.go
new file mode 100644
index 000000000..983232a0a
--- /dev/null
+++ b/internal/offlinecleanup/wire_test.go
@@ -0,0 +1,293 @@
+package offlinecleanup
+
+import (
+ "encoding/json"
+ "strings"
+ "testing"
+ "time"
+)
+
+const (
+ tBox = "ie-a"
+ tHost = "shop.example.com"
+ tMove = "mv_zyxwvutsrqponmlkjihgfedcba"
+)
+
+var tAt = time.Date(2026, 10, 24, 9, 0, 0, 0, time.UTC)
+
+// validRetired is an obligation as gone writes it: two keys, one with a
+// noncurrent version and a current one, one whose current entry is a
+// delete marker over an object version.
+func validRetired() Retired {
+ p := "instances/" + tHost + "/"
+ return Retired{
+ V: 1, ID: CleanupID(tBox, tHost, tMove), MoveID: tMove, Host: tHost, Account: "bab", App: "shop", Box: tBox,
+ Incarnation: 1, From: "", To: "uk", OpenAt: tAt.Add(-time.Hour), FinalTXID: 77, TombstoneOwner: "moved:" + tMove,
+ Bucket: "data-ie", BucketRegion: "eu-west-1", Prefix: p,
+ BucketInventory: []InventoryKey{
+ {Key: p + "db/0001.ltx", Latest: "v3", Versions: []InventoryVersion{
+ {Seq: 1, VersionID: "v1", ETag: "e1", Size: 10, LastModified: tAt.Add(-3 * time.Minute)},
+ {Seq: 0, VersionID: "v3", ETag: "e3", Size: 12, Latest: true, LastModified: tAt.Add(-time.Minute)},
+ }},
+ {Key: p + "files.tar", Latest: "v5", Versions: []InventoryVersion{
+ {Seq: 3, VersionID: "v4", ETag: "e4", Size: 9, LastModified: tAt.Add(-2 * time.Minute)},
+ {Seq: 2, VersionID: "v5", DeleteMarker: true, Latest: true, LastModified: tAt.Add(-time.Minute)},
+ }},
+ },
+ LocalInventory: []LocalPath{{Which: WhichLiveLink, Identity: FileIdentity{Dev: 1, Inode: 2, Mode: ModeSymlink, LinkTarget: "/opt/carlos/versions/v42"}}},
+ LocalLeft: []KeptPath{{Which: WhichProgramsFile, Reason: ReasonWritableLayout}},
+ WrittenAt: tAt, State: StateOpen, Rev: 1,
+ }
+}
+
+func TestCleanupIDIsDeterministic(t *testing.T) {
+ a := CleanupID(tBox, tHost, tMove)
+ if a != CleanupID(tBox, tHost, tMove) {
+ t.Fatal("two calls disagree")
+ }
+ if !strings.HasPrefix(a, "cu_") || len(a) != len("cu_")+16 {
+ t.Fatalf("id %q is not cu_ and 16 hex digits", a)
+ }
+ // The pinned value: "cu_" + hex(sha256("ie-a\nshop.example.com\nmv_…"))[:16].
+ if a != "cu_301be3c2f4f65c8b" {
+ t.Fatalf("id %q, want the pinned digest", a)
+ }
+ // Each input matters, and the separator keeps them apart.
+ for _, other := range []string{
+ CleanupID("ie-b", tHost, tMove), CleanupID(tBox, "bab.example.com", tMove), CleanupID(tBox, tHost, "mv_abcdefghijklmnopqrstuvwxy2"),
+ CleanupID(tBox+"\n"+tHost, "", tMove),
+ } {
+ if other == a {
+ t.Fatalf("a different triple gave the same id %q", a)
+ }
+ }
+ if MarkPrefix != "cleaning:" {
+ t.Fatalf("MarkPrefix = %q", MarkPrefix)
+ }
+}
+
+func TestValidateRetired(t *testing.T) {
+ if err := ValidateRetired(validRetired()); err != nil {
+ t.Fatalf("the fixture: %v", err)
+ }
+ k0 := func(r *Retired) *InventoryKey { return &r.BucketInventory[0] }
+ cases := []struct {
+ name string
+ mut func(r *Retired)
+ want string
+ }{
+ {"version", func(r *Retired) { r.V = 2 }, "version"},
+ {"id", func(r *Retired) { r.ID = CleanupID("ie-b", tHost, tMove) }, "id"},
+ {"move id not a plain segment", func(r *Retired) { r.MoveID = "../x"; r.ID = CleanupID(r.Box, r.Host, r.MoveID) }, "move"},
+ {"host shape", func(r *Retired) { r.Host = "Shop.Example.com"; r.ID = CleanupID(r.Box, r.Host, r.MoveID) }, "host"},
+ {"account", func(r *Retired) { r.Account = "" }, "account"},
+ {"app", func(r *Retired) { r.App = "" }, "app"},
+ {"box", func(r *Retired) { r.Box = ""; r.ID = CleanupID("", r.Host, r.MoveID) }, "box"},
+ {"incarnation", func(r *Retired) { r.Incarnation = 0 }, "incarnation"},
+ {"from is to", func(r *Retired) { r.To = r.From }, "from"},
+ {"unknown state", func(r *Retired) { r.State = "done" }, "state"},
+ {"corrupt is not a state of a file", func(r *Retired) { r.State = StateCorrupt }, "state"},
+ {"outcome before releasing", func(r *Retired) { r.State = StateCleaning; r.Outcome = OutcomeCleaned }, "outcome"},
+ {"no outcome at releasing", func(r *Retired) { r.State = StateReleasing }, "outcome"},
+ {"unknown outcome", func(r *Retired) { r.State = StateEnded; r.Outcome = "gone" }, "outcome"},
+ {"unknown supersession", func(r *Retired) {
+ r.State = StateCleaning
+ r.Keys = map[string]KeyOutcome{k0(r).Key: {State: KeyPending}}
+ r.InFlight = &InFlightDelete{Key: k0(r).Key, VersionID: "v1", At: tAt}
+ r.Superseding = "because"
+ }, "supersession"},
+ {"superseding outside cleaning", func(r *Retired) { r.State = StateFinished; r.Superseding = SupersededRow }, "superseding"},
+ {"superseding with nothing in flight", func(r *Retired) { r.State = StateCleaning; r.Superseding = SupersededRow }, "superseding"},
+ {"written at", func(r *Retired) { r.WrittenAt = time.Time{} }, "written"},
+ {"prefix without a slash", func(r *Retired) { r.Prefix = strings.TrimSuffix(r.Prefix, "/") }, "prefix"},
+ {"duplicate key", func(r *Retired) { r.BucketInventory[1].Key = r.BucketInventory[0].Key }, "twice"},
+ {"key outside the prefix", func(r *Retired) { k0(r).Key = "instances/shop.example.co/db" }, "prefix"},
+ {"lock.json", func(r *Retired) { k0(r).Key = r.Prefix + "lock.json" }, "excluded"},
+ {"build-state.json", func(r *Retired) { k0(r).Key = r.Prefix + "build-state.json" }, "excluded"},
+ {"no versions", func(r *Retired) { k0(r).Versions = nil }, "no version"},
+ {"null id", func(r *Retired) { k0(r).Versions[0].VersionID = "null" }, `"null"`},
+ {"empty id", func(r *Retired) { k0(r).Versions[0].VersionID = "" }, "version id"},
+ {"duplicate id within a key", func(r *Retired) { k0(r).Versions[0].VersionID = "v3" }, "twice"},
+ {"duplicate seq", func(r *Retired) { r.BucketInventory[1].Versions[0].Seq = 1 }, "seq"},
+ {"negative seq", func(r *Retired) { k0(r).Versions[0].Seq = -1 }, "seq"},
+ {"no latest", func(r *Retired) { k0(r).Versions[1].Latest = false }, "latest"},
+ {"two latest", func(r *Retired) { k0(r).Versions[0].Latest = true }, "latest"},
+ {"latest not last", func(r *Retired) {
+ v := k0(r).Versions
+ v[0], v[1] = v[1], v[0]
+ }, "latest"},
+ {"key latest names another id", func(r *Retired) { k0(r).Latest = "v1" }, "latest"},
+ {"older versions out of order", func(r *Retired) {
+ k := k0(r)
+ k.Versions = []InventoryVersion{
+ {Seq: 4, VersionID: "v0", ETag: "e0", LastModified: tAt.Add(-2 * time.Minute)},
+ {Seq: 1, VersionID: "v1", ETag: "e1", LastModified: tAt.Add(-3 * time.Minute)},
+ k.Versions[1],
+ }
+ }, "order"},
+ {"a tie out of seq order", func(r *Retired) {
+ k := k0(r)
+ k.Versions = []InventoryVersion{
+ {Seq: 1, VersionID: "v1", ETag: "e1", LastModified: tAt.Add(-3 * time.Minute)},
+ {Seq: 4, VersionID: "v0", ETag: "e0", LastModified: tAt.Add(-3 * time.Minute)},
+ k.Versions[1],
+ }
+ }, "order"},
+ {"object version without an etag", func(r *Retired) { k0(r).Versions[0].ETag = "" }, "etag"},
+ {"marker with an etag", func(r *Retired) { r.BucketInventory[1].Versions[1].ETag = "e5" }, "etag"},
+ {"unknown which", func(r *Retired) { r.LocalLeft[0].Which = "db" }, "which"},
+ {"which twice", func(r *Retired) { r.LocalLeft[0].Which = WhichLiveLink }, "twice"},
+ {"which twice within the inventory", func(r *Retired) { r.LocalInventory = append(r.LocalInventory, r.LocalInventory[0]) }, "twice"},
+ {"kept with no reason", func(r *Retired) { r.LocalLeft[0].Reason = "" }, "reason"},
+ {"identity of another file type", func(r *Retired) { r.LocalInventory[0].Identity.Mode = "directory" }, "mode"},
+ {"symlink with no target", func(r *Retired) { r.LocalInventory[0].Identity.LinkTarget = "" }, "target"},
+ {"regular file with a target", func(r *Retired) { r.LocalInventory[0].Identity.Mode = ModeRegular }, "target"},
+ {"keys entry names no inventoried key", func(r *Retired) { r.Keys = map[string]KeyOutcome{r.Prefix + "other": {State: KeyPending}} }, "inventory"},
+ {"keys entry unknown state", func(r *Retired) { r.Keys = map[string]KeyOutcome{k0(r).Key: {State: "gone"}} }, "state"},
+ {"keys entry done past its versions", func(r *Retired) { r.Keys = map[string]KeyOutcome{k0(r).Key: {State: KeyPending, Done: 3}} }, "done"},
+ {"keys entry done negative", func(r *Retired) { r.Keys = map[string]KeyOutcome{k0(r).Key: {State: KeyPending, Done: -1}} }, "done"},
+ {"in flight not inventoried", func(r *Retired) {
+ r.Keys = map[string]KeyOutcome{k0(r).Key: {State: KeyPending}}
+ r.InFlight = &InFlightDelete{Key: k0(r).Key, VersionID: "v9", At: tAt}
+ }, "in flight"},
+ {"in flight on a key not pending", func(r *Retired) {
+ r.Keys = map[string]KeyOutcome{k0(r).Key: {State: KeyDeleted, Done: 2}}
+ r.InFlight = &InFlightDelete{Key: k0(r).Key, VersionID: "v1", At: tAt}
+ }, "in flight"},
+ {"in flight with no keys entry", func(r *Retired) { r.InFlight = &InFlightDelete{Key: k0(r).Key, VersionID: "v1", At: tAt} }, "in flight"},
+ {"rev zero", func(r *Retired) { r.Rev = 0 }, "rev"},
+ {"published past rev", func(r *Retired) { r.PublishedRev = 2 }, "rev"},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ r := validRetired()
+ c.mut(&r)
+ err := ValidateRetired(r)
+ if err == nil {
+ t.Fatal("accepted")
+ }
+ if !strings.Contains(err.Error(), c.want) {
+ t.Fatalf("error %q does not name %q", err, c.want)
+ }
+ })
+ }
+
+ // What the lifecycle legally holds validates.
+ ok := []func(r *Retired){
+ func(r *Retired) { r.State = StateReleasing; r.Outcome = OutcomeCleaned },
+ func(r *Retired) { r.State = StateEnded; r.Outcome = OutcomeSuperseded },
+ func(r *Retired) { r.State = StateEnded; r.Outcome = OutcomeCancelled },
+ func(r *Retired) {
+ r.State = StateCleaning
+ r.Keys = map[string]KeyOutcome{k0(r).Key: {State: KeyPending, Done: 1}, r.BucketInventory[1].Key: {State: KeySuperseded}}
+ r.InFlight = &InFlightDelete{Key: k0(r).Key, VersionID: "v3", At: tAt}
+ r.Superseding = SupersededRecord
+ },
+ func(r *Retired) { r.Keys = map[string]KeyOutcome{k0(r).Key: {State: KeyUnsettled, Done: 2}} },
+ func(r *Retired) { r.LocalInventory, r.LocalLeft = nil, nil; r.BucketInventory = nil },
+ func(r *Retired) { r.Rev, r.PublishedRev = 4, 4 },
+ // A tie in LastModified is ordered by Seq descending.
+ func(r *Retired) {
+ k := k0(r)
+ k.Versions = []InventoryVersion{
+ {Seq: 4, VersionID: "v0", ETag: "e0", LastModified: tAt.Add(-3 * time.Minute)},
+ {Seq: 1, VersionID: "v1", ETag: "e1", LastModified: tAt.Add(-3 * time.Minute)},
+ k.Versions[1],
+ }
+ },
+ }
+ for i, mut := range ok {
+ r := validRetired()
+ mut(&r)
+ if err := ValidateRetired(r); err != nil {
+ t.Errorf("legal shape %d refused: %v", i, err)
+ }
+ }
+}
+
+func TestRetiredJSONIsSnakeCaseWithZeroTimesOmitted(t *testing.T) {
+ b, err := json.Marshal(validRetired())
+ if err != nil {
+ t.Fatal(err)
+ }
+ var m map[string]any
+ if err := json.Unmarshal(b, &m); err != nil {
+ t.Fatal(err)
+ }
+ for _, k := range []string{"v", "id", "move_id", "host", "account", "app", "box", "incarnation", "from", "to", "open_at", "final_txid",
+ "tombstone_owner", "bucket", "bucket_region", "prefix", "bucket_inventory", "local_inventory", "local_left", "written_at", "state", "rev"} {
+ if _, ok := m[k]; !ok {
+ t.Errorf("no %q in %s", k, b)
+ }
+ }
+ for _, k := range []string{"finished_at", "reserved_at", "marked_at", "root_requested_at", "cancel_requested_at", "ended_at"} {
+ if _, ok := m[k]; ok {
+ t.Errorf("a zero %s is written: %s", k, b)
+ }
+ }
+ inv := m["bucket_inventory"].([]any)[0].(map[string]any)
+ ver := inv["versions"].([]any)[0].(map[string]any)
+ for _, k := range []string{"seq", "version_id", "etag", "size", "last_modified"} {
+ if _, ok := ver[k]; !ok {
+ t.Errorf("no %q in a version: %v", k, ver)
+ }
+ }
+ var back Retired
+ if err := json.Unmarshal(b, &back); err != nil || ValidateRetired(back) != nil {
+ t.Fatalf("round trip: %v, %v", err, ValidateRetired(back))
+ }
+}
+
+func TestValidateCorruptStub(t *testing.T) {
+ good := CorruptStub{Corrupt: true, ID: CleanupID(tBox, tHost, tMove), Host: tHost, FoundAt: tAt}
+ if err := ValidateCorruptStub(good, tBox, tHost, tMove); err != nil {
+ t.Fatal(err)
+ }
+ b, _ := json.Marshal(good)
+ if string(b) != `{"corrupt":true,"id":"`+good.ID+`","host":"shop.example.com","found_at":"2026-10-24T09:00:00Z"}` {
+ t.Fatalf("stub body %s", b)
+ }
+ for name, mut := range map[string]func(c *CorruptStub){
+ "not corrupt": func(c *CorruptStub) { c.Corrupt = false },
+ "other host": func(c *CorruptStub) { c.Host = "bab.example.com" },
+ "other id": func(c *CorruptStub) { c.ID = CleanupID("ie-b", tHost, tMove) },
+ "no found at": func(c *CorruptStub) { c.FoundAt = time.Time{} },
+ } {
+ c := good
+ mut(&c)
+ if ValidateCorruptStub(c, tBox, tHost, tMove) == nil {
+ t.Errorf("%s: accepted", name)
+ }
+ }
+ // A Retired is not a stub, and a stub is not a Retired.
+ var asRetired Retired
+ _ = json.Unmarshal(b, &asRetired)
+ if ValidateRetired(asRetired) == nil {
+ t.Fatal("a stub validates as a Retired")
+ }
+ rb, _ := json.Marshal(validRetired())
+ var asStub CorruptStub
+ _ = json.Unmarshal(rb, &asStub)
+ if ValidateCorruptStub(asStub, tBox, tHost, tMove) == nil {
+ t.Fatal("a Retired validates as a stub")
+ }
+}
+
+func TestRetryScheduleIsOneTwoFourThenEveryTen(t *testing.T) {
+ want := []time.Duration{time.Second, 2 * time.Second, 4 * time.Second, 10 * time.Second, 10 * time.Second, 10 * time.Second}
+ for n, w := range want {
+ if got := RetrySchedule.Wait(n); got != w {
+ t.Errorf("wait before resend %d = %s, want %s", n+1, got, w)
+ }
+ }
+ if got := RetrySchedule.Wait(-1); got != time.Second {
+ t.Errorf("a negative attempt waits %s", got)
+ }
+ fast := NewBackoff(time.Millisecond, time.Microsecond)
+ if fast.Wait(0) != time.Microsecond || fast.Wait(1) != time.Millisecond || fast.Wait(7) != time.Millisecond {
+ t.Fatalf("a test schedule: %s %s", fast.Wait(0), fast.Wait(1))
+ }
+ if RetrySchedule.Wait(0) != time.Second {
+ t.Fatal("building a schedule changed the shared one")
+ }
+}