amadan
stopevidence: Reconstruct, stop evidence from cgroup, proc and lease state (§5.3)
stopevidence.Reconstruct runs §5.3 steps 1 to 4 from durable state: kill and drain carlos-exec@<host> (an absent group is an empty tree), no stray pid in the agent's own leaf (read under the cgroup2 mount that /proc/self/mountinfo names, ruling G32), no process holding the database, WAL or SHM, and the lease free. Step 5 is the caller's. !linux returns ErrUnsupported. moves.StoppedReport carries the steps, with TreeStopped (steps 1 to 4) and Complete (every step, StepReplica included, failing closed on any other failed step). lease.LockState reads <location>/lock.json through Acquire's client; the location is the claim holder's (ruling G3). A lock counts as released only once expired by more than RenewMargin; anything unreadable is an error. Copy rows MV-N159 to MV-N162 (pending); the four new files join movesScanFiles.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
10 files changed,
+1203
−1
approved_moves_copy_test.go+6 −1docs/superpowers/plans/2026-10-06-moves-phase1-copy.md+4 −0internal/lease/lockstate.go+95 −0internal/lease/lockstate_test.go+109 −0internal/moves/evidence.go+81 −0internal/moves/evidence_test.go+76 −0internal/stopevidence/reconstruct.go+25 −0internal/stopevidence/reconstruct_linux.go+323 −0internal/stopevidence/reconstruct_linux_test.go+469 −0internal/stopevidence/reconstruct_other.go+15 −0
diff --git a/approved_moves_copy_test.go b/approved_moves_copy_test.go| index 74df2e507..9eb8afdc7 100644 |
| --- a/approved_moves_copy_test.go |
| +++ b/approved_moves_copy_test.go |
| @@ -65,7 +65,7 @@ var movesCopyIDs = []string{ |
| "MV-N139", "MV-N140", "MV-N141", "MV-N142", "MV-N143", "MV-N144", |
| "MV-N145", "MV-N146", "MV-N147", "MV-N148", "MV-N149", "MV-N150", |
| "MV-N151", "MV-N152", "MV-N153", "MV-N154", "MV-N155", "MV-N156", |
| - "MV-N157", "MV-N158", |
| + "MV-N157", "MV-N158", "MV-N159", "MV-N160", "MV-N161", "MV-N162", |
| } |
| // copyRow is one row of the record. The File cell is the file the sentence is |
| @@ -569,6 +569,7 @@ var movesScanFiles = []string{ |
| "internal/lease/lease.go", |
| "internal/lease/leasetest/bucket.go", |
| "internal/lease/leasetest/s3server.go", |
| + "internal/lease/lockstate.go", |
| "internal/lease/mirror.go", |
| "internal/lease/staging.go", |
| "internal/moves/archive/archive.go", |
| @@ -583,6 +584,7 @@ var movesScanFiles = []string{ |
| "internal/moves/budget.go", |
| "internal/moves/check.go", |
| "internal/moves/envreport.go", |
| + "internal/moves/evidence.go", |
| "internal/moves/floor.go", |
| "internal/moves/ids.go", |
| "internal/moves/lease.go", |
| @@ -597,6 +599,9 @@ var movesScanFiles = []string{ |
| "internal/release/materializeheld.go", |
| "internal/release/proofgate.go", |
| "internal/release/refusal.go", |
| + "internal/stopevidence/reconstruct.go", |
| + "internal/stopevidence/reconstruct_linux.go", |
| + "internal/stopevidence/reconstruct_other.go", |
| } |
| // The scan's base is the main commit moves-phase1 last merged: the diff from |
diff --git a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md| index 011cea1fe..58de9a907 100644 |
| --- a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md |
| +++ b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md |
| @@ -198,6 +198,10 @@ Columns: id, file, text, status. |
| | MV-N156 | internal/console/apiinstancemove.go | the source box is not eligible to move from | pending | |
| | MV-N157 | internal/console/apiinstancemove.go | could not read the host roster. try again | pending | |
| | MV-N158 | internal/provision/provision.go | stop evidence budget | pending | |
| +| MV-N159 | internal/stopevidence/reconstruct_linux.go | the instance's processes could not be stopped | pending | |
| +| MV-N160 | internal/stopevidence/reconstruct_linux.go | pid <pid> in the agent's cgroup is not the agent | pending | |
| +| MV-N161 | internal/stopevidence/reconstruct_linux.go | <count> process(es) hold the database open | pending | |
| +| MV-N162 | internal/stopevidence/reconstruct_linux.go | lock.json still held | pending | |
| ## Plan 2 |
diff --git a/internal/lease/lockstate.go b/internal/lease/lockstate.go| new file mode 100644 |
| index 000000000..53d268104 |
| --- /dev/null |
| +++ b/internal/lease/lockstate.go |
| @@ -0,0 +1,95 @@ |
| +package lease |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "fmt" |
| + "io" |
| + "path" |
| + "strings" |
| + "time" |
| + |
| + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" |
| + "github.com/benbjohnson/litestream" |
| +) |
| + |
| +// ErrLockLocation is LockState's answer for a bucket or location it will |
| +// not read: empty, absolute, or not in clean form. |
| +var ErrLockLocation = errors.New("lease: invalid lock location") |
| + |
| +// lockBodyLimit bounds the lock.json read. The leaser writes a few dozen |
| +// bytes; anything near this is not a lock. |
| +const lockBodyLimit = 64 << 10 |
| + |
| +// LockState reads the single-writer lease at s3://bucket/<location>/lock.json |
| +// as the litestream leaser writes it (Acquire, lease.go), through the same |
| +// client Acquire builds, so the read signs for the region and reaches the |
| +// endpoint the writer used. It is §5.3 step 4's observation: it holds no |
| +// lease and writes nothing. |
| +// |
| +// location is the claim holder's Location from a fresh claim read (the |
| +// location moves.ReplicaSource derives its prefix from, ruling G3), never |
| +// activator.HibernatePrefix and never the g-<generation>/ prefix: the lock |
| +// sits at the location and Held.BindGeneration only fixes the data under |
| +// it. One trailing slash is accepted, so ReplicaSource's root prefix reads |
| +// the same lock. |
| +// |
| +// held is true while the lock is present and its expiry has not passed by |
| +// more than RenewMargin, so a holder whose clock runs slightly behind |
| +// still counts as holding. An absent lock (never acquired, or released: |
| +// the leaser deletes it) is held=false with a zero owner and expiry and a |
| +// nil error. An expired lock is held=false with its owner and expiry, for |
| +// the record. Anything else (a missing bucket, an unreadable or garbled |
| +// body, a lock with no expiry) is an error, never "not held". |
| +func LockState(ctx context.Context, bucket, location, region string) (held bool, owner string, expires time.Time, err error) { |
| + key, err := lockKey(bucket, location) |
| + if err != nil { |
| + return false, "", time.Time{}, err |
| + } |
| + client, _, err := newLeaserClient(ctx, region) |
| + if err != nil { |
| + return false, "", time.Time{}, err |
| + } |
| + out, err := client.GetObject(ctx, &awss3.GetObjectInput{Bucket: &bucket, Key: &key}) |
| + if err != nil { |
| + if errors.Is(mapReadErr(err), ErrMirrorNotFound) { |
| + return false, "", time.Time{}, nil |
| + } |
| + return false, "", time.Time{}, fmt.Errorf("lease: read %s: %w", key, err) |
| + } |
| + defer out.Body.Close() |
| + body, err := io.ReadAll(io.LimitReader(out.Body, lockBodyLimit+1)) |
| + if err != nil { |
| + return false, "", time.Time{}, fmt.Errorf("lease: read %s: %w", key, err) |
| + } |
| + if len(body) > lockBodyLimit { |
| + return false, "", time.Time{}, fmt.Errorf("lease: %s is larger than a lock", key) |
| + } |
| + var lock litestream.Lease |
| + if err := json.Unmarshal(body, &lock); err != nil { |
| + return false, "", time.Time{}, fmt.Errorf("lease: decode %s: %w", key, err) |
| + } |
| + if lock.ExpiresAt.IsZero() { |
| + return false, "", time.Time{}, fmt.Errorf("lease: %s has no expiry", key) |
| + } |
| + return lockHeld(lock.ExpiresAt, time.Now()), lock.Owner, lock.ExpiresAt, nil |
| +} |
| + |
| +// lockHeld is §5.3 step 4's rule: a lock is released only once its expiry |
| +// has passed by more than the renewal margin. |
| +func lockHeld(expires, now time.Time) bool { |
| + return !now.After(expires.Add(RenewMargin)) |
| +} |
| + |
| +// lockKey is the leaser's key for location (lss3.Leaser.lockKey with Path |
| +// set, as AcquireTTL sets it), after refusing what Acquire would never be |
| +// given: an empty bucket or location, an absolute one, or one not in clean |
| +// form. |
| +func lockKey(bucket, location string) (string, error) { |
| + loc := strings.TrimSuffix(location, "/") |
| + if bucket == "" || loc == "" || strings.HasPrefix(loc, "/") || strings.Contains(loc, `\`) || path.Clean(loc) != loc || loc == "." || strings.HasPrefix(loc, "../") || loc == ".." { |
| + return "", ErrLockLocation |
| + } |
| + return loc + "/lock.json", nil |
| +} |
diff --git a/internal/lease/lockstate_test.go b/internal/lease/lockstate_test.go| new file mode 100644 |
| index 000000000..4aaa2dc2d |
| --- /dev/null |
| +++ b/internal/lease/lockstate_test.go |
| @@ -0,0 +1,109 @@ |
| +package lease_test |
| + |
| +import ( |
| + "context" |
| + "net/http/httptest" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/lease" |
| + "github.com/carlosframework/platform/internal/lease/leasetest" |
| +) |
| + |
| +// lockFixture points the package's S3 clients at leasetest's local store, |
| +// with credentials the AWS chain resolves without touching the machine's. |
| +func lockFixture(t *testing.T) *leasetest.VersionedBucket { |
| + t.Helper() |
| + bucket := leasetest.NewVersionedBucket() |
| + bucket.AddBucket("data") |
| + handler, _ := leasetest.NewS3Server(bucket) |
| + srv := httptest.NewServer(handler) |
| + t.Cleanup(srv.Close) |
| + t.Setenv("AWS_ACCESS_KEY_ID", "test-key") |
| + t.Setenv("AWS_SECRET_ACCESS_KEY", "test-secret") |
| + t.Setenv("AWS_SESSION_TOKEN", "") |
| + t.Setenv("AWS_EC2_METADATA_DISABLED", "true") |
| + t.Setenv("AWS_CONFIG_FILE", t.TempDir()+"/absent") |
| + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", t.TempDir()+"/absent") |
| + t.Setenv("CARLOS_S3_ENDPOINT", srv.URL) |
| + return bucket |
| +} |
| + |
| +func TestLockStateFollowsAcquireAndRelease(t *testing.T) { |
| + lockFixture(t) |
| + ctx := context.Background() |
| + held, err := lease.Acquire(ctx, "data", "instances/h.example.com", "test-region") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { _ = held.Release(context.Background()) }) |
| + got, _, expires, err := lease.LockState(ctx, "data", "instances/h.example.com", "test-region") |
| + if err != nil || !got || !expires.After(time.Now()) { |
| + t.Fatalf("after acquire: held %v expires %v err %v, want held with an expiry in the future", got, expires, err) |
| + } |
| + // ReplicaSource's prefix carries a trailing slash; the lock is the same. |
| + if got, _, _, err := lease.LockState(ctx, "data", "instances/h.example.com/", "test-region"); err != nil || !got { |
| + t.Fatalf("trailing slash: held %v err %v", got, err) |
| + } |
| + if err := held.Release(ctx); err != nil { |
| + t.Fatal(err) |
| + } |
| + if got, owner, expires, err := lease.LockState(ctx, "data", "instances/h.example.com", "test-region"); err != nil || got || owner != "" || !expires.IsZero() { |
| + t.Fatalf("after release: held %v owner %q expires %v err %v", got, owner, expires, err) |
| + } |
| +} |
| + |
| +func TestLockStateAbsentPrefixIsNotHeld(t *testing.T) { |
| + lockFixture(t) |
| + got, owner, expires, err := lease.LockState(context.Background(), "data", "instances/never.example.com/inc-1", "test-region") |
| + if err != nil || got || owner != "" || !expires.IsZero() { |
| + t.Fatalf("absent: held %v owner %q expires %v err %v", got, owner, expires, err) |
| + } |
| +} |
| + |
| +// §5.3 step 4: a lock counts as released only once it has expired by more |
| +// than the lease's renewal margin, so a holder whose clock runs a little |
| +// behind is still counted as holding. |
| +func TestLockStateExpiredWithinTheMarginIsHeld(t *testing.T) { |
| + bucket := lockFixture(t) |
| + ctx := context.Background() |
| + put := func(loc string, expires time.Time) { |
| + bucket.Put("data", loc+"/lock.json", []byte(`{"generation":3,"expires_at":"`+expires.UTC().Format(time.RFC3339Nano)+`","owner":"box-a"}`)) |
| + } |
| + recent := time.Now().Add(-lease.RenewMargin / 2) |
| + put("instances/recent.example.com", recent) |
| + got, owner, expires, err := lease.LockState(ctx, "data", "instances/recent.example.com", "test-region") |
| + if err != nil || !got || owner != "box-a" || !expires.Equal(recent) { |
| + t.Fatalf("expired within the margin: held %v owner %q expires %v err %v", got, owner, expires, err) |
| + } |
| + old := time.Now().Add(-2 * lease.RenewMargin) |
| + put("instances/old.example.com", old) |
| + got, owner, expires, err = lease.LockState(ctx, "data", "instances/old.example.com", "test-region") |
| + if err != nil || got || owner != "box-a" || !expires.Equal(old) { |
| + t.Fatalf("expired past the margin: held %v owner %q expires %v err %v", got, owner, expires, err) |
| + } |
| +} |
| + |
| +// Anything the reader cannot positively call absent or released is an |
| +// error, never "not held". |
| +func TestLockStateFailsClosed(t *testing.T) { |
| + bucket := lockFixture(t) |
| + ctx := context.Background() |
| + bucket.Put("data", "instances/garbled.example.com/lock.json", []byte(`not json`)) |
| + bucket.Put("data", "instances/noexpiry.example.com/lock.json", []byte(`{"generation":1,"owner":"x"}`)) |
| + for _, c := range []struct{ name, bucket, location string }{ |
| + {"garbled body", "data", "instances/garbled.example.com"}, |
| + {"no expiry", "data", "instances/noexpiry.example.com"}, |
| + {"missing bucket", "nobucket", "instances/h.example.com"}, |
| + {"empty location", "data", ""}, |
| + {"absolute location", "data", "/instances/h.example.com"}, |
| + {"dot-dot location", "data", "instances/../h.example.com"}, |
| + {"double slash", "data", "instances//h.example.com"}, |
| + {"empty bucket", "", "instances/h.example.com"}, |
| + } { |
| + got, _, _, err := lease.LockState(ctx, c.bucket, c.location, "test-region") |
| + if err == nil || got { |
| + t.Fatalf("%s: held %v err %v, want an error", c.name, got, err) |
| + } |
| + } |
| +} |
diff --git a/internal/moves/evidence.go b/internal/moves/evidence.go| new file mode 100644 |
| index 000000000..95e39bab8 |
| --- /dev/null |
| +++ b/internal/moves/evidence.go |
| @@ -0,0 +1,81 @@ |
| +package moves |
| + |
| +import "time" |
| + |
| +// The steps of a StoppedReport, in §5.3's order. StepTree to StepLease |
| +// are what stopevidence.Reconstruct observes from the box's durable state; |
| +// StepReplica is the caller's (the cut's step 5: the final checkpoint and |
| +// the digests), set after the checkpoint. |
| +const ( |
| + // StepTree: the host's cgroup was killed and drained (or never existed). |
| + StepTree = "tree" |
| + // StepStray: the agent's own cgroup leaf holds only the agent. |
| + StepStray = "stray" |
| + // StepFDs: no visible process holds the database, its WAL or its SHM. |
| + StepFDs = "fds" |
| + // StepLease: the host's lock.json is absent, released, or expired by |
| + // more than the lease's renewal margin. |
| + StepLease = "lease" |
| + // StepReplica: the final checkpoint reached the replica. |
| + StepReplica = "replica" |
| +) |
| + |
| +// treeSteps are §5.3 steps 1 to 4, what `stopping` needs before the final |
| +// checkpoint. allSteps adds the replica. |
| +var ( |
| + treeSteps = [...]string{StepTree, StepStray, StepFDs, StepLease} |
| + allSteps = [...]string{StepTree, StepStray, StepFDs, StepLease, StepReplica} |
| +) |
| + |
| +// StoppedReport is a role's stop evidence for one move operation (§5.3), |
| +// written to moves/<acct>/<app>/<host>/<op id>/stopped-<role>.json. Every |
| +// observation is from durable, observable state, so a restarted agent |
| +// writes the same report. Steps holds one Check per step id; a step that |
| +// never ran is absent, and an absent step is not OK. Stray lists the pids |
| +// StepStray found in the agent's leaf besides the agent, for the alert. |
| +type StoppedReport struct { |
| + V int `json:"v"` |
| + Role string `json:"role"` |
| + Box string `json:"box"` |
| + Incarnation string `json:"incarnation"` |
| + ClaimGen uint64 `json:"claim_gen"` |
| + Host string `json:"host"` |
| + OpID string `json:"op_id"` |
| + Steps map[string]Check `json:"steps"` |
| + Stray []int `json:"stray,omitempty"` |
| + FinalTXID uint64 `json:"final_txid"` |
| + DBSHA256 string `json:"db_sha256"` |
| + FilesPlain string `json:"files_plain"` |
| + TimersSHA256 string `json:"timers_sha256"` |
| + CheckedAt time.Time `json:"checked_at"` |
| +} |
| + |
| +// TreeStopped reports §5.3 steps 1 to 4 all present and OK: the writer has |
| +// fully stopped, so the final checkpoint may run. |
| +func (r StoppedReport) TreeStopped() bool { |
| + return r.stepsOK(treeSteps[:]) |
| +} |
| + |
| +// Complete reports every step OK, StepReplica included. It fails closed on |
| +// any other step recorded in the report and not OK, so a step a later |
| +// version adds cannot be ignored by an older reader. |
| +func (r StoppedReport) Complete() bool { |
| + if !r.stepsOK(allSteps[:]) { |
| + return false |
| + } |
| + for _, c := range r.Steps { |
| + if !c.OK { |
| + return false |
| + } |
| + } |
| + return true |
| +} |
| + |
| +func (r StoppedReport) stepsOK(ids []string) bool { |
| + for _, id := range ids { |
| + if c, ok := r.Steps[id]; !ok || !c.OK { |
| + return false |
| + } |
| + } |
| + return true |
| +} |
diff --git a/internal/moves/evidence_test.go b/internal/moves/evidence_test.go| new file mode 100644 |
| index 000000000..01f28c88e |
| --- /dev/null |
| +++ b/internal/moves/evidence_test.go |
| @@ -0,0 +1,76 @@ |
| +package moves |
| + |
| +import ( |
| + "encoding/json" |
| + "strings" |
| + "testing" |
| +) |
| + |
| +func stoppedWith(ok map[string]bool) StoppedReport { |
| + r := StoppedReport{V: 1, Steps: map[string]Check{}} |
| + for id, pass := range ok { |
| + if pass { |
| + r.Steps[id] = Check{OK: true} |
| + } else { |
| + r.Steps[id] = Check{Reason: "no"} |
| + } |
| + } |
| + return r |
| +} |
| + |
| +func TestStoppedReportTreeStoppedNeedsStepsOneToFour(t *testing.T) { |
| + all := map[string]bool{StepTree: true, StepStray: true, StepFDs: true, StepLease: true} |
| + if r := stoppedWith(all); !r.TreeStopped() || r.Complete() { |
| + t.Fatalf("steps 1 to 4 OK: TreeStopped %v Complete %v, want true false", r.TreeStopped(), r.Complete()) |
| + } |
| + for _, step := range []string{StepTree, StepStray, StepFDs, StepLease} { |
| + failed := map[string]bool{StepTree: true, StepStray: true, StepFDs: true, StepLease: true, StepReplica: true} |
| + failed[step] = false |
| + if r := stoppedWith(failed); r.TreeStopped() || r.Complete() { |
| + t.Fatalf("%s failed but TreeStopped %v Complete %v", step, r.TreeStopped(), r.Complete()) |
| + } |
| + missing := map[string]bool{StepTree: true, StepStray: true, StepFDs: true, StepLease: true, StepReplica: true} |
| + delete(missing, step) |
| + if r := stoppedWith(missing); r.TreeStopped() || r.Complete() { |
| + t.Fatalf("%s never ran but TreeStopped %v Complete %v", step, r.TreeStopped(), r.Complete()) |
| + } |
| + } |
| + if (StoppedReport{}).TreeStopped() || (StoppedReport{}).Complete() { |
| + t.Fatal("the zero report passed") |
| + } |
| +} |
| + |
| +func TestStoppedReportCompleteNeedsTheReplicaAndEveryStep(t *testing.T) { |
| + all := map[string]bool{StepTree: true, StepStray: true, StepFDs: true, StepLease: true, StepReplica: true} |
| + if r := stoppedWith(all); !r.Complete() || !r.TreeStopped() { |
| + t.Fatalf("every step OK: Complete %v TreeStopped %v", r.Complete(), r.TreeStopped()) |
| + } |
| + all[StepReplica] = false |
| + if r := stoppedWith(all); r.Complete() || !r.TreeStopped() { |
| + t.Fatalf("replica failed: Complete %v TreeStopped %v, want false true", r.Complete(), r.TreeStopped()) |
| + } |
| + // A step this version does not know about still has to pass: Complete |
| + // fails closed on anything a newer writer recorded as failed. |
| + all[StepReplica] = true |
| + all["later"] = false |
| + if r := stoppedWith(all); r.Complete() { |
| + t.Fatal("a failed extra step passed Complete") |
| + } |
| +} |
| + |
| +func TestStoppedReportWireNames(t *testing.T) { |
| + r := StoppedReport{V: 1, Role: "source", Box: "b", Incarnation: "i", ClaimGen: 7, Host: "h.example.com", OpID: "op", |
| + Steps: map[string]Check{StepTree: {OK: true}}, Stray: []int{9}, FinalTXID: 12, |
| + DBSHA256: "d", FilesPlain: "f", TimersSHA256: "t"} |
| + raw, err := json.Marshal(r) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, name := range []string{`"v":1`, `"role":"source"`, `"box":"b"`, `"incarnation":"i"`, `"claim_gen":7`, |
| + `"host":"h.example.com"`, `"op_id":"op"`, `"steps":{"tree":{"ok":true,"reason":""}}`, `"stray":[9]`, |
| + `"final_txid":12`, `"db_sha256":"d"`, `"files_plain":"f"`, `"timers_sha256":"t"`, `"checked_at":`} { |
| + if !strings.Contains(string(raw), name) { |
| + t.Fatalf("%s missing from %s", name, raw) |
| + } |
| + } |
| +} |
diff --git a/internal/stopevidence/reconstruct.go b/internal/stopevidence/reconstruct.go| new file mode 100644 |
| index 000000000..46466ac20 |
| --- /dev/null |
| +++ b/internal/stopevidence/reconstruct.go |
| @@ -0,0 +1,25 @@ |
| +package stopevidence |
| + |
| +import ( |
| + "context" |
| + |
| + "github.com/carlosframework/platform/internal/cgroup" |
| +) |
| + |
| +// ReconstructInputs is what Reconstruct observes for one host. |
| +// |
| +// Cgroups is the exec backend's manager (activator.ExecBackend.Cgroups), |
| +// handed in by the host package when it arms the moves pass; the activator |
| +// cannot call Reconstruct itself because this package imports it. DBPaths |
| +// are the host's database files, absolute; each one's -wal and -shm are |
| +// checked with it. ProcRoot is "/proc" on a box ("" means that) and a |
| +// fixture in tests. LeaseFree reports whether the host's lock.json is |
| +// absent, released or expired past the renewal margin: the caller binds |
| +// lease.LockState to the location of a fresh claim read (ruling G3). |
| +type ReconstructInputs struct { |
| + Host string |
| + Cgroups *cgroup.Manager |
| + DBPaths []string |
| + ProcRoot string |
| + LeaseFree func(context.Context) (bool, error) |
| +} |
diff --git a/internal/stopevidence/reconstruct_linux.go b/internal/stopevidence/reconstruct_linux.go| new file mode 100644 |
| index 000000000..bce6e089c |
| --- /dev/null |
| +++ b/internal/stopevidence/reconstruct_linux.go |
| @@ -0,0 +1,323 @@ |
| +//go:build linux |
| + |
| +package stopevidence |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "fmt" |
| + "os" |
| + "path" |
| + "path/filepath" |
| + "slices" |
| + "sort" |
| + "strconv" |
| + "strings" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/cgroup" |
| + "github.com/carlosframework/platform/internal/controlname" |
| + "github.com/carlosframework/platform/internal/moves" |
| +) |
| + |
| +// treeDrainLimit bounds step 1's kill-and-drain when the caller's context |
| +// has no earlier deadline (KillAndEmpty refuses a context without one). It |
| +// matches the activator's strictDrainTimeout. |
| +const treeDrainLimit = 30 * time.Second |
| + |
| +// treeGroup is the host cgroup as step 1 uses it: *cgroup.Cleanup, or a |
| +// fake in tests. |
| +type treeGroup interface { |
| + KillAndEmpty(context.Context) error |
| + Close() error |
| +} |
| + |
| +// Reconstruct is §5.3's evidence from durable, observable state: it reads |
| +// nothing from the agent's memory, so a restarted agent gives the same |
| +// answer. It runs steps 1 to 4 and records each in the report's Steps; the |
| +// caller does step 5 (the final checkpoint and the digests, then |
| +// StepReplica), since only it knows the replica it closes. Role, Box, |
| +// Incarnation, ClaimGen and OpID are the caller's to fill in too. |
| +// |
| +// A step that observed a failure is a Check with OK false and a reason; |
| +// the pass retries until its stop evidence budget runs out. A step that |
| +// could not observe (an unreadable /proc or cgroup file, a lease read |
| +// error, the caller's context ending) is a returned error, and the report |
| +// beside it is partial and must not be written. |
| +// |
| +// Reconstruct takes neither per-host gate. schedule.HostGate is the timers |
| +// drain's gate (the runner, the converger and moveCfg.TimersDrained); the |
| +// activator's strictGates (ExecBackend.gate) serialise a strict spawn with |
| +// its own exit cleanup. By the time the cut calls this, the registry hold |
| +// and the activator's stop fence have closed admission, so nothing spawns |
| +// for the host; the kill works on the group whoever last owned it, and a |
| +// strict owner's concurrent cleanup only kills and drains the same group. |
| +func Reconstruct(ctx context.Context, in ReconstructInputs) (moves.StoppedReport, error) { |
| + if in.Cgroups == nil { |
| + return moves.StoppedReport{V: 1, Host: in.Host}, ErrInvalidInput |
| + } |
| + m := in.Cgroups |
| + return reconstruct(ctx, in, func(host string) (treeGroup, error) { |
| + c, err := m.OpenCleanup(host) |
| + if err != nil { |
| + return nil, err |
| + } |
| + return c, nil |
| + }) |
| +} |
| + |
| +func reconstruct(ctx context.Context, in ReconstructInputs, open func(string) (treeGroup, error)) (moves.StoppedReport, error) { |
| + rep := moves.StoppedReport{V: 1, Host: in.Host, Steps: map[string]moves.Check{}} |
| + if ctx == nil || open == nil || in.LeaseFree == nil || !controlname.ValidHostShape(in.Host) { |
| + return rep, ErrInvalidInput |
| + } |
| + procRoot := in.ProcRoot |
| + if procRoot == "" { |
| + procRoot = "/proc" |
| + } |
| + if !filepath.IsAbs(procRoot) { |
| + return rep, ErrInvalidInput |
| + } |
| + dbFiles, err := dbPathsWithSidecars(in.DBPaths) |
| + if err != nil { |
| + return rep, err |
| + } |
| + if err := ctx.Err(); err != nil { |
| + return rep, err |
| + } |
| + |
| + // 1. Kill and drain the tree: cgroup.kill on carlos-exec@<host>, then |
| + // wait for it to empty. Phase 1 moves flat exec instances only (S1 |
| + // refuses a nested one), so this is the whole of step 1. |
| + tree, err := stopTree(ctx, in.Host, open) |
| + if err != nil { |
| + return rep, err |
| + } |
| + rep.Steps[moves.StepTree] = tree |
| + |
| + // 2. No stray process in the agent's own leaf. A spawn that escaped its |
| + // host cgroup (best-effort placement before the move went strict) lands |
| + // here, which is why an absent host cgroup counts as an empty tree only |
| + // alongside this step. So does any short-lived child the agent forks |
| + // for itself: it fails StepStray for this pass, and the cut's retry |
| + // until its deadline is the intended behaviour. Only a repeat is worth |
| + // the alert (M-C20). |
| + pids, err := ownLeafProcs(procRoot) |
| + if err != nil { |
| + return rep, err |
| + } |
| + self := os.Getpid() |
| + if !slices.Contains(pids, self) { |
| + return rep, fmt.Errorf("stopevidence: the agent (pid %d) is not in the cgroup its /proc entry names", self) |
| + } |
| + rep.Steps[moves.StepStray] = strayCheck(pids, self) |
| + for _, p := range pids { |
| + if p != self { |
| + rep.Stray = append(rep.Stray, p) |
| + } |
| + } |
| + |
| + // 3. No open database. The scan covers every process /proc shows. On a |
| + // box the agent runs under ProtectProc=invisible (carlos-edge.service), |
| + // so /proc shows only the carlos uid's processes: §5.3's "restricted to |
| + // the carlos uid". A process it can see but not read makes the scan |
| + // incomplete, which is an error, never a pass. |
| + holders, err := scanFDs(ctx, procRoot, dbFiles) |
| + if err != nil { |
| + return rep, err |
| + } |
| + if len(holders) == 0 { |
| + rep.Steps[moves.StepFDs] = moves.Check{OK: true} |
| + } else { |
| + rep.Steps[moves.StepFDs] = moves.Check{Reason: fmt.Sprintf("%d process(es) hold the database open", len(holders))} |
| + } |
| + |
| + // 4. Lease released. |
| + free, err := in.LeaseFree(ctx) |
| + if err != nil { |
| + return rep, err |
| + } |
| + if free { |
| + rep.Steps[moves.StepLease] = moves.Check{OK: true} |
| + } else { |
| + rep.Steps[moves.StepLease] = moves.Check{Reason: "lock.json still held"} |
| + } |
| + |
| + // The evidence time is when the last step was observed, not when the |
| + // pass began: §5.3's late-completion horizon runs from it, and a later |
| + // time is the safe direction. |
| + rep.CheckedAt = time.Now().UTC() |
| + return rep, nil |
| +} |
| + |
| +// stopTree is step 1. A group that never existed, or that its last strict |
| +// owner removed, is an empty tree. A drain that runs out of time is an |
| +// observed failure; any other failure to open, kill or read the group is |
| +// an error. |
| +func stopTree(ctx context.Context, host string, open func(string) (treeGroup, error)) (moves.Check, error) { |
| + c, err := open(host) |
| + if errors.Is(err, cgroup.ErrNoCgroup) { |
| + return moves.Check{OK: true}, nil |
| + } |
| + if err != nil { |
| + return moves.Check{}, err |
| + } |
| + defer c.Close() |
| + drain, cancel := context.WithTimeout(ctx, treeDrainLimit) |
| + defer cancel() |
| + err = c.KillAndEmpty(drain) |
| + switch { |
| + case err == nil: |
| + return moves.Check{OK: true}, nil |
| + case ctx.Err() != nil: |
| + return moves.Check{}, ctx.Err() |
| + case errors.Is(err, context.DeadlineExceeded): |
| + return moves.Check{Reason: "the instance's processes could not be stopped"}, nil |
| + } |
| + return moves.Check{}, err |
| +} |
| + |
| +func strayCheck(pids []int, self int) moves.Check { |
| + for _, p := range pids { |
| + if p != self { |
| + return moves.Check{Reason: fmt.Sprintf("pid %d in the agent's cgroup is not the agent", p)} |
| + } |
| + } |
| + return moves.Check{OK: true} |
| +} |
| + |
| +// dbPathsWithSidecars is every database file with its -wal and -shm, clean, |
| +// sorted and without repeats. Each path must be absolute, and there must be |
| +// at least one: a cut always has a database path, so an empty list is a |
| +// caller's mistake, not "nothing to hold open". |
| +func dbPathsWithSidecars(paths []string) ([]string, error) { |
| + if len(paths) == 0 { |
| + return nil, ErrInvalidInput |
| + } |
| + seen := map[string]bool{} |
| + for _, p := range paths { |
| + if !filepath.IsAbs(p) { |
| + return nil, ErrInvalidInput |
| + } |
| + p = filepath.Clean(p) |
| + seen[p], seen[p+"-wal"], seen[p+"-shm"] = true, true, true |
| + } |
| + out := make([]string, 0, len(seen)) |
| + for p := range seen { |
| + out = append(out, p) |
| + } |
| + sort.Strings(out) |
| + return out, nil |
| +} |
| + |
| +// ownLeafProcs reads the pids in the agent's own cgroup leaf: the v2 path |
| +// <procRoot>/self/cgroup names, under the cgroup2 mount |
| +// <procRoot>/self/mountinfo names (ruling G32: never a hard-coded |
| +// /sys/fs/cgroup). |
| +func ownLeafProcs(procRoot string) ([]int, error) { |
| + self, err := os.ReadFile(filepath.Join(procRoot, "self", "cgroup")) |
| + if err != nil { |
| + return nil, err |
| + } |
| + leaf, ok := cgroup.ParseSelfCgroup(string(self)) |
| + if !ok { |
| + return nil, errors.New("stopevidence: own cgroup unreadable") |
| + } |
| + mounts, err := os.ReadFile(filepath.Join(procRoot, "self", "mountinfo")) |
| + if err != nil { |
| + return nil, err |
| + } |
| + dir, err := cgroupLeafDir(string(mounts), leaf) |
| + if err != nil { |
| + return nil, err |
| + } |
| + return readProcs(filepath.Join(dir, "cgroup.procs")) |
| +} |
| + |
| +// cgroupLeafDir is the directory of the v2 cgroup leaf (as |
| +// ParseSelfCgroup returns it, relative to the hierarchy's root) under the |
| +// cgroup2 mount that contains it. A mount's root field is the part of the |
| +// hierarchy it shows; the mount whose root holds the leaf most closely |
| +// wins, and of equal roots the first listed. A leaf that climbs out of |
| +// the hierarchy (a cgroup outside the agent's cgroup namespace shows as |
| +// "/../..") or that no cgroup2 mount shows is an error. |
| +func cgroupLeafDir(mountinfo, leaf string) (string, error) { |
| + abs := "/" + leaf |
| + if path.Clean(abs) != abs { |
| + return "", fmt.Errorf("stopevidence: own cgroup %q is outside the hierarchy", leaf) |
| + } |
| + best, bestRoot := "", -1 |
| + for _, line := range strings.Split(mountinfo, "\n") { |
| + fields := strings.Fields(line) |
| + if len(fields) < 7 { |
| + continue |
| + } |
| + sep := slices.Index(fields[6:], "-") |
| + if sep < 0 || 6+sep+1 >= len(fields) || fields[6+sep+1] != "cgroup2" { |
| + continue |
| + } |
| + root, mnt := unescapeMount(fields[3]), unescapeMount(fields[4]) |
| + if !filepath.IsAbs(mnt) { |
| + continue |
| + } |
| + var rel string |
| + switch { |
| + case root == "/": |
| + rel = abs |
| + case abs == root: |
| + rel = "/" |
| + case strings.HasPrefix(abs, root+"/"): |
| + rel = strings.TrimPrefix(abs, root) |
| + default: |
| + continue |
| + } |
| + if len(root) > bestRoot { |
| + best, bestRoot = filepath.Join(mnt, rel), len(root) |
| + } |
| + } |
| + if bestRoot < 0 { |
| + return "", fmt.Errorf("stopevidence: no cgroup2 mount shows own cgroup %q", leaf) |
| + } |
| + return best, nil |
| +} |
| + |
| +// unescapeMount undoes mountinfo's octal escapes (\040 for a space, \011, |
| +// \012, \134). |
| +func unescapeMount(s string) string { |
| + if !strings.Contains(s, `\`) { |
| + return s |
| + } |
| + var b strings.Builder |
| + for i := 0; i < len(s); i++ { |
| + if s[i] == '\\' && i+4 <= len(s) { |
| + if v, err := strconv.ParseUint(s[i+1:i+4], 8, 8); err == nil { |
| + b.WriteByte(byte(v)) |
| + i += 3 |
| + continue |
| + } |
| + } |
| + b.WriteByte(s[i]) |
| + } |
| + return b.String() |
| +} |
| + |
| +// readProcs parses cgroup.procs: one pid per line. |
| +func readProcs(file string) ([]int, error) { |
| + raw, err := os.ReadFile(file) |
| + if err != nil { |
| + return nil, err |
| + } |
| + var pids []int |
| + for _, line := range strings.Split(string(raw), "\n") { |
| + line = strings.TrimSpace(line) |
| + if line == "" { |
| + continue |
| + } |
| + pid, err := strconv.Atoi(line) |
| + if err != nil || pid <= 0 { |
| + return nil, fmt.Errorf("stopevidence: %s: invalid pid %q", file, line) |
| + } |
| + pids = append(pids, pid) |
| + } |
| + return pids, nil |
| +} |
diff --git a/internal/stopevidence/reconstruct_linux_test.go b/internal/stopevidence/reconstruct_linux_test.go| new file mode 100644 |
| index 000000000..d33058aca |
| --- /dev/null |
| +++ b/internal/stopevidence/reconstruct_linux_test.go |
| @@ -0,0 +1,469 @@ |
| +//go:build linux |
| + |
| +package stopevidence |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "os" |
| + "os/exec" |
| + "path/filepath" |
| + "reflect" |
| + "strconv" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/cgroup" |
| + "github.com/carlosframework/platform/internal/moves" |
| +) |
| + |
| +// §14.1: Reconstruct against a real cgroup v2 hierarchy with a forked |
| +// descendant that closes its descriptors and outlives its parent (witness |
| +// wEvidenceWholeTree). Needs a delegated cgroup root: CARLOS_CGROUP_TEST=1 |
| +// and CARLOS_CGROUP_ROOT=<writable cgroup2 dir>, for instance from inside |
| +// `systemd-run --user --scope -p Delegate=yes`. |
| +func TestReconstructStopEvidenceKillsTheWholeTree(t *testing.T) { |
| + root := os.Getenv("CARLOS_CGROUP_ROOT") |
| + if os.Getenv("CARLOS_CGROUP_TEST") != "1" || root == "" { |
| + t.Skip("needs CARLOS_CGROUP_TEST=1 and a delegated cgroup v2 root") |
| + } |
| + m, err := cgroup.SetupAt(root, t.Logf) |
| + if err != nil { |
| + t.Skip(err) |
| + } |
| + host := "ev.go.example.com" |
| + // A parent that forks a child which detaches and sleeps, then exits. |
| + cmd, err := m.Start(host, cgroup.Limit{}, func() *exec.Cmd { |
| + return exec.Command("sh", "-c", "(sleep 300 </dev/null >/dev/null 2>&1 &) ; exit 0") |
| + }) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + cmd.Wait() |
| + c, err := m.OpenCleanup(host) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + defer c.Close() |
| + if pop, err := c.Populated(); err != nil || !pop { |
| + t.Fatalf("the descendant should still be alive before Reconstruct: populated %v err %v", pop, err) |
| + } |
| + // The real /proc entry of this process, projected into an otherwise |
| + // empty scan root (as TestFDHoldersRealProc does): the whole-machine |
| + // scan needs every uid's /proc, which the agent has under |
| + // ProtectProc=invisible and a test usually does not. |
| + procRoot := projectedProc(t) |
| + db := filepath.Join(t.TempDir(), "ev.go.example.com.db") |
| + rep, err := Reconstruct(context.Background(), ReconstructInputs{Host: host, Cgroups: m, DBPaths: []string{db}, ProcRoot: procRoot, |
| + LeaseFree: func(context.Context) (bool, error) { return true, nil }}) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if !rep.Steps[moves.StepTree].OK || !rep.TreeStopped() || rep.Complete() { |
| + t.Fatalf("tree evidence after the parent exited: %+v (Complete needs the caller's replica step)", rep) |
| + } |
| + if pop, err := c.Populated(); err != nil || pop { |
| + t.Fatalf("the descendant survived: populated %v err %v", pop, err) |
| + } |
| +} |
| + |
| +// Step 2 against the running kernel: the leaf /proc/self/cgroup names is |
| +// found under the cgroup2 mount /proc/self/mountinfo names, and its |
| +// cgroup.procs holds this process. Whether StepStray passes depends on |
| +// what else shares the test's cgroup (alone in a fresh scope, it passes). |
| +func TestReconstructReadsTheRealOwnLeaf(t *testing.T) { |
| + if _, ok := cgroup.ParseSelfCgroup(readOrSkip(t, "/proc/self/cgroup")); !ok { |
| + t.Skip("no cgroup v2 leaf for this process") |
| + } |
| + db := filepath.Join(t.TempDir(), "h.example.com.db") |
| + rep, err := reconstruct(context.Background(), ReconstructInputs{Host: "h.example.com", DBPaths: []string{db}, ProcRoot: projectedProc(t), |
| + LeaseFree: leaseFree}, opener(&fakeTree{}, nil)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, ok := rep.Steps[moves.StepStray]; !ok { |
| + t.Fatal("no stray step") |
| + } |
| + for _, p := range rep.Stray { |
| + if p == os.Getpid() { |
| + t.Fatal("the agent counted itself as a stray") |
| + } |
| + } |
| + t.Logf("stray step %+v, %d others in the leaf", rep.Steps[moves.StepStray], len(rep.Stray)) |
| +} |
| + |
| +func readOrSkip(t *testing.T, path string) string { |
| + t.Helper() |
| + b, err := os.ReadFile(path) |
| + if err != nil { |
| + t.Skip(err) |
| + } |
| + return string(b) |
| +} |
| + |
| +func TestReconstructStopEvidenceRefusesAStrayProcess(t *testing.T) { |
| + // A fake self-cgroup listing with a second pid refuses with StepStray. |
| + rep := strayCheck([]int{os.Getpid(), 99999}, os.Getpid()) |
| + if rep.OK { |
| + t.Fatal("a stray pid passed") |
| + } |
| + if rep.Reason != "pid 99999 in the agent's cgroup is not the agent" { |
| + t.Fatalf("reason %q", rep.Reason) |
| + } |
| + if got := strayCheck([]int{os.Getpid()}, os.Getpid()); !got.OK { |
| + t.Fatalf("only the agent: %+v", got) |
| + } |
| +} |
| + |
| +// projectedProc is a scan root holding this process's real /proc entry, |
| +// under its pid and as self. |
| +func projectedProc(t *testing.T) string { |
| + t.Helper() |
| + root := t.TempDir() |
| + pid := strconv.Itoa(os.Getpid()) |
| + for _, name := range []string{pid, "self"} { |
| + if err := os.Symlink(filepath.Join("/proc", pid), filepath.Join(root, name)); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + return root |
| +} |
| + |
| +// boxFixture is a fake box: a proc root whose self/cgroup names the |
| +// agent's leaf, a self/mountinfo mounting a cgroup2 hierarchy at a temp |
| +// directory (never /sys/fs/cgroup), and that leaf's cgroup.procs. |
| +type boxFixture struct { |
| + proc, mount, leafDir, db string |
| +} |
| + |
| +func newBoxFixture(t *testing.T, leafPIDs ...int) *boxFixture { |
| + t.Helper() |
| + dir := t.TempDir() |
| + f := &boxFixture{proc: filepath.Join(dir, "proc"), mount: filepath.Join(dir, "cg fs"), db: filepath.Join(dir, "data", "h.example.com.db")} |
| + f.leafDir = filepath.Join(f.mount, "system.slice", "carlos-edge.service", "supervise") |
| + for _, d := range []string{filepath.Join(f.proc, "self"), f.leafDir, filepath.Dir(f.db)} { |
| + if err := os.MkdirAll(d, 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + write(t, filepath.Join(f.proc, "self", "cgroup"), "0::/system.slice/carlos-edge.service/supervise\n") |
| + write(t, filepath.Join(f.proc, "self", "mountinfo"), |
| + "22 1 0:21 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw\n"+ |
| + "32 24 0:27 / "+escapeMount(f.mount)+" rw,nosuid,nodev,noexec,relatime shared:8 - cgroup2 cgroup2 rw,nsdelegate\n") |
| + if leafPIDs == nil { |
| + leafPIDs = []int{os.Getpid()} |
| + } |
| + f.setLeaf(t, leafPIDs...) |
| + return f |
| +} |
| + |
| +func (f *boxFixture) setLeaf(t *testing.T, pids ...int) { |
| + t.Helper() |
| + var body string |
| + for _, p := range pids { |
| + body += strconv.Itoa(p) + "\n" |
| + } |
| + write(t, filepath.Join(f.leafDir, "cgroup.procs"), body) |
| +} |
| + |
| +// holdDB makes pid a process with the path open as descriptor 7. |
| +func (f *boxFixture) holdDB(t *testing.T, pid int, path string) { |
| + t.Helper() |
| + fd := filepath.Join(f.proc, strconv.Itoa(pid), "task", strconv.Itoa(pid), "fd") |
| + if err := os.MkdirAll(fd, 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.Symlink(path, filepath.Join(fd, "7")); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +func escapeMount(p string) string { |
| + out := "" |
| + for _, r := range p { |
| + if r == ' ' { |
| + out += `\040` |
| + continue |
| + } |
| + out += string(r) |
| + } |
| + return out |
| +} |
| + |
| +func write(t *testing.T, path, body string) { |
| + t.Helper() |
| + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +// fakeTree is the host cgroup as reconstruct sees it. |
| +type fakeTree struct { |
| + killErr error |
| + deadline bool |
| + killed bool |
| + closed bool |
| +} |
| + |
| +func (g *fakeTree) KillAndEmpty(ctx context.Context) error { |
| + g.killed = true |
| + _, g.deadline = ctx.Deadline() |
| + return g.killErr |
| +} |
| +func (g *fakeTree) Close() error { g.closed = true; return nil } |
| + |
| +func opener(g *fakeTree, err error) func(string) (treeGroup, error) { |
| + return func(string) (treeGroup, error) { |
| + if err != nil { |
| + return nil, err |
| + } |
| + return g, nil |
| + } |
| +} |
| + |
| +func leaseFree(context.Context) (bool, error) { return true, nil } |
| + |
| +func (f *boxFixture) inputs() ReconstructInputs { |
| + return ReconstructInputs{Host: "h.example.com", DBPaths: []string{f.db}, ProcRoot: f.proc, LeaseFree: leaseFree} |
| +} |
| + |
| +func TestReconstructAllStepsPass(t *testing.T) { |
| + f := newBoxFixture(t) |
| + g := &fakeTree{} |
| + rep, err := reconstruct(context.Background(), f.inputs(), opener(g, nil)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, step := range []string{moves.StepTree, moves.StepStray, moves.StepFDs, moves.StepLease} { |
| + if c := rep.Steps[step]; !c.OK || c.Reason != "" { |
| + t.Fatalf("%s: %+v", step, c) |
| + } |
| + } |
| + if !rep.TreeStopped() || rep.Complete() || len(rep.Stray) != 0 || rep.V != 1 || rep.Host != "h.example.com" || rep.CheckedAt.IsZero() { |
| + t.Fatalf("report %+v", rep) |
| + } |
| + if !g.killed || !g.closed { |
| + t.Fatalf("the tree was not killed and closed: %+v", g) |
| + } |
| + // KillAndEmpty refuses a context with no deadline, so Reconstruct |
| + // bounds the drain itself when its caller did not. |
| + if !g.deadline { |
| + t.Fatal("the drain ran without a deadline") |
| + } |
| +} |
| + |
| +func TestReconstructNoHostCgroupIsAnEmptyTree(t *testing.T) { |
| + f := newBoxFixture(t) |
| + rep, err := reconstruct(context.Background(), f.inputs(), opener(nil, cgroup.ErrNoCgroup)) |
| + if err != nil || !rep.Steps[moves.StepTree].OK || !rep.TreeStopped() { |
| + t.Fatalf("no cgroup: %+v %v", rep, err) |
| + } |
| +} |
| + |
| +func TestReconstructTreeThatWillNotEmpty(t *testing.T) { |
| + f := newBoxFixture(t) |
| + g := &fakeTree{killErr: context.DeadlineExceeded} |
| + rep, err := reconstruct(context.Background(), f.inputs(), opener(g, nil)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if c := rep.Steps[moves.StepTree]; c.OK || c.Reason != "the instance's processes could not be stopped" || rep.TreeStopped() { |
| + t.Fatalf("tree: %+v", rep) |
| + } |
| + if !g.closed { |
| + t.Fatal("the cleanup handle leaked") |
| + } |
| + // The later steps still ran, so the report says everything it saw. |
| + if !rep.Steps[moves.StepStray].OK || !rep.Steps[moves.StepFDs].OK || !rep.Steps[moves.StepLease].OK { |
| + t.Fatalf("later steps: %+v", rep.Steps) |
| + } |
| +} |
| + |
| +func TestReconstructOpenFailureIsAnError(t *testing.T) { |
| + f := newBoxFixture(t) |
| + boom := errors.New("boom") |
| + if _, err := reconstruct(context.Background(), f.inputs(), opener(nil, boom)); !errors.Is(err, boom) { |
| + t.Fatalf("err %v", err) |
| + } |
| + // A kill that fails for any reason but the drain running out of time |
| + // could not observe the tree: an error, not a recorded step. |
| + g := &fakeTree{killErr: boom} |
| + if _, err := reconstruct(context.Background(), f.inputs(), opener(g, nil)); !errors.Is(err, boom) || !g.closed { |
| + t.Fatalf("kill error: %v, closed %v", err, g.closed) |
| + } |
| +} |
| + |
| +func TestReconstructCallerCancelledIsAnError(t *testing.T) { |
| + f := newBoxFixture(t) |
| + ctx, cancel := context.WithCancel(context.Background()) |
| + g := &fakeTree{} |
| + g.killErr = context.Canceled |
| + open := func(string) (treeGroup, error) { cancel(); return g, nil } |
| + if _, err := reconstruct(ctx, f.inputs(), open); !errors.Is(err, context.Canceled) { |
| + t.Fatalf("err %v", err) |
| + } |
| +} |
| + |
| +func TestReconstructStrayInTheAgentsLeaf(t *testing.T) { |
| + f := newBoxFixture(t, os.Getpid(), 4243, 4244) |
| + rep, err := reconstruct(context.Background(), f.inputs(), opener(&fakeTree{}, nil)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if c := rep.Steps[moves.StepStray]; c.OK || c.Reason != "pid 4243 in the agent's cgroup is not the agent" || rep.TreeStopped() { |
| + t.Fatalf("stray: %+v", rep) |
| + } |
| + if !reflect.DeepEqual(rep.Stray, []int{4243, 4244}) { |
| + t.Fatalf("Stray %v", rep.Stray) |
| + } |
| +} |
| + |
| +// The leaf is read where /proc/self/mountinfo mounts the hierarchy, so a |
| +// leaf that names the agent but is not the agent's is an error, not a pass. |
| +func TestReconstructRefusesALeafWithoutTheAgent(t *testing.T) { |
| + f := newBoxFixture(t, 4243) |
| + if _, err := reconstruct(context.Background(), f.inputs(), opener(&fakeTree{}, nil)); err == nil { |
| + t.Fatal("a leaf without the agent passed") |
| + } |
| +} |
| + |
| +func TestReconstructOwnCgroupUnreadable(t *testing.T) { |
| + for name, mutate := range map[string]func(f *boxFixture){ |
| + "no v2 line": func(f *boxFixture) { |
| + os.WriteFile(filepath.Join(f.proc, "self", "cgroup"), []byte("1:name=systemd:/x\n"), 0o644) |
| + }, |
| + "root cgroup": func(f *boxFixture) { os.WriteFile(filepath.Join(f.proc, "self", "cgroup"), []byte("0::/\n"), 0o644) }, |
| + "no mount": func(f *boxFixture) { os.WriteFile(filepath.Join(f.proc, "self", "mountinfo"), []byte(""), 0o644) }, |
| + "no procs": func(f *boxFixture) { os.Remove(filepath.Join(f.leafDir, "cgroup.procs")) }, |
| + "garbled pid": func(f *boxFixture) { os.WriteFile(filepath.Join(f.leafDir, "cgroup.procs"), []byte("12x\n"), 0o644) }, |
| + "no mountinfo": func(f *boxFixture) { os.Remove(filepath.Join(f.proc, "self", "mountinfo")) }, |
| + } { |
| + f := newBoxFixture(t) |
| + mutate(f) |
| + if _, err := reconstruct(context.Background(), f.inputs(), opener(&fakeTree{}, nil)); err == nil { |
| + t.Fatalf("%s: passed", name) |
| + } |
| + } |
| +} |
| + |
| +func TestReconstructDatabaseHolders(t *testing.T) { |
| + for _, path := range []string{"", "-wal", "-shm"} { |
| + f := newBoxFixture(t) |
| + f.holdDB(t, 5150, f.db+path) |
| + rep, err := reconstruct(context.Background(), f.inputs(), opener(&fakeTree{}, nil)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if c := rep.Steps[moves.StepFDs]; c.OK || c.Reason != "1 process(es) hold the database open" || rep.TreeStopped() { |
| + t.Fatalf("holder of %q: %+v", path, rep.Steps) |
| + } |
| + } |
| + // A descriptor on some other file is not a holder. |
| + f := newBoxFixture(t) |
| + f.holdDB(t, 5150, filepath.Join(filepath.Dir(f.db), "other.db")) |
| + if rep, err := reconstruct(context.Background(), f.inputs(), opener(&fakeTree{}, nil)); err != nil || !rep.Steps[moves.StepFDs].OK { |
| + t.Fatalf("other file: %+v %v", rep.Steps, err) |
| + } |
| +} |
| + |
| +func TestReconstructLeaseStep(t *testing.T) { |
| + f := newBoxFixture(t) |
| + in := f.inputs() |
| + in.LeaseFree = func(context.Context) (bool, error) { return false, nil } |
| + rep, err := reconstruct(context.Background(), in, opener(&fakeTree{}, nil)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if c := rep.Steps[moves.StepLease]; c.OK || c.Reason != "lock.json still held" || rep.TreeStopped() { |
| + t.Fatalf("lease held: %+v", rep.Steps) |
| + } |
| + boom := errors.New("boom") |
| + in.LeaseFree = func(context.Context) (bool, error) { return true, boom } |
| + if _, err := reconstruct(context.Background(), in, opener(&fakeTree{}, nil)); !errors.Is(err, boom) { |
| + t.Fatalf("lease read error: %v", err) |
| + } |
| +} |
| + |
| +func TestReconstructInvalidInputs(t *testing.T) { |
| + f := newBoxFixture(t) |
| + for name, mutate := range map[string]func(*ReconstructInputs){ |
| + "bad host": func(in *ReconstructInputs) { in.Host = "../etc" }, |
| + "no lease reader": func(in *ReconstructInputs) { in.LeaseFree = nil }, |
| + "no database": func(in *ReconstructInputs) { in.DBPaths = nil }, |
| + "relative db": func(in *ReconstructInputs) { in.DBPaths = []string{"h.example.com.db"} }, |
| + "relative proc": func(in *ReconstructInputs) { in.ProcRoot = "proc" }, |
| + "empty db in list": func(in *ReconstructInputs) { in.DBPaths = []string{f.db, ""} }, |
| + } { |
| + in := f.inputs() |
| + mutate(&in) |
| + g := &fakeTree{} |
| + if _, err := reconstruct(context.Background(), in, opener(g, nil)); !errors.Is(err, ErrInvalidInput) { |
| + t.Fatalf("%s: err %v", name, err) |
| + } |
| + if g.killed { |
| + t.Fatalf("%s: killed the tree before refusing the input", name) |
| + } |
| + } |
| + in := f.inputs() |
| + if _, err := Reconstruct(context.Background(), in); !errors.Is(err, ErrInvalidInput) { |
| + t.Fatalf("nil manager: err %v", err) |
| + } |
| + if _, err := reconstruct(nil, f.inputs(), opener(&fakeTree{}, nil)); !errors.Is(err, ErrInvalidInput) { //nolint:staticcheck // a nil context is the input under test |
| + t.Fatalf("nil context: err %v", err) |
| + } |
| +} |
| + |
| +func TestCgroupLeafDir(t *testing.T) { |
| + line := func(root, mnt, fstype string) string { |
| + return "40 30 0:27 " + root + " " + mnt + " rw,relatime shared:8 - " + fstype + " " + fstype + " rw\n" |
| + } |
| + for _, c := range []struct { |
| + name, mountinfo, leaf, want string |
| + ok bool |
| + }{ |
| + {"root mount", line("/", "/sys/fs/cgroup", "cgroup2"), "a.service/supervise", "/sys/fs/cgroup/a.service/supervise", true}, |
| + {"mounted elsewhere", line("/", "/run/cg", "cgroup2"), "a.service/supervise", "/run/cg/a.service/supervise", true}, |
| + {"escaped space", line("/", `/run/c\040g`, "cgroup2"), "a/b", "/run/c g/a/b", true}, |
| + {"subtree bind", line("/a.service", "/mnt/a", "cgroup2"), "a.service/supervise", "/mnt/a/supervise", true}, |
| + {"subtree is the leaf", line("/a.service/supervise", "/mnt/s", "cgroup2"), "a.service/supervise", "/mnt/s", true}, |
| + {"longest root wins", line("/", "/sys/fs/cgroup", "cgroup2") + line("/a.service", "/mnt/a", "cgroup2"), "a.service/x", "/mnt/a/x", true}, |
| + {"first of equal roots", line("/", "/sys/fs/cgroup", "cgroup2") + line("/", "/other", "cgroup2"), "a/b", "/sys/fs/cgroup/a/b", true}, |
| + {"sibling subtree only", line("/a.serviceX", "/mnt/x", "cgroup2"), "a.service/supervise", "", false}, |
| + {"v1 only", line("/", "/sys/fs/cgroup/memory", "cgroup"), "a/b", "", false}, |
| + {"no mounts", "", "a/b", "", false}, |
| + {"outside the namespace", line("/", "/sys/fs/cgroup", "cgroup2"), "../../x", "", false}, |
| + {"garbled line", "40 30 0:27 / /sys/fs/cgroup\n", "a/b", "", false}, |
| + } { |
| + got, err := cgroupLeafDir(c.mountinfo, c.leaf) |
| + if c.ok != (err == nil) || got != c.want { |
| + t.Fatalf("%s: %q %v, want %q ok=%v", c.name, got, err, c.want, c.ok) |
| + } |
| + } |
| +} |
| + |
| +// The drain bound never extends the caller's own deadline. |
| +func TestReconstructKeepsTheCallersDeadline(t *testing.T) { |
| + f := newBoxFixture(t) |
| + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) |
| + defer cancel() |
| + want, _ := ctx.Deadline() |
| + var got time.Time |
| + open := func(string) (treeGroup, error) { |
| + return deadlineTree(func(d time.Time) { got = d }), nil |
| + } |
| + if _, err := reconstruct(ctx, f.inputs(), open); err != nil { |
| + t.Fatal(err) |
| + } |
| + if got.After(want) || got.IsZero() { |
| + t.Fatalf("drain deadline %v, caller's %v", got, want) |
| + } |
| +} |
| + |
| +type deadlineTree func(time.Time) |
| + |
| +func (d deadlineTree) KillAndEmpty(ctx context.Context) error { |
| + dl, _ := ctx.Deadline() |
| + d(dl) |
| + return nil |
| +} |
| +func (deadlineTree) Close() error { return nil } |
diff --git a/internal/stopevidence/reconstruct_other.go b/internal/stopevidence/reconstruct_other.go| new file mode 100644 |
| index 000000000..9b71b5e41 |
| --- /dev/null |
| +++ b/internal/stopevidence/reconstruct_other.go |
| @@ -0,0 +1,15 @@ |
| +//go:build !linux |
| + |
| +package stopevidence |
| + |
| +import ( |
| + "context" |
| + |
| + "github.com/carlosframework/platform/internal/moves" |
| +) |
| + |
| +// Reconstruct needs cgroup v2 and /proc: on any other platform it observes |
| +// nothing and returns ErrUnsupported, so no move can cut from such a box. |
| +func Reconstruct(context.Context, ReconstructInputs) (moves.StoppedReport, error) { |
| + return moves.StoppedReport{}, ErrUnsupported |
| +} |