amadan

carlos / platform Public

park: the claim's park engine (claim Plan 4)

internal/park: exact-version remote parking with create-only manifests
and resumable copy, delete and verify steps (never a bare-key delete);
the late-completion horizon from the persisted original evidence and the
quiet_after acknowledgement; the at-rest sweep (never the live set,
walking unfinished runs across regions, holderless sweeps on free hosts,
covering the predecessors it parks); journaled descriptor-relative local
retirement under the host guard. hostclaim gains the persisted park scope
(instances and staging locations) and the holderless sweep; born-covered
counts only a park of the holder's own location. Claim spec rev 6.23.
Libraries only.

Co-Authored-By: Codex <noreply@openai.com>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Squashed from feat/hostclaim-park into main.
Co-Authored-By: claude <claude~paul@keymail.dev>
codex pushed by claude 7f403a77aa3f8bd615b79bf28596197920b8077e parent e112199
16 files changed, +4018 −13
  • internal/hostclaim/claim.go +53 −10
  • internal/hostclaim/generation.go +5 −1
  • internal/hostclaim/park_test.go +119 −0
  • internal/hostclaim/rev713_test.go +9 −2
  • internal/hostguard/retirement.go +95 −0
  • internal/hostguard/retirement_test.go +127 −0
  • internal/park/local.go +270 −0
  • internal/park/local_linux.go +348 −0
  • internal/park/local_linux_test.go +673 −0
  • internal/park/local_other.go +9 −0
  • internal/park/moves_test.go +204 −0
  • internal/park/remote.go +574 −0
  • internal/park/remote_test.go +574 −0
  • internal/park/sweep.go +319 −0
  • internal/park/sweep_coverage_test.go +249 −0
  • internal/park/sweep_test.go +390 −0

CI passed — run details

diff --git a/internal/hostclaim/claim.go b/internal/hostclaim/claim.go
index 025f1d58b..43ed0bb84 100644
--- a/internal/hostclaim/claim.go
+++ b/internal/hostclaim/claim.go
@@ -11,6 +11,7 @@ import (
"encoding/hex"
"errors"
"fmt"
+ "strings"
"time"
"github.com/carlosframework/platform/internal/controlname"
@@ -106,15 +107,37 @@ type InventoryEntry struct {
Paths map[string]uint64 `json:"paths"` // path -> stamp
}
+type ParkScope struct {
+ Location string `json:"location"`
+ Incarnation string `json:"incarnation"`
+ EvidenceAt time.Time `json:"evidence_at"`
+ RequestLimit time.Duration `json:"request_limit_ns"`
+}
+
+func (p *ParkScope) Validate(host string) error {
+ if p == nil || !isHex64(p.Incarnation) || p.Incarnation != strings.ToLower(p.Incarnation) || p.EvidenceAt.IsZero() || p.RequestLimit <= 0 || p.RequestLimit > time.Duration(1<<63-1)-15*time.Minute {
+ return fmt.Errorf("%w: park scope", ErrMalformed)
+ }
+ base, staging := "instances/"+host, "moves/"+host
+ if p.Location == base || p.Location == base+"/"+p.Incarnation || p.Location == staging {
+ return nil
+ }
+ if op, ok := strings.CutPrefix(p.Location, staging+"/"); ok && op != "" && op != "." && op != ".." && !strings.Contains(op, "/") {
+ return nil
+ }
+ return fmt.Errorf("%w: park location", ErrMalformed)
+}
+
type Obligation struct {
- ID string `json:"id"`
- Op string `json:"op"`
- Kind string `json:"kind"` // "park-replica" | "park-local" | "park-generation"
- Region string `json:"region,omitempty"`
- Generation string `json:"generation,omitempty"`
- Box string `json:"box,omitempty"`
- Retirement string `json:"retirement"`
- Owner string `json:"owner"` // "region:<r>" | "box:<b>"
+ Park *ParkScope `json:"park,omitempty"`
+ ID string `json:"id"`
+ Op string `json:"op"`
+ Kind string `json:"kind"` // "park-replica" | "park-local" | "park-generation"
+ Region string `json:"region,omitempty"`
+ Generation string `json:"generation,omitempty"`
+ Box string `json:"box,omitempty"`
+ Retirement string `json:"retirement"`
+ Owner string `json:"owner"` // "region:<r>" | "box:<b>"
}
type CancelledCreate struct {
@@ -200,10 +223,10 @@ func (c *Claim) Validate() error {
return fmt.Errorf("%w: op %q without id", ErrMalformed, c.Op.Kind)
}
if c.Holder == nil {
- if c.Op.Kind != "" && c.Op.Kind != KindPurging { // a purge fence has no holder (CC21)
+ if c.Op.Kind != "" && c.Op.Kind != KindPurging && !holderlessSweep(c) { // a purge fence has no holder (CC21)
return fmt.Errorf("%w: op %q without holder", ErrMalformed, c.Op.Kind)
}
- if len(c.Obligations) > 0 {
+ if len(c.Obligations) > 0 && !holderlessSweep(c) {
return fmt.Errorf("%w: a free claim with outstanding obligations", ErrMalformed)
}
} else {
@@ -239,6 +262,14 @@ func (c *Claim) Validate() error {
if o.ID == "" || o.Op == "" || o.Retirement == "" || seen[o.ID] {
return fmt.Errorf("%w: obligation ids", ErrMalformed)
}
+ if o.Park != nil {
+ if o.Kind != "park-replica" && o.Kind != "park-generation" {
+ return fmt.Errorf("%w: park scope", ErrMalformed)
+ }
+ if err := o.Park.Validate(c.Host); err != nil {
+ return err
+ }
+ }
seen[o.ID] = true
switch o.Kind {
case "park-generation":
@@ -259,3 +290,15 @@ func (c *Claim) Validate() error {
}
return nil
}
+
+func holderlessSweep(c *Claim) bool {
+ if c.Holder != nil || c.Op.Kind != KindParking || c.Op.ID == "" || c.Op.Intent["park_sweep"] != "v1" || c.Op.Exec != "" || len(c.Obligations) == 0 {
+ return false
+ }
+ for _, o := range c.Obligations {
+ if o.Op != c.Op.ID || o.Park == nil || (o.Kind != "park-replica" && o.Kind != "park-generation") {
+ return false
+ }
+ }
+ return true
+}
diff --git a/internal/hostclaim/generation.go b/internal/hostclaim/generation.go
index 5491b62f0..01710b0c8 100644
--- a/internal/hostclaim/generation.go
+++ b/internal/hostclaim/generation.go
@@ -94,7 +94,11 @@ func AdmitWriter(in Admission) func(*Claim) error {
AdditionalWritesLost: append([]WritesLost(nil), in.AdditionalWritesLost...),
}
for _, ob := range c.Obligations {
- if ob.Kind == "park-replica" && ob.Region == entry.PredecessorRegion {
+ // Only a park of the holder's whole location retires the
+ // predecessor; a scoped park of another location (an old
+ // incarnation's residue, a staging prefix) does not.
+ if ob.Kind == "park-replica" && ob.Region == entry.PredecessorRegion &&
+ (ob.Park == nil || ob.Park.Location == h.Location) {
entry.Covered = true
break
}
diff --git a/internal/hostclaim/park_test.go b/internal/hostclaim/park_test.go
new file mode 100644
index 000000000..3ca5fc2b9
--- /dev/null
+++ b/internal/hostclaim/park_test.go
@@ -0,0 +1,119 @@
+package hostclaim
+
+import (
+ "errors"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+func TestParkScopeAndHolderlessSweep(t *testing.T) {
+ c := &Claim{V: 2, Host: "shop.bab.example.com", Gen: 1, Op: Op{Kind: KindParking, ID: "sweep", Intent: map[string]string{"park_sweep": "v1"}}}
+ c.Obligations = []Obligation{{ID: "ob", Op: "sweep", Kind: "park-replica", Region: "ie", Retirement: "rt", Owner: "region:ie", Park: &ParkScope{Location: "instances/" + c.Host + "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Incarnation: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", EvidenceAt: time.Now(), RequestLimit: time.Second}}}
+ if err := c.Validate(); err != nil {
+ t.Fatal(err)
+ }
+ for _, change := range []func(*Claim){
+ func(c *Claim) { c.Obligations[0].Park = nil },
+ func(c *Claim) { c.Op.Exec = "box" },
+ func(c *Claim) { c.Obligations[0].Op = "another" },
+ func(c *Claim) { c.Op.Intent = nil },
+ func(c *Claim) { c.Obligations[0].Park.RequestLimit = 0 },
+ func(c *Claim) { c.Obligations[0].Park.EvidenceAt = time.Time{} },
+ func(c *Claim) { c.Obligations[0].Park.Location = "instances/other.example" },
+ } {
+ copy := *c
+ copy.Obligations = append([]Obligation(nil), c.Obligations...)
+ scope := *c.Obligations[0].Park
+ copy.Obligations[0].Park = &scope
+ change(&copy)
+ if err := copy.Validate(); err == nil {
+ t.Fatal("invalid sweep admitted")
+ }
+ }
+}
+
+func TestParkScopeLocationGrammar(t *testing.T) {
+ host := "shop.bab.example.com"
+ inc := strings.Repeat("a", 64)
+ instances, moves := "instances/"+host, "moves/"+host
+ for _, tc := range []struct {
+ location string
+ valid bool
+ }{
+ {instances, true},
+ {instances + "/" + inc, true},
+ {moves, true},
+ {moves + "/move-1", true},
+ {moves + "/" + strings.Repeat("b", 64), true},
+ {instances + "/", false},
+ {instances + "/" + inc + "/", false},
+ {instances + "/" + inc + "/x", false},
+ {instances + "/" + strings.Repeat("b", 64), false},
+ {moves + "/", false},
+ {moves + "/move-1/", false},
+ {moves + "/move-1/x", false},
+ {moves + "/.", false},
+ {moves + "/..", false},
+ {moves + "//move-1", false},
+ {"moves/other.example/move-1", false},
+ {"instances/other.example", false},
+ {"other/" + host, false},
+ } {
+ t.Run(tc.location, func(t *testing.T) {
+ c := &Claim{V: 2, Host: host, Gen: 1, Op: Op{Kind: KindParking, ID: "sweep", Intent: map[string]string{"park_sweep": "v1"}}}
+ c.Obligations = []Obligation{{ID: "ob", Op: "sweep", Kind: "park-replica", Region: "ie", Retirement: "rt", Owner: "region:ie", Park: &ParkScope{Location: tc.location, Incarnation: inc, EvidenceAt: time.Now(), RequestLimit: time.Second}}}
+ err := c.Validate()
+ if tc.valid && err != nil {
+ t.Fatalf("valid scope rejected: %v", err)
+ }
+ if !tc.valid && !errors.Is(err, ErrMalformed) {
+ t.Fatalf("invalid scope: %v", err)
+ }
+ })
+ }
+ for _, location := range []string{instances, instances + "/" + strings.ToUpper(inc), moves, moves + "/move-1"} {
+ t.Run("uppercase-archive/"+location, func(t *testing.T) {
+ c := &Claim{V: 2, Host: host, Gen: 1, Op: Op{Kind: KindParking, ID: "sweep", Intent: map[string]string{"park_sweep": "v1"}}}
+ c.Obligations = []Obligation{{ID: "ob", Op: "sweep", Kind: "park-replica", Region: "ie", Retirement: "rt", Owner: "region:ie", Park: &ParkScope{Location: location, Incarnation: strings.ToUpper(inc), EvidenceAt: time.Now(), RequestLimit: time.Second}}}
+ if err := c.Validate(); !errors.Is(err, ErrMalformed) {
+ t.Fatalf("uppercase archive incarnation accepted: %v", err)
+ }
+ })
+ }
+}
+
+func TestHolderlessSweepJoinsRegion(t *testing.T) {
+ host := "shop.bab.example.com"
+ st := release.NewDirStore(t.TempDir())
+ chore := Obligation{ID: "ie-ob", Op: "sweep", Kind: "park-replica", Region: "ie", Retirement: "ie-rt", Owner: "region:ie", Park: &ParkScope{Location: "moves/" + host, Incarnation: strings.Repeat("a", 64), EvidenceAt: time.Now(), RequestLimit: time.Second}}
+ obs, err := Update(t.Context(), st, Observed{Host: host, Absent: true}, func(c *Claim) error {
+ c.Op = Op{Kind: KindParking, ID: "sweep", Intent: map[string]string{"park_sweep": "v1"}}
+ c.Obligations = []Obligation{chore}
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ joined := chore
+ joined.ID, joined.Region, joined.Retirement, joined.Owner = "uk-ob", "uk", "uk-rt", "region:uk"
+ obs, err = Update(t.Context(), st, obs, func(c *Claim) error {
+ c.Obligations = append(c.Obligations, joined)
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if obs.Claim.Free() || obs.Claim.Holder != nil || obs.Claim.Op.ID != "sweep" || len(obs.Claim.Obligations) != 2 {
+ t.Fatalf("invalid joined sweep: %+v", obs.Claim)
+ }
+ _, err = Update(t.Context(), st, obs, func(c *Claim) error {
+ c.Obligations[1].Op = "other-sweep"
+ return nil
+ })
+ if !errors.Is(err, ErrMalformed) {
+ t.Fatalf("joined region with a different op accepted: %v", err)
+ }
+}
diff --git a/internal/hostclaim/rev713_test.go b/internal/hostclaim/rev713_test.go
index b14aa245f..930764dc0 100644
--- a/internal/hostclaim/rev713_test.go
+++ b/internal/hostclaim/rev713_test.go
@@ -7,6 +7,7 @@ import (
"fmt"
"reflect"
"testing"
+ "time"
"github.com/carlosframework/platform/internal/release"
)
@@ -54,7 +55,7 @@ func TestAdmitWriterReturnRegion(t *testing.T) {
}
func TestAdmitWriterPendingParkCoverage(t *testing.T) {
- for _, name := range []string{"whole predecessor region", "no park", "other region", "generation only", "local only"} {
+ for _, name := range []string{"whole predecessor region", "scoped holder location", "no park", "other region", "generation only", "local only", "scoped other location"} {
t.Run(name, func(t *testing.T) {
ctx := context.Background()
s := release.NewDirStore(t.TempDir())
@@ -72,6 +73,12 @@ func TestAdmitWriterPendingParkCoverage(t *testing.T) {
ob.Kind, ob.Generation = "park-generation", "GT"
case "local only":
ob.Kind, ob.Region, ob.Owner, ob.Box = "park-local", "", "box:boxT", "boxT"
+ case "scoped holder location", "scoped other location":
+ loc := c.Holder.Location
+ if name == "scoped other location" {
+ loc = "moves/" + c.Host + "/oldmove"
+ }
+ ob.Park = &ParkScope{Location: loc, Incarnation: c.Holder.Incarnation, EvidenceAt: time.Unix(1, 0).UTC(), RequestLimit: time.Second}
}
c.Obligations = []Obligation{ob}
return nil
@@ -82,7 +89,7 @@ func TestAdmitWriterPendingParkCoverage(t *testing.T) {
Box: "boxS", Mode: "cold", Region: "ie", NewGeneration: "GS2",
}))
last := o.Claim.Holder.Admissions[1]
- if last.Covered != (name == "whole predecessor region") || last.PredecessorRegion != "uk" || last.WritesLost == nil {
+ if last.Covered != (name == "whole predecessor region" || name == "scoped holder location") || last.PredecessorRegion != "uk" || last.WritesLost == nil {
t.Fatalf("return admission = %+v", last)
}
if !reflect.DeepEqual(before.Claim.Obligations, o.Claim.Obligations) {
diff --git a/internal/hostguard/retirement.go b/internal/hostguard/retirement.go
new file mode 100644
index 000000000..2c5403ad6
--- /dev/null
+++ b/internal/hostguard/retirement.go
@@ -0,0 +1,95 @@
+package hostguard
+
+import (
+ "context"
+ "errors"
+ "github.com/carlosframework/platform/internal/hostclaim"
+)
+
+func (g Guard) RetireInstallationWith(ctx context.Context, host string, captured Captured, check func(*hostclaim.Claim) error, fn func() error) error {
+ return g.retireInstallationWith(ctx, host, captured, check, fn, nil)
+}
+
+// persist is injected by package tests to interrupt the record's durable write.
+func (g Guard) retireInstallationWith(ctx context.Context, host string, captured Captured, check func(*hostclaim.Claim) error, fn func() error, persist func(*recordDir, record) error) error {
+ if captured.Stamp == 0 || check == nil || fn == nil {
+ return ErrInvalidInput
+ }
+ return g.locked(ctx, host, func(d *recordDir) error {
+ c, err := g.fresh(ctx, host)
+ if err != nil {
+ return err
+ }
+ if c.Holder.Incarnation != captured.Incarnation {
+ return ErrClaimIncarnation
+ }
+ if err := check(c); err != nil {
+ return err
+ }
+ r, err := d.read(host)
+ if err != nil {
+ return err
+ }
+ if r.Incarnation != captured.Incarnation {
+ return ErrRecordIncarnation
+ }
+ if r.ActiveStamp != 0 {
+ if r.ActiveStamp != captured.Stamp {
+ return ErrStampMismatch
+ }
+ r.retire()
+ if persist == nil {
+ persist = (*recordDir).write
+ }
+ } else {
+ n := len(r.RetiredStamps)
+ if n == 0 || r.RetiredStamps[n-1] != captured.Stamp {
+ return ErrStampMismatch
+ }
+ if persist == nil {
+ persist = func(d *recordDir, _ record) error { return d.sync(host) }
+ }
+ }
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ // An inactive record can be visible after rename without being durable.
+ // Sync it on retries before the callback may move any artifact.
+ if err := persist(d, r); err != nil {
+ return err
+ }
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ return fn()
+ })
+}
+
+func (g Guard) WithNoInstallation(ctx context.Context, host, incarnation string, check func(*hostclaim.Claim) error, fn func() error) error {
+ if incarnation == "" || check == nil || fn == nil {
+ return ErrInvalidInput
+ }
+ return g.locked(ctx, host, func(d *recordDir) error {
+ c, err := g.fresh(ctx, host)
+ if err != nil {
+ return err
+ }
+ if c.Holder.Incarnation != incarnation {
+ return ErrClaimIncarnation
+ }
+ if err := check(c); err != nil {
+ return err
+ }
+ _, err = d.read(host)
+ if !errors.Is(err, ErrNoRecord) {
+ if err != nil {
+ return err
+ }
+ return ErrStampMismatch
+ }
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ return fn()
+ })
+}
diff --git a/internal/hostguard/retirement_test.go b/internal/hostguard/retirement_test.go
new file mode 100644
index 000000000..34388221a
--- /dev/null
+++ b/internal/hostguard/retirement_test.go
@@ -0,0 +1,127 @@
+//go:build unix
+
+package hostguard
+
+import (
+ "errors"
+ "os"
+ "path/filepath"
+ "reflect"
+ "testing"
+
+ "golang.org/x/sys/unix"
+)
+
+func TestRetireInstallationWithSyncsInactiveRecord(t *testing.T) {
+ f := newGuardFixture(t)
+ host := f.obs.Host
+ if err := f.g.Activate(t.Context(), host, "uk", nil); err != nil {
+ t.Fatal(err)
+ }
+ captured, err := Capture(f.g.DataDir, host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ failure := errors.New("injected retirement sync failure")
+ failKind := "directory"
+ var calls []string
+ f.g.openRecords = func(path string) (*recordDir, error) {
+ d, err := openRecordDir(path)
+ if err != nil {
+ return nil, err
+ }
+ d.fsync = func(fd int) error {
+ kind := "file"
+ if fd == d.fd {
+ kind = "directory"
+ }
+ calls = append(calls, kind)
+ if kind == failKind {
+ return failure
+ }
+ return unix.Fsync(fd)
+ }
+ return d, nil
+ }
+ callbacks := 0
+ retire := func() error {
+ return f.g.RetireInstallationWith(t.Context(), host, captured, allow, func() error {
+ callbacks++
+ return nil
+ })
+ }
+ if err := retire(); !errors.Is(err, failure) || callbacks != 0 {
+ t.Fatalf("initial failure: %v, callbacks=%d", err, callbacks)
+ }
+ path := filepath.Join(f.g.DataDir, "hostguard", host+".json")
+ // Hold the inode so replacement writes cannot reuse its number.
+ original, err := os.Open(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer original.Close()
+ defer unchangedFile(t, path)()
+ for _, kind := range []string{"file", "directory", ""} {
+ failKind, calls = kind, nil
+ err := retire()
+ if kind == "" {
+ if err != nil || callbacks != 1 {
+ t.Fatalf("durable retry: %v, callbacks=%d", err, callbacks)
+ }
+ } else if !errors.Is(err, failure) || callbacks != 0 {
+ t.Fatalf("%s failure: %v, callbacks=%d", kind, err, callbacks)
+ }
+ want := []string{"file", "directory"}
+ if kind == "file" {
+ want = []string{"file"}
+ }
+ if !reflect.DeepEqual(calls, want) {
+ t.Fatalf("syncs=%v, want=%v", calls, want)
+ }
+ }
+}
+
+func TestRetireInstallationWithRepairsDurability(t *testing.T) {
+ f := newGuardFixture(t)
+ if err := f.g.Activate(t.Context(), f.obs.Host, "uk", nil); err != nil {
+ t.Fatal(err)
+ }
+ captured, err := Capture(f.g.DataDir, f.obs.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ crash := errors.New("crash after record rename before directory fsync")
+ callbacks, writes := 0, 0
+ callback := func() error { callbacks++; return nil }
+ persist := func(d *recordDir, r record) error {
+ writes++
+ if r.ActiveStamp != 0 || len(r.RetiredStamps) != 1 || r.RetiredStamps[0] != captured.Stamp {
+ t.Fatal("incorrect inactive record", r)
+ }
+ return d.writeUsing(r, ".retire-crash", func(a int, old string, b int, new string) error {
+ if err := unix.Renameat(a, old, b, new); err != nil {
+ return err
+ }
+ return crash
+ })
+ }
+ for attempt := 0; attempt < 2; attempt++ {
+ if err := f.g.retireInstallationWith(t.Context(), f.obs.Host, captured, allow, callback, persist); !errors.Is(err, crash) {
+ t.Fatalf("attempt %d: %v", attempt, err)
+ }
+ if callbacks != 0 || writes != attempt+1 {
+ t.Fatal("callback ran without durable record", callbacks, writes)
+ }
+ visible, err := Capture(f.g.DataDir, f.obs.Host)
+ if err != nil || visible.Stamp != 0 {
+ t.Fatal("did not reach rename boundary", visible, err)
+ }
+ }
+ // Exercise the public entry point with its real writer after both failures.
+ if err := f.g.RetireInstallationWith(t.Context(), f.obs.Host, captured, allow, callback); err != nil {
+ t.Fatal(err)
+ }
+ if callbacks != 1 {
+ t.Fatal("durable retry did not invoke callback")
+ }
+}
diff --git a/internal/park/local.go b/internal/park/local.go
new file mode 100644
index 000000000..c5945eeae
--- /dev/null
+++ b/internal/park/local.go
@@ -0,0 +1,270 @@
+package park
+
+import (
+ "context"
+ "errors"
+ "path"
+ "sort"
+ "strings"
+
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/hostguard"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+var (
+ ErrLocalIdentity = errors.New("park: local retirement identity mismatch")
+ ErrLocalInventory = errors.New("park: missing or unsafe inventory")
+ ErrLocalJournal = errors.New("park: invalid or conflicting local journal")
+ ErrLocalUnsupported = errors.New("park: local retirement unsupported")
+ errLocalComplete = errors.New("park: local obligation already completed")
+)
+
+type Local struct {
+ Authorize func(context.Context, *hostclaim.Claim, hostclaim.Obligation) error
+ Guard hostguard.Guard
+ Claims release.Store
+ DataDir, Box string
+ afterRename func(string) error
+}
+
+type localItem struct {
+ Path string
+ Stamp uint64
+ Dev, Ino uint64
+ Type uint32
+ Missing, Done bool
+}
+
+type localJournal struct {
+ V int
+ Host, Incarnation, Op, Obligation, Retirement, Mode string
+ Installation, ArtifactStamp uint64
+ Items []localItem
+}
+
+func localComponent(s string) bool {
+ return s != "" && s != "." && s != ".." && !strings.ContainsAny(s, "/\\\x00")
+}
+
+func localPath(s string) bool {
+ if s == "" || path.IsAbs(s) || path.Clean(s) != s || strings.ContainsAny(s, "\\\x00") {
+ return false
+ }
+ for _, p := range strings.Split(s, "/") {
+ if !localComponent(p) {
+ return false
+ }
+ }
+ return s != "hostguard" && !strings.HasPrefix(s, "hostguard/") && s != ".parked" && !strings.HasPrefix(s, ".parked/")
+}
+
+func (l Local) chore(c *hostclaim.Claim, captured hostguard.Captured, id, op string) (hostclaim.Obligation, error) {
+ if c == nil {
+ return hostclaim.Obligation{}, ErrLocalIdentity
+ }
+ var found *hostclaim.Obligation
+ for i := range c.Obligations {
+ if c.Obligations[i].ID == id && c.Obligations[i].Op == op {
+ found = &c.Obligations[i]
+ break
+ }
+ }
+ if found == nil {
+ return hostclaim.Obligation{}, errLocalComplete
+ }
+ if c.Holder == nil || c.Holder.Incarnation != captured.Incarnation || found.Kind != "park-local" || found.Box != l.Box || found.Owner != "box:"+l.Box || !localComponent(found.Retirement) {
+ return hostclaim.Obligation{}, ErrLocalIdentity
+ }
+ if _, ok := c.Inventory[l.Box]; !ok {
+ return hostclaim.Obligation{}, ErrLocalInventory
+ }
+ return *found, nil
+}
+
+func (l Local) RetireArtifacts(ctx context.Context, host string, captured hostguard.Captured, id, op string, stamp uint64) error {
+ if stamp == 0 {
+ return ErrLocalIdentity
+ }
+ return l.retire(ctx, host, captured, id, op, "artifacts", stamp)
+}
+
+func (l Local) RetireInstallation(ctx context.Context, host string, captured hostguard.Captured, id, op string) error {
+ return l.retire(ctx, host, captured, id, op, "installation", 0)
+}
+
+func (l Local) retire(ctx context.Context, host string, captured hostguard.Captured, id, op, mode string, stamp uint64) error {
+ if !localSupported() {
+ return ErrLocalUnsupported
+ }
+ if ctx == nil || l.Authorize == nil || l.Claims == nil || l.Box == "" || l.DataDir == "" || l.Guard.DataDir != l.DataDir || (captured.Stamp == 0 && mode != "empty") || id == "" || op == "" {
+ return ErrLocalIdentity
+ }
+ obs, err := hostclaim.Read(ctx, l.Claims, host)
+ if err != nil {
+ return err
+ }
+ if obs.Absent {
+ return ErrLocalIdentity
+ }
+ if _, err := l.chore(obs.Claim, captured, id, op); err != nil {
+ if errors.Is(err, errLocalComplete) {
+ return nil
+ }
+ return err
+ }
+ var j localJournal
+ var inventory map[string]uint64
+ check := func(c *hostclaim.Claim) error {
+ o, err := l.chore(c, captured, id, op)
+ if err != nil {
+ return err
+ }
+ if err := l.Authorize(ctx, c, o); err != nil {
+ return err
+ }
+ inventory = c.Inventory[l.Box].Paths
+ if mode == "empty" && len(inventory) != 0 {
+ return ErrLocalInventory
+ }
+ paths := make([]string, 0, len(inventory))
+ for p, s := range inventory {
+ if !localPath(p) || s == 0 {
+ return ErrLocalInventory
+ }
+ paths = append(paths, p)
+ }
+ sort.Strings(paths)
+ for i := range paths {
+ for k := i + 1; k < len(paths); k++ {
+ if strings.HasPrefix(paths[k], paths[i]+"/") {
+ return ErrLocalInventory
+ }
+ }
+ }
+ j = localJournal{V: 1, Host: host, Incarnation: captured.Incarnation, Op: op, Obligation: id, Retirement: o.Retirement, Mode: mode, Installation: captured.Stamp, ArtifactStamp: stamp}
+ for _, p := range paths {
+ if mode == "installation" || inventory[p] == stamp {
+ j.Items = append(j.Items, localItem{Path: p, Stamp: inventory[p]})
+ }
+ }
+ return nil
+ }
+ work := func() error {
+ if err := l.parkLocal(ctx, &j); err != nil {
+ return err
+ }
+ fresh, err := hostclaim.Read(ctx, l.Claims, host)
+ if err != nil {
+ return err
+ }
+ if fresh.Absent {
+ return ErrLocalIdentity
+ }
+ _, err = hostclaim.Update(ctx, l.Claims, fresh, func(c *hostclaim.Claim) error {
+ o, err := l.chore(c, captured, id, op)
+ if err != nil {
+ return err
+ }
+ if err := l.Authorize(ctx, c, o); err != nil {
+ return err
+ }
+ entry := c.Inventory[l.Box]
+ for _, item := range j.Items {
+ if entry.Paths[item.Path] != item.Stamp {
+ return ErrLocalInventory
+ }
+ }
+ if (mode == "installation" || mode == "empty") && len(entry.Paths) != len(j.Items) {
+ return ErrLocalInventory
+ }
+ for _, item := range j.Items {
+ delete(entry.Paths, item.Path)
+ }
+ c.Inventory[l.Box] = entry
+ return hostclaim.CompleteObligation(op, id)(c)
+ })
+ return err
+ }
+ if mode == "installation" {
+ err = l.Guard.RetireInstallationWith(ctx, host, captured, check, work)
+ } else if mode == "empty" {
+ err = l.Guard.WithNoInstallation(ctx, host, captured.Incarnation, check, work)
+ } else {
+ err = l.Guard.With(ctx, host, captured, check, work)
+ }
+ if errors.Is(err, errLocalComplete) {
+ return nil
+ }
+ return err
+}
+
+func (l Local) RecordInventory(ctx context.Context, host string, captured hostguard.Captured, layout string, paths []string, check func(*hostclaim.Claim) error) error {
+ if !localSupported() {
+ return ErrLocalUnsupported
+ }
+ if ctx == nil || l.Claims == nil || l.Box == "" || l.Guard.DataDir != l.DataDir || l.DataDir == "" || check == nil || captured.Stamp == 0 || (layout != "flat" && layout != "nested") {
+ return ErrLocalInventory
+ }
+ for _, p := range paths {
+ if !localPath(p) {
+ return ErrLocalInventory
+ }
+ }
+ return l.Guard.With(ctx, host, captured, check, func() error {
+ obs, err := hostclaim.Read(ctx, l.Claims, host)
+ if err != nil {
+ return err
+ }
+ if obs.Absent {
+ return ErrLocalIdentity
+ }
+ _, err = hostclaim.Update(ctx, l.Claims, obs, func(c *hostclaim.Claim) error {
+ if c.Holder == nil || c.Holder.Incarnation != captured.Incarnation {
+ return ErrLocalIdentity
+ }
+ if err := check(c); err != nil {
+ return err
+ }
+ if c.Inventory == nil {
+ c.Inventory = make(map[string]hostclaim.InventoryEntry)
+ }
+ entry, exists := c.Inventory[l.Box]
+ if exists && entry.Layout != layout && len(entry.Paths) != 0 {
+ return ErrLocalInventory
+ }
+ entry.Layout = layout
+ if entry.Paths == nil {
+ entry.Paths = make(map[string]uint64)
+ }
+ for _, p := range paths {
+ if old := entry.Paths[p]; old != 0 && old != captured.Stamp {
+ return ErrLocalInventory
+ }
+ entry.Paths[p] = captured.Stamp
+ }
+ all := make([]string, 0, len(entry.Paths))
+ for p, s := range entry.Paths {
+ if !localPath(p) || s == 0 {
+ return ErrLocalInventory
+ }
+ all = append(all, p)
+ }
+ sort.Strings(all)
+ for i := range all {
+ for k := i + 1; k < len(all); k++ {
+ if strings.HasPrefix(all[k], all[i]+"/") {
+ return ErrLocalInventory
+ }
+ }
+ }
+ c.Inventory[l.Box] = entry
+ return nil
+ })
+ return err
+ })
+}
+
+func (l Local) AcknowledgeEmpty(ctx context.Context, host, incarnation, id, op string) error {
+ return l.retire(ctx, host, hostguard.Captured{Incarnation: incarnation}, id, op, "empty", 0)
+}
diff --git a/internal/park/local_linux.go b/internal/park/local_linux.go
new file mode 100644
index 000000000..a9866a923
--- /dev/null
+++ b/internal/park/local_linux.go
@@ -0,0 +1,348 @@
+//go:build linux
+
+package park
+
+import (
+ "bytes"
+ "context"
+ "crypto/rand"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "io"
+ "os"
+ "reflect"
+ "strings"
+
+ "golang.org/x/sys/unix"
+)
+
+func localSupported() bool { return true }
+
+func localDirAt(parent int, name string, create bool) (int, error) {
+ if !localComponent(name) {
+ return -1, ErrLocalInventory
+ }
+ if create {
+ if err := unix.Mkdirat(parent, name, 0770); err != nil && !errors.Is(err, unix.EEXIST) {
+ return -1, err
+ }
+ }
+ fd, err := unix.Openat(parent, name, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0)
+ if err != nil {
+ return -1, err
+ }
+ var st unix.Stat_t
+ if err := unix.Fstat(fd, &st); err != nil || st.Mode&unix.S_IFMT != unix.S_IFDIR {
+ unix.Close(fd)
+ if err != nil {
+ return -1, err
+ }
+ return -1, ErrLocalInventory
+ }
+ if create {
+ // A retry can encounter another worker's directory. Root also respects its owner.
+ if st.Uid == uint32(os.Geteuid()) {
+ if err := unix.Fchmod(fd, 02770); err != nil {
+ unix.Close(fd)
+ return -1, err
+ }
+ } else if st.Mode&02770 != 02770 {
+ unix.Close(fd)
+ return -1, ErrLocalInventory
+ }
+ if err := unix.Fsync(fd); err != nil {
+ unix.Close(fd)
+ return -1, err
+ }
+ if err := unix.Fsync(parent); err != nil {
+ unix.Close(fd)
+ return -1, err
+ }
+ }
+ return fd, nil
+}
+
+func localWalk(root int, parts []string, create bool) (int, error) {
+ fd, err := unix.Dup(root)
+ if err != nil {
+ return -1, err
+ }
+ unix.CloseOnExec(fd)
+ for _, p := range parts {
+ next, err := localDirAt(fd, p, create)
+ unix.Close(fd)
+ if err != nil {
+ return -1, err
+ }
+ fd = next
+ }
+ return fd, nil
+}
+
+func localParent(root int, p string, create bool) (int, string, error) {
+ if !localPath(p) {
+ return -1, "", ErrLocalInventory
+ }
+ parts := strings.Split(p, "/")
+ fd, err := localWalk(root, parts[:len(parts)-1], create)
+ return fd, parts[len(parts)-1], err
+}
+
+func localStat(fd int, name string) (unix.Stat_t, error) {
+ var st unix.Stat_t
+ err := unix.Fstatat(fd, name, &st, unix.AT_SYMLINK_NOFOLLOW)
+ if err == nil && st.Mode&unix.S_IFMT != unix.S_IFDIR && st.Mode&unix.S_IFMT != unix.S_IFREG {
+ return st, ErrLocalInventory
+ }
+ return st, err
+}
+
+func localSame(st unix.Stat_t, item localItem) bool {
+ return uint64(st.Dev) == item.Dev && st.Ino == item.Ino && st.Mode&unix.S_IFMT == item.Type
+}
+
+func localReadJournal(dir int) (*localJournal, error) {
+ fd, err := unix.Openat(dir, ".journal.json", unix.O_RDONLY|unix.O_NOFOLLOW|unix.O_CLOEXEC|unix.O_NONBLOCK, 0)
+ if err != nil {
+ return nil, err
+ }
+ f := os.NewFile(uintptr(fd), "journal")
+ defer f.Close()
+ st, err := f.Stat()
+ if err != nil {
+ return nil, err
+ }
+ if !st.Mode().IsRegular() {
+ return nil, ErrLocalJournal
+ }
+ b, err := io.ReadAll(io.LimitReader(f, (4<<20)+1))
+ if err != nil {
+ return nil, err
+ }
+ if len(b) > 4<<20 {
+ return nil, ErrLocalJournal
+ }
+ d := json.NewDecoder(bytes.NewReader(b))
+ d.DisallowUnknownFields()
+ var j localJournal
+ if err := d.Decode(&j); err != nil {
+ return nil, err
+ }
+ if _, err := d.Token(); err != io.EOF {
+ return nil, ErrLocalJournal
+ }
+ return &j, nil
+}
+
+func localWriteJournal(dir int, j *localJournal) error {
+ b, err := json.Marshal(j)
+ if err != nil {
+ return err
+ }
+ var nonce [16]byte
+ if _, err := rand.Read(nonce[:]); err != nil {
+ return err
+ }
+ name := ".journal-" + hex.EncodeToString(nonce[:])
+ fd, err := unix.Openat(dir, name, unix.O_WRONLY|unix.O_CREAT|unix.O_EXCL|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0660)
+ if err != nil {
+ return err
+ }
+ f := os.NewFile(uintptr(fd), "journal-temp")
+ defer f.Close()
+ defer unix.Unlinkat(dir, name, 0)
+ if err := unix.Fchmod(fd, 0660); err != nil {
+ return err
+ }
+ if _, err := f.Write(b); err != nil {
+ return err
+ }
+ if err := f.Sync(); err != nil {
+ return err
+ }
+ if err := unix.Renameat(dir, name, dir, ".journal.json"); err != nil {
+ return err
+ }
+ return unix.Fsync(dir)
+}
+
+func localJournalMatches(a, b localJournal) bool {
+ if len(a.Items) != len(b.Items) {
+ return false
+ }
+ for i := range a.Items {
+ if a.Items[i].Path != b.Items[i].Path || a.Items[i].Stamp != b.Items[i].Stamp {
+ return false
+ }
+ if a.Items[i].Missing {
+ if a.Items[i].Dev != 0 || a.Items[i].Ino != 0 || a.Items[i].Type != 0 {
+ return false
+ }
+ } else if a.Items[i].Ino == 0 || (a.Items[i].Type != unix.S_IFDIR && a.Items[i].Type != unix.S_IFREG) {
+ return false
+ }
+ }
+ a.Items, b.Items = nil, nil
+ return reflect.DeepEqual(a, b)
+}
+
+func (l Local) parkLocal(ctx context.Context, want *localJournal) error {
+ root, err := unix.Open(l.DataDir, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0)
+ if err != nil {
+ return err
+ }
+ defer unix.Close(root)
+ parked, err := localDirAt(root, ".parked", false)
+ if err != nil {
+ return err
+ }
+ defer unix.Close(parked)
+ event, err := localWalk(parked, []string{want.Incarnation, want.Host, want.Retirement}, true)
+ if err != nil {
+ return err
+ }
+ defer unix.Close(event)
+ dest, err := localDirAt(event, "artifacts", true)
+ if err != nil {
+ return err
+ }
+ defer unix.Close(dest)
+ prior, err := localReadJournal(event)
+ if errors.Is(err, unix.ENOENT) {
+ for i := range want.Items {
+ it := &want.Items[i]
+ src, name, e := localParent(root, it.Path, false)
+ if errors.Is(e, unix.ENOENT) {
+ it.Missing = true
+ continue
+ }
+ if e != nil {
+ return e
+ }
+ st, e := localStat(src, name)
+ unix.Close(src)
+ if errors.Is(e, unix.ENOENT) {
+ it.Missing = true
+ continue
+ }
+ if e != nil {
+ return e
+ }
+ it.Dev, it.Ino, it.Type = uint64(st.Dev), st.Ino, st.Mode&unix.S_IFMT
+ dp, dn, e := localParent(dest, it.Path, true)
+ if e != nil {
+ return e
+ }
+ _, e = localStat(dp, dn)
+ unix.Close(dp)
+ if !errors.Is(e, unix.ENOENT) {
+ if e != nil {
+ return e
+ }
+ return ErrLocalJournal
+ }
+ }
+ if err := localWriteJournal(event, want); err != nil {
+ return err
+ }
+ } else if err != nil {
+ return err
+ } else {
+ if !localJournalMatches(*prior, *want) {
+ return ErrLocalJournal
+ }
+ *want = *prior
+ }
+ for i := range want.Items {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ it := &want.Items[i]
+ if it.Missing {
+ // A still-active installation can recreate an absent path between attempts.
+ sp, sn, err := localParent(root, it.Path, false)
+ if err == nil {
+ _, err = localStat(sp, sn)
+ unix.Close(sp)
+ }
+ if !errors.Is(err, unix.ENOENT) {
+ if err != nil {
+ return err
+ }
+ return ErrLocalJournal
+ }
+ it.Done = true
+ continue
+ }
+ dp, dn, err := localParent(dest, it.Path, true)
+ if err != nil {
+ return err
+ }
+ sp, sn, err := localParent(root, it.Path, false)
+ if err != nil {
+ unix.Close(dp)
+ return err
+ }
+ err = l.moveLocalItem(sp, sn, dp, dn, it)
+ if err == nil {
+ err = unix.Fsync(sp)
+ }
+ if err == nil {
+ err = unix.Fsync(dp)
+ }
+ unix.Close(sp)
+ unix.Close(dp)
+ if err != nil {
+ return err
+ }
+ it.Done = true
+ if err := localWriteJournal(event, want); err != nil {
+ return err
+ }
+ }
+ if err := localWriteJournal(event, want); err != nil {
+ return err
+ }
+ if err := unix.Fsync(dest); err != nil {
+ return err
+ }
+ return unix.Fsync(event)
+}
+
+func (l Local) moveLocalItem(sp int, sn string, dp int, dn string, it *localItem) error {
+ dst, de := localStat(dp, dn)
+ if de == nil {
+ if !localSame(dst, *it) {
+ return ErrLocalJournal
+ }
+ _, se := localStat(sp, sn)
+ if se == nil {
+ return ErrLocalJournal
+ }
+ if se != nil && !errors.Is(se, unix.ENOENT) {
+ return se
+ }
+ return nil
+ }
+ if !errors.Is(de, unix.ENOENT) {
+ return de
+ }
+ if it.Done {
+ return ErrLocalJournal
+ }
+ src, err := localStat(sp, sn)
+ if err != nil {
+ return err
+ }
+ if !localSame(src, *it) {
+ return ErrLocalJournal
+ }
+ if err := unix.Renameat2(sp, sn, dp, dn, unix.RENAME_NOREPLACE); err != nil {
+ return err
+ }
+ if l.afterRename != nil {
+ return l.afterRename(it.Path)
+ }
+ return nil
+}
diff --git a/internal/park/local_linux_test.go b/internal/park/local_linux_test.go
new file mode 100644
index 000000000..aae79d7f9
--- /dev/null
+++ b/internal/park/local_linux_test.go
@@ -0,0 +1,673 @@
+//go:build linux
+
+package park
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "syscall"
+ "testing"
+ "time"
+
+ "golang.org/x/sys/unix"
+
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/hostguard"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+type localFixture struct {
+ l Local
+ host string
+ captured hostguard.Captured
+}
+
+func localFixtureFor(t *testing.T, paths map[string]uint64) localFixture {
+ t.Helper()
+ host := "shop.bab.oncarlos.com"
+ data, locks := t.TempDir(), t.TempDir()
+ for _, name := range []string{"hostguard", ".parked"} {
+ if err := os.Mkdir(filepath.Join(data, name), 0770); err != nil {
+ t.Fatal(err)
+ }
+ }
+ s := release.NewDirStore(t.TempDir())
+ at := time.Date(2026, 10, 4, 1, 0, 0, 0, time.UTC)
+ inc, err := hostclaim.Incarnation("bab", "jam", host, at)
+ if err != nil {
+ t.Fatal(err)
+ }
+ c := &hostclaim.Claim{V: 2, Host: host, Gen: 5,
+ Holder: &hostclaim.Holder{Family: "instance", Key: "apps/bab/jam/instances/" + host + ".json", Account: "bab", App: "jam", CreatedAt: at, Incarnation: inc, Location: "instances/" + host + "/" + inc, Stamp: 5},
+ Op: hostclaim.Op{Kind: hostclaim.KindParking, ID: "delete", Exec: "box"},
+ Inventory: map[string]hostclaim.InventoryEntry{"box": {Layout: "flat", Paths: paths}},
+ Obligations: []hostclaim.Obligation{{ID: "local", Op: "delete", Kind: "park-local", Box: "box", Owner: "box:box", Retirement: "retirement"}}}
+ if err := c.Validate(); err != nil {
+ t.Fatal(err)
+ }
+ b, err := json.Marshal(c)
+ if err != nil {
+ t.Fatal(err)
+ }
+ key, err := hostclaim.Key(host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := s.Put(t.Context(), key, b, ""); err != nil {
+ t.Fatal(err)
+ }
+ g := hostguard.Guard{DataDir: data, LockDir: locks, Claims: hostguard.StoreReader{Store: s}}
+ if err := g.Activate(t.Context(), host, "box", nil); err != nil {
+ t.Fatal(err)
+ }
+ capture, err := hostguard.Capture(data, host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return localFixture{l: Local{Authorize: func(context.Context, *hostclaim.Claim, hostclaim.Obligation) error { return nil }, Guard: g, Claims: s, DataDir: data, Box: "box"}, host: host, captured: capture}
+}
+
+func (f localFixture) write(t *testing.T, name, body string) {
+ t.Helper()
+ p := filepath.Join(f.l.DataDir, name)
+ if err := os.MkdirAll(filepath.Dir(p), 0770); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(p, []byte(body), 0660); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func (f localFixture) parked(name string) string {
+ return filepath.Join(f.l.DataDir, ".parked", f.captured.Incarnation, f.host, "retirement", "artifacts", name)
+}
+
+func TestLocalArtifactStampAndInstallationRetirement(t *testing.T) {
+ for _, mode := range []string{"artifacts", "installation"} {
+ t.Run(mode, func(t *testing.T) {
+ f := localFixtureFor(t, map[string]uint64{"old.db": 3, "new.db": 5, "nested/state": 3})
+ for _, p := range []string{"old.db", "new.db", "nested/state"} {
+ f.write(t, p, p)
+ }
+ var err error
+ if mode == "artifacts" {
+ err = f.l.RetireArtifacts(t.Context(), f.host, f.captured, "local", "delete", 3)
+ } else {
+ err = f.l.RetireInstallation(t.Context(), f.host, f.captured, "local", "delete")
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, p := range []string{"old.db", "nested/state"} {
+ if b, err := os.ReadFile(f.parked(p)); err != nil || string(b) != p {
+ t.Fatalf("parked %s: %q %v", p, b, err)
+ }
+ }
+ got, err := hostguard.Capture(f.l.DataDir, f.host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if mode == "artifacts" {
+ if got != f.captured {
+ t.Fatal("artifact retirement changed stamp")
+ }
+ if _, err := os.Stat(filepath.Join(f.l.DataDir, "new.db")); err != nil {
+ t.Fatal(err)
+ }
+ } else if got.Stamp != 0 {
+ t.Fatal("installation still active")
+ }
+ obs, err := hostclaim.Read(t.Context(), f.l.Claims, f.host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(obs.Claim.Obligations) != 0 {
+ t.Fatal("not acknowledged")
+ }
+ want := 0
+ if mode == "artifacts" {
+ want = 1
+ }
+ if len(obs.Claim.Inventory["box"].Paths) != want {
+ t.Fatal("inventory mismatch")
+ }
+ if err := f.l.RetireInstallation(t.Context(), f.host, f.captured, "local", "delete"); err != nil {
+ t.Fatal("completion retry", err)
+ }
+ })
+ }
+}
+
+func TestLocalResumeAfterEveryRenameHoldsGuard(t *testing.T) {
+ for _, stop := range []string{"a", "b"} {
+ t.Run(stop, func(t *testing.T) {
+ f := localFixtureFor(t, map[string]uint64{"a": 5, "b": 5})
+ f.write(t, "a", "a")
+ f.write(t, "b", "b")
+ crash := errors.New("crash after rename")
+ f.l.afterRename = func(p string) error {
+ if p != stop {
+ return nil
+ }
+ ctx, cancel := context.WithTimeout(t.Context(), 30*time.Millisecond)
+ defer cancel()
+ if err := f.l.Guard.With(ctx, f.host, f.captured, func(*hostclaim.Claim) error { return nil }, func() error { t.Fatal("lock released inside retirement"); return nil }); !errors.Is(err, context.DeadlineExceeded) {
+ t.Fatalf("competing writer %v", err)
+ }
+ return crash
+ }
+ if err := f.l.RetireInstallation(t.Context(), f.host, f.captured, "local", "delete"); !errors.Is(err, crash) {
+ t.Fatal(err)
+ }
+ current, err := hostguard.Capture(f.l.DataDir, f.host)
+ if err != nil || current.Stamp != 0 {
+ t.Fatalf("not retired before rename: %+v %v", current, err)
+ }
+ obs, err := hostclaim.Read(t.Context(), f.l.Claims, f.host)
+ if err != nil || len(obs.Claim.Obligations) != 1 {
+ t.Fatal("early completion", err)
+ }
+ f.l.afterRename = nil
+ if err := f.l.RetireInstallation(t.Context(), f.host, f.captured, "local", "delete"); err != nil {
+ t.Fatal(err)
+ }
+ for _, p := range []string{"a", "b"} {
+ if b, err := os.ReadFile(f.parked(p)); err != nil || string(b) != p {
+ t.Fatalf("lost %s", p)
+ }
+ }
+ })
+ }
+}
+
+func TestLocalRestrictiveUmask(t *testing.T) {
+ // Umask is process-wide: this test and its fixture setup must not run in parallel.
+ f := localFixtureFor(t, map[string]uint64{"nested/a": 5, "nested/b": 5})
+ f.write(t, "nested/a", "a")
+ f.write(t, "nested/b", "b")
+ group := os.Getegid()
+ if os.Geteuid() == 0 {
+ group = 65534
+ }
+ for _, path := range []string{f.l.DataDir, filepath.Join(f.l.DataDir, ".parked"), filepath.Join(f.l.DataDir, "nested")} {
+ dir, err := os.Open(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if os.Geteuid() == 0 {
+ if err := dir.Chown(0, group); err != nil {
+ dir.Close()
+ t.Fatal(err)
+ }
+ }
+ if err := dir.Chmod(os.ModeSetgid | 0770); err != nil {
+ dir.Close()
+ t.Fatal(err)
+ }
+ dir.Close()
+ }
+ oldUmask := syscall.Umask(0o077)
+ defer syscall.Umask(oldUmask)
+ crash := errors.New("crash under restrictive umask")
+ f.l.afterRename = func(string) error { return crash }
+ if err := f.l.RetireInstallation(t.Context(), f.host, f.captured, "local", "delete"); !errors.Is(err, crash) {
+ t.Fatal(err)
+ }
+ base := filepath.Join(f.l.DataDir, ".parked", f.captured.Incarnation)
+ event := filepath.Join(base, f.host, "retirement")
+ for _, path := range []string{base, filepath.Join(base, f.host), event, filepath.Join(event, "artifacts"), filepath.Join(event, "artifacts", "nested")} {
+ var st unix.Stat_t
+ if err := unix.Lstat(path, &st); err != nil {
+ t.Fatal(err)
+ }
+ if st.Mode&unix.S_IFMT != unix.S_IFDIR || st.Mode&07777 != 02770 || st.Gid != uint32(group) {
+ t.Fatalf("archive directory %s: mode=%#o gid=%d", path, st.Mode, st.Gid)
+ }
+ }
+ if os.Geteuid() == 0 {
+ body, err := os.ReadFile(filepath.Join(event, ".journal.json"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ dir, err := os.Open(f.l.DataDir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer dir.Close()
+ exe, err := os.Executable()
+ if err != nil {
+ t.Fatal(err)
+ }
+ run := func(refuse bool) {
+ t.Helper()
+ cmd := exec.Command(exe, "-test.run=^TestLocalRestrictiveUmaskWorker$")
+ cmd.ExtraFiles = []*os.File{dir}
+ cmd.Env = append(os.Environ(), "PARK_UMASK_WORKER=1", "PARK_UMASK_JOURNAL="+string(body))
+ if refuse {
+ cmd.Env = append(cmd.Env, "PARK_UMASK_REFUSE=1")
+ }
+ if out, err := cmd.CombinedOutput(); err != nil {
+ t.Fatalf("group worker: %v\n%s", err, out)
+ }
+ }
+ // A worker must refuse, not chmod, directories owned by another UID.
+ for _, mode := range []os.FileMode{os.ModeSetgid | 0750, 0770} {
+ path := filepath.Join(event, "artifacts")
+ if err := os.Chmod(path, mode); err != nil {
+ t.Fatal(err)
+ }
+ run(true)
+ st, err := os.Stat(path)
+ if err != nil || st.Mode()&(os.ModeSetgid|0777) != mode {
+ t.Fatal("foreign directory permissions changed", err)
+ }
+ if err := os.Chmod(path, os.ModeSetgid|0770); err != nil {
+ t.Fatal(err)
+ }
+ }
+ run(false)
+ } else {
+ t.Log("mode and group assertions passed; cross-user subprocess requires root")
+ }
+ f.l.afterRename = nil
+ if err := f.l.RetireInstallation(t.Context(), f.host, f.captured, "local", "delete"); err != nil {
+ t.Fatal(err)
+ }
+ for _, name := range []string{"a", "b"} {
+ if body, err := os.ReadFile(f.parked("nested/" + name)); err != nil || string(body) != name {
+ t.Fatalf("resumed artifact %s: %q %v", name, body, err)
+ }
+ }
+ obs, err := hostclaim.Read(t.Context(), f.l.Claims, f.host)
+ if err != nil || len(obs.Claim.Obligations) != 0 {
+ t.Fatal("resumed retirement not acknowledged", err)
+ }
+}
+
+func TestLocalRestrictiveUmaskWorker(t *testing.T) {
+ if os.Getenv("PARK_UMASK_WORKER") != "1" {
+ t.Skip("subprocess helper")
+ }
+ var want localJournal
+ if err := json.Unmarshal([]byte(os.Getenv("PARK_UMASK_JOURNAL")), &want); err != nil {
+ t.Fatal(err)
+ }
+ // Enter the fixture through an inherited descriptor; its temp ancestors may be private.
+ if err := unix.Fchdir(3); err != nil {
+ t.Fatal(err)
+ }
+ for _, change := range []func() error{
+ func() error { return syscall.Setgroups([]int{65534}) },
+ func() error { return syscall.Setgid(65534) },
+ func() error { return syscall.Setuid(65534) },
+ } {
+ if err := change(); err != nil {
+ t.Fatal(err)
+ }
+ }
+ oldUmask := syscall.Umask(0o077)
+ defer syscall.Umask(oldUmask)
+ err := (Local{DataDir: "."}).parkLocal(t.Context(), &want)
+ if os.Getenv("PARK_UMASK_REFUSE") == "1" {
+ if !errors.Is(err, ErrLocalInventory) {
+ t.Fatalf("foreign directory without required mode accepted: %v", err)
+ }
+ return
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Stat("nested/b"); !errors.Is(err, os.ErrNotExist) {
+ t.Fatal("second process did not retire remaining artifact", err)
+ }
+}
+
+func TestLocalRefusesSymlinksAndOverlaps(t *testing.T) {
+ for _, place := range []string{"source", "ancestor", "park-root", "destination", "overlap", "missing-inventory"} {
+ t.Run(place, func(t *testing.T) {
+ paths := map[string]uint64{"sub/a": 5}
+ if place == "overlap" {
+ paths["sub"] = 3
+ }
+ f := localFixtureFor(t, paths)
+ f.write(t, "sub/a", "source")
+ outside := t.TempDir()
+ sentinel := filepath.Join(outside, "a")
+ if err := os.WriteFile(sentinel, []byte("sentinel"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ switch place {
+ case "source":
+ if err := os.Remove(filepath.Join(f.l.DataDir, "sub/a")); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(sentinel, filepath.Join(f.l.DataDir, "sub/a")); err != nil {
+ t.Fatal(err)
+ }
+ case "ancestor":
+ if err := os.Rename(filepath.Join(f.l.DataDir, "sub"), filepath.Join(f.l.DataDir, "safe")); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(outside, filepath.Join(f.l.DataDir, "sub")); err != nil {
+ t.Fatal(err)
+ }
+ case "park-root":
+ if err := os.Remove(filepath.Join(f.l.DataDir, ".parked")); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(outside, filepath.Join(f.l.DataDir, ".parked")); err != nil {
+ t.Fatal(err)
+ }
+ case "destination":
+ if err := os.MkdirAll(filepath.Dir(f.parked("sub/a")), 0770); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(sentinel, f.parked("sub/a")); err != nil {
+ t.Fatal(err)
+ }
+ case "missing-inventory":
+ obs, err := hostclaim.Read(t.Context(), f.l.Claims, f.host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := hostclaim.Update(t.Context(), f.l.Claims, obs, func(c *hostclaim.Claim) error { delete(c.Inventory, "box"); return nil }); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := f.l.RetireInstallation(t.Context(), f.host, f.captured, "local", "delete"); err == nil {
+ t.Fatal("unsafe retirement accepted")
+ }
+ if b, err := os.ReadFile(sentinel); err != nil || string(b) != "sentinel" {
+ t.Fatal("symlink target changed")
+ }
+ obs, err := hostclaim.Read(t.Context(), f.l.Claims, f.host)
+ if err != nil || len(obs.Claim.Obligations) != 1 {
+ t.Fatal("acknowledged failure", err)
+ }
+ })
+ }
+}
+
+func TestLocalEmptyAndAbsentArtifactsAcknowledge(t *testing.T) {
+ for _, paths := range []map[string]uint64{{}, {"missing/file": 5}} {
+ f := localFixtureFor(t, paths)
+ if err := f.l.RetireInstallation(t.Context(), f.host, f.captured, "local", "delete"); err != nil {
+ t.Fatal(err)
+ }
+ obs, err := hostclaim.Read(t.Context(), f.l.Claims, f.host)
+ if err != nil || len(obs.Claim.Obligations) != 0 {
+ t.Fatal("not complete", err)
+ }
+ }
+}
+
+func TestLocalOldRetirementCannotFenceSuccessor(t *testing.T) {
+ f := localFixtureFor(t, map[string]uint64{"a": 5})
+ f.write(t, "a", "old")
+ f.l.afterRename = func(string) error { return errors.New("crash") }
+ if err := f.l.RetireInstallation(t.Context(), f.host, f.captured, "local", "delete"); err == nil {
+ t.Fatal("expected crash")
+ }
+ obs, err := hostclaim.Read(t.Context(), f.l.Claims, f.host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := hostclaim.Update(t.Context(), f.l.Claims, obs, func(c *hostclaim.Claim) error {
+ c.Holder.Stamp = c.Gen + 1
+ c.Inventory["box"].Paths["a"] = c.Gen + 1
+ return nil
+ }); err != nil {
+ t.Fatal(err)
+ }
+ if err := f.l.Guard.Activate(t.Context(), f.host, "box", nil); err != nil {
+ t.Fatal(err)
+ }
+ f.write(t, "a", "successor")
+ f.l.afterRename = nil
+ if err := f.l.RetireInstallation(t.Context(), f.host, f.captured, "local", "delete"); !errors.Is(err, hostguard.ErrStampMismatch) {
+ t.Fatal(err)
+ }
+ if b, err := os.ReadFile(filepath.Join(f.l.DataDir, "a")); err != nil || string(b) != "successor" {
+ t.Fatal("successor damaged")
+ }
+}
+
+func TestLocalArtifactResumeRefusesReusedSource(t *testing.T) {
+ f := localFixtureFor(t, map[string]uint64{"a": 3})
+ f.write(t, "a", "predecessor")
+ f.l.afterRename = func(string) error { return errors.New("crash") }
+ if err := f.l.RetireArtifacts(t.Context(), f.host, f.captured, "local", "delete", 3); err == nil {
+ t.Fatal("expected crash")
+ }
+ f.write(t, "a", "replacement")
+ f.l.afterRename = nil
+ if err := f.l.RetireArtifacts(t.Context(), f.host, f.captured, "local", "delete", 3); !errors.Is(err, ErrLocalJournal) {
+ t.Fatal(err)
+ }
+ if b, err := os.ReadFile(filepath.Join(f.l.DataDir, "a")); err != nil || string(b) != "replacement" {
+ t.Fatal("replacement changed")
+ }
+ if b, err := os.ReadFile(f.parked("a")); err != nil || string(b) != "predecessor" {
+ t.Fatal("archive changed")
+ }
+ obs, err := hostclaim.Read(t.Context(), f.l.Claims, f.host)
+ if err != nil || len(obs.Claim.Obligations) != 1 || len(obs.Claim.Inventory["box"].Paths) != 1 {
+ t.Fatal("source reuse acknowledged", err)
+ }
+}
+
+func TestLocalMissingPathRecreatedAfterCrash(t *testing.T) {
+ for _, kind := range []string{"file", "directory", "leaf-symlink", "ancestor-symlink", "still-missing"} {
+ t.Run(kind, func(t *testing.T) {
+ missing := "a-missing"
+ if kind == "ancestor-symlink" {
+ missing += "/child"
+ }
+ f := localFixtureFor(t, map[string]uint64{missing: 5, "z-present": 5})
+ f.write(t, "z-present", "old")
+ crash := errors.New("crash after missing check and artifact rename")
+ f.l.afterRename = func(string) error { return crash }
+ if err := f.l.RetireArtifacts(t.Context(), f.host, f.captured, "local", "delete", 5); !errors.Is(err, crash) {
+ t.Fatal(err)
+ }
+ f.l.afterRename = nil
+ outside := t.TempDir()
+ target := filepath.Join(outside, "child")
+ if err := os.WriteFile(target, []byte("outside"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ path := filepath.Join(f.l.DataDir, "a-missing")
+ switch kind {
+ case "file":
+ f.write(t, missing, "new")
+ case "directory":
+ if err := os.Mkdir(path, 0770); err != nil {
+ t.Fatal(err)
+ }
+ case "leaf-symlink":
+ if err := os.Symlink(target, path); err != nil {
+ t.Fatal(err)
+ }
+ case "ancestor-symlink":
+ if err := os.Symlink(outside, path); err != nil {
+ t.Fatal(err)
+ }
+ }
+ err := f.l.RetireArtifacts(t.Context(), f.host, f.captured, "local", "delete", 5)
+ if kind == "still-missing" {
+ if err != nil {
+ t.Fatal(err)
+ }
+ } else if err == nil {
+ t.Fatal("acknowledged a recreated missing path")
+ }
+ obs, readErr := hostclaim.Read(t.Context(), f.l.Claims, f.host)
+ if readErr != nil {
+ t.Fatal(readErr)
+ }
+ if kind == "still-missing" {
+ if len(obs.Claim.Obligations) != 0 || len(obs.Claim.Inventory["box"].Paths) != 0 {
+ t.Fatal("missing path did not acknowledge")
+ }
+ } else {
+ if len(obs.Claim.Obligations) != 1 || len(obs.Claim.Inventory["box"].Paths) != 2 {
+ t.Fatal("refusal changed inventory or chore")
+ }
+ if _, err := os.Lstat(path); err != nil {
+ t.Fatal("recreated artifact lost", err)
+ }
+ }
+ if got, err := hostguard.Capture(f.l.DataDir, f.host); err != nil || got != f.captured {
+ t.Fatal("artifact retirement changed installation", err)
+ }
+ if b, err := os.ReadFile(target); err != nil || string(b) != "outside" {
+ t.Fatal("followed planted symlink", err)
+ }
+ if b, err := os.ReadFile(f.parked("z-present")); err != nil || string(b) != "old" {
+ t.Fatal("lost parked artifact", err)
+ }
+ })
+ }
+}
+
+func TestLocalResolvedDirectoryAndDestinationConflict(t *testing.T) {
+ t.Run("nested directory", func(t *testing.T) {
+ f := localFixtureFor(t, map[string]uint64{"nested/instance": 5})
+ f.write(t, "nested/instance/data/db", "database")
+ if err := f.l.RetireInstallation(t.Context(), f.host, f.captured, "local", "delete"); err != nil {
+ t.Fatal(err)
+ }
+ if b, err := os.ReadFile(f.parked("nested/instance/data/db")); err != nil || string(b) != "database" {
+ t.Fatal("nested tree lost", err)
+ }
+ })
+ t.Run("destination conflict", func(t *testing.T) {
+ f := localFixtureFor(t, map[string]uint64{"a": 5})
+ f.write(t, "a", "source")
+ if err := os.MkdirAll(filepath.Dir(f.parked("a")), 0770); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(f.parked("a"), []byte("foreign"), 0660); err != nil {
+ t.Fatal(err)
+ }
+ if err := f.l.RetireInstallation(t.Context(), f.host, f.captured, "local", "delete"); !errors.Is(err, ErrLocalJournal) {
+ t.Fatal(err)
+ }
+ if b, err := os.ReadFile(f.parked("a")); err != nil || string(b) != "foreign" {
+ t.Fatal("destination overwritten")
+ }
+ if b, err := os.ReadFile(filepath.Join(f.l.DataDir, "a")); err != nil || string(b) != "source" {
+ t.Fatal("source lost")
+ }
+ })
+}
+
+func TestLocalAuthorizationAndInventoryEnrollment(t *testing.T) {
+ t.Run("authorization", func(t *testing.T) {
+ f := localFixtureFor(t, map[string]uint64{"a": 5})
+ f.write(t, "a", "data")
+ denied := errors.New("stop evidence unavailable")
+ f.l.Authorize = func(context.Context, *hostclaim.Claim, hostclaim.Obligation) error { return denied }
+ if err := f.l.RetireInstallation(t.Context(), f.host, f.captured, "local", "delete"); !errors.Is(err, denied) {
+ t.Fatal(err)
+ }
+ got, err := hostguard.Capture(f.l.DataDir, f.host)
+ if err != nil || got != f.captured {
+ t.Fatal("unauthorized stamp mutation", err)
+ }
+ if _, err := os.Stat(filepath.Join(f.l.DataDir, "a")); err != nil {
+ t.Fatal(err)
+ }
+ })
+ t.Run("enrollment", func(t *testing.T) {
+ f := localFixtureFor(t, map[string]uint64{"old": 3})
+ allow := func(*hostclaim.Claim) error { return nil }
+ if err := f.l.RecordInventory(t.Context(), f.host, f.captured, "flat", []string{"new"}, allow); err != nil {
+ t.Fatal(err)
+ }
+ obs, err := hostclaim.Read(t.Context(), f.l.Claims, f.host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if obs.Claim.Inventory["box"].Paths["old"] != 3 || obs.Claim.Inventory["box"].Paths["new"] != 5 {
+ t.Fatal("inventory was replaced")
+ }
+ if _, err := os.Stat(filepath.Join(f.l.DataDir, "new")); !errors.Is(err, os.ErrNotExist) {
+ t.Fatal("enrollment materialized state")
+ }
+ for _, p := range []string{"old", "old/child", "new/child", "../escape", "hostguard/record", ".parked/file"} {
+ if err := f.l.RecordInventory(t.Context(), f.host, f.captured, "flat", []string{p}, allow); !errors.Is(err, ErrLocalInventory) {
+ t.Fatalf("%s: %v", p, err)
+ }
+ }
+ if err := f.l.RecordInventory(t.Context(), f.host, f.captured, "nested", nil, allow); !errors.Is(err, ErrLocalInventory) {
+ t.Fatal("changed populated layout", err)
+ }
+ })
+}
+
+func TestLocalCompletionRechecksAuthority(t *testing.T) {
+ f := localFixtureFor(t, map[string]uint64{"a": 5})
+ f.write(t, "a", "data")
+ denied := errors.New("fence lost before acknowledgement")
+ calls := 0
+ f.l.Authorize = func(context.Context, *hostclaim.Claim, hostclaim.Obligation) error {
+ calls++
+ if calls > 1 {
+ return denied
+ }
+ return nil
+ }
+ if err := f.l.RetireInstallation(t.Context(), f.host, f.captured, "local", "delete"); !errors.Is(err, denied) {
+ t.Fatal(err)
+ }
+ obs, err := hostclaim.Read(t.Context(), f.l.Claims, f.host)
+ if err != nil || len(obs.Claim.Obligations) != 1 {
+ t.Fatal("completed without authority", err)
+ }
+ f.l.Authorize = func(context.Context, *hostclaim.Claim, hostclaim.Obligation) error { return nil }
+ if err := f.l.RetireInstallation(t.Context(), f.host, f.captured, "local", "delete"); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestLocalUntouchedBoxAcknowledgement(t *testing.T) {
+ for _, kind := range []string{"absent-record", "active-record", "corrupt-record", "nonempty-inventory"} {
+ t.Run(kind, func(t *testing.T) {
+ paths := map[string]uint64{}
+ if kind == "nonempty-inventory" {
+ paths["a"] = 5
+ }
+ f := localFixtureFor(t, paths)
+ record := filepath.Join(f.l.DataDir, "hostguard", f.host+".json")
+ if kind == "corrupt-record" {
+ if err := os.WriteFile(record, []byte("{"), 0660); err != nil {
+ t.Fatal(err)
+ }
+ } else if kind != "active-record" {
+ if err := os.Remove(record); err != nil {
+ t.Fatal(err)
+ }
+ }
+ err := f.l.AcknowledgeEmpty(t.Context(), f.host, f.captured.Incarnation, "local", "delete")
+ if kind == "absent-record" {
+ if err != nil {
+ t.Fatal(err)
+ }
+ obs, err := hostclaim.Read(t.Context(), f.l.Claims, f.host)
+ if err != nil || len(obs.Claim.Obligations) != 0 {
+ t.Fatal("empty box not acknowledged", err)
+ }
+ if _, err := os.Stat(record); !errors.Is(err, os.ErrNotExist) {
+ t.Fatal("empty retirement created stamp", err)
+ }
+ } else if err == nil {
+ t.Fatal("unsafe empty acknowledgement")
+ }
+ })
+ }
+}
diff --git a/internal/park/local_other.go b/internal/park/local_other.go
new file mode 100644
index 000000000..6f4ff1621
--- /dev/null
+++ b/internal/park/local_other.go
@@ -0,0 +1,9 @@
+//go:build !linux
+
+package park
+
+import "context"
+
+func (l Local) parkLocal(context.Context, *localJournal) error { return ErrLocalUnsupported }
+
+func localSupported() bool { return false }
diff --git a/internal/park/moves_test.go b/internal/park/moves_test.go
new file mode 100644
index 000000000..92ba9d2ea
--- /dev/null
+++ b/internal/park/moves_test.go
@@ -0,0 +1,204 @@
+package park
+
+import (
+ "context"
+ "errors"
+ "net/url"
+ "reflect"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/lease"
+)
+
+func movesFixture(t *testing.T, op string) *fixture {
+ t.Helper()
+ f := newFixture(t)
+ obs, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ f.ob.Park.Location = "moves/" + f.w.Host
+ if op != "" {
+ f.ob.Park.Location += "/" + op
+ }
+ _, err = hostclaim.Update(t.Context(), f.e.Claims, obs, func(c *hostclaim.Claim) error {
+ c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box"}
+ c.Holder.Generation = "live"
+ c.Obligations = []hostclaim.Obligation{f.ob}
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ f.e.View = func(_ context.Context, o hostclaim.Observed) (View, error) {
+ return View{ClaimETag: o.ETag, Region: "ie", AuthoritativeRegion: "ie", MarkerKnown: true, MarkerGeneration: "unrelated", GenerationOf: func(Ref) (string, bool) { return "live", true }}, nil
+ }
+ return f
+}
+
+func TestMovesScopeParksExactPrefixAndResumes(t *testing.T) {
+ for _, op := range []string{"move-1", strings.Repeat("b", 64), ""} {
+ t.Run("op="+op, func(t *testing.T) {
+ f := movesFixture(t, op)
+ p := f.ob.Park.Location + "/"
+ expected := map[Ref]string{}
+ for _, name := range []string{"return/data", "other-op/data"} {
+ for _, body := range []string{"old", "new"} {
+ id := f.b.Put("data", p+name, []byte(body))
+ expected[Ref{p + name, id}] = body
+ }
+ }
+ f.b.BareDelete("data", p+"return/data")
+ lock := Ref{p + "lock.json", f.b.Put("data", p+"lock.json", []byte("lease"))}
+ untouched := map[Ref]string{lock: "lease"}
+ keys := []string{"instances/" + f.w.Host + "/0000/data", "instances/" + f.w.Host + "/" + f.ob.Park.Incarnation + "/live/data", "moves/other.example/move-1/data", "moves/" + f.w.Host + "-other/move-1/data"}
+ if op != "" {
+ keys = append(keys, f.ob.Park.Location+"-sibling/data", "moves/"+f.w.Host+"/sibling/data")
+ }
+ for _, key := range keys {
+ untouched[Ref{key, f.b.Put("data", key, []byte("untouched"))}] = "untouched"
+ }
+ listings := 0
+ f.b.FailOn = func(action, key string) error {
+ if action == "list" && !strings.HasPrefix(key, "parked/") {
+ listings++
+ if key != p {
+ t.Errorf("listing %q, want only %q", key, p)
+ return ErrUnsafe
+ }
+ }
+ return nil
+ }
+ f.e.After = func(step string) error {
+ if step == "delete" {
+ return ErrPending
+ }
+ return nil
+ }
+ if err := f.e.Run(t.Context(), f.w); !errors.Is(err, ErrPending) {
+ t.Fatalf("interrupted retirement: %v", err)
+ }
+ f.e.After = nil
+ checkListing := f.b.FailOn
+ f.b.FailOn = func(action, key string) error {
+ if action == "copy" {
+ t.Error("manifest retry copied a source")
+ return ErrUnsafe
+ }
+ return checkListing(action, key)
+ }
+ if err := f.e.Run(t.Context(), f.w); err != nil {
+ t.Fatalf("resumed retirement: %v", err)
+ }
+ if listings != 2 || f.held.released != 1 {
+ t.Fatalf("listings=%d releases=%d", listings, f.held.released)
+ }
+ f.done(t)
+ m, err := f.e.manifest(t.Context(), f.w, f.ob)
+ if err != nil || m == nil || len(m.Archive) != len(expected) || len(m.Delete) != len(expected)+1 {
+ t.Fatalf("manifest: %+v %v", m, err)
+ }
+ for _, copied := range m.Archive {
+ body, ok := expected[ref(copied.Source)]
+ wantKey := "parked/" + f.ob.Park.Incarnation + "/" + f.w.Host + "/" + f.ob.Retirement + "/moves/" + strings.TrimPrefix(copied.Source.Key, p) + "@" + url.QueryEscape(copied.Source.VersionID)
+ if !ok || copied.Destination.Key != wantKey {
+ t.Fatalf("unexpected archive copy: %+v", copied)
+ }
+ got, err := f.b.GetVersion(t.Context(), "data", copied.Destination.Key, copied.Destination.VersionID)
+ if err != nil || string(got) != body {
+ t.Fatalf("archive body=%q, want %q: %v", got, body, err)
+ }
+ }
+ for _, source := range m.Delete {
+ if _, err := f.b.GetVersion(t.Context(), "data", source.Key, source.VersionID); !errors.Is(err, lease.ErrMirrorNotFound) {
+ t.Fatalf("source remains: %+v %v", source, err)
+ }
+ }
+ for r, body := range untouched {
+ got, err := f.b.GetVersion(t.Context(), "data", r.Key, r.VersionID)
+ if err != nil || string(got) != body {
+ t.Fatalf("unrelated version changed: %+v %q %v", r, got, err)
+ }
+ }
+ })
+ }
+}
+
+func TestMovesScopeUsesPersistedEvidenceHorizon(t *testing.T) {
+ for _, op := range []string{"move-1", ""} {
+ for _, offset := range []time.Duration{-time.Minute, 0, time.Hour} {
+ t.Run("op="+op+"/offset="+offset.String(), func(t *testing.T) {
+ f := movesFixture(t, op)
+ f.b.Put("data", f.ob.Park.Location+"/staging/data", []byte("staged"))
+ deadline := f.ob.Park.EvidenceAt.Add(SignatureAge + f.ob.Park.RequestLimit)
+ f.now = deadline.Add(offset)
+ f.e.After = func(step string) error {
+ if step == "verify" {
+ f.now = deadline.Add(2 * time.Hour)
+ }
+ return nil
+ }
+ err := f.e.Run(t.Context(), f.w)
+ if offset > 0 {
+ if err != nil {
+ t.Fatalf("delayed cleanup did not acknowledge immediately: %v", err)
+ }
+ } else {
+ if !errors.Is(err, ErrPending) {
+ t.Fatalf("early verification: %v", err)
+ }
+ obs, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(obs.Claim.Obligations) != 1 || len(obs.Claim.QuietAfter) != 0 || f.held.released != 0 {
+ t.Fatal("acknowledged before a listing after the horizon")
+ }
+ if !reflect.DeepEqual(obs.Claim.Obligations[0].Park, f.ob.Park) {
+ t.Fatal("persisted evidence changed")
+ }
+ f.e.After = nil
+ if err := f.e.Run(t.Context(), f.w); err != nil {
+ t.Fatalf("later verification: %v", err)
+ }
+ }
+ f.done(t)
+ })
+ }
+ }
+}
+
+func TestBindMovesScope(t *testing.T) {
+ f := newFixture(t)
+ obs, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = hostclaim.Update(t.Context(), f.e.Claims, obs, func(c *hostclaim.Claim) error {
+ c.Obligations[0].Park = nil
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ p := *f.ob.Park
+ p.Location = "moves/" + f.w.Host + "/move-1"
+ authorized := false
+ f.e.Authorize = func(_ context.Context, _ hostclaim.Observed, o hostclaim.Obligation) error {
+ authorized = reflect.DeepEqual(o.Park, &p)
+ return nil
+ }
+ if err := f.e.Bind(t.Context(), f.w, p); err != nil {
+ t.Fatal(err)
+ }
+ obs, err = hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !authorized || !reflect.DeepEqual(obs.Claim.Obligations[0].Park, &p) {
+ t.Fatal("staging scope not authorized and persisted")
+ }
+}
diff --git a/internal/park/remote.go b/internal/park/remote.go
new file mode 100644
index 000000000..983d584c0
--- /dev/null
+++ b/internal/park/remote.go
@@ -0,0 +1,574 @@
+package park
+
+import (
+ "bytes"
+ "context"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "io"
+ "net/url"
+ "reflect"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/lease"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+const SignatureAge = 15 * time.Minute
+
+var (
+ ErrPending = errors.New("park: waiting for verification after horizon")
+ ErrUnsafe = errors.New("park: unsafe or incomplete input")
+ ErrChanged = errors.New("park: authority changed")
+)
+
+type Ref struct{ Key, VersionID string }
+
+func ref(v lease.ObjectVersion) Ref { return Ref{v.Key, v.VersionID} }
+
+type Copy struct {
+ Source lease.ObjectVersion
+ Destination Ref
+}
+type Manifest struct {
+ V int
+ Host, Obligation, Op, Retirement, Region, Location, Incarnation, Generation string
+ Archive []Copy
+ Delete []lease.ObjectVersion
+}
+type Work struct{ Host, ID, Op string }
+
+// Close stops renewal; Release conditionally removes only the held lock version.
+// Acquire must renew until Close, and Check must refuse expired/lost leases.
+type HeldLease interface {
+ Check(context.Context) error
+ Release(context.Context) error
+ Close()
+}
+type Leases interface {
+ Acquire(context.Context, string, string) (HeldLease, error)
+}
+
+type Engine struct {
+ Store lease.VersionedStore
+ Claims release.Store
+ Bucket, Region string
+ Leases Leases
+ Now func() time.Time
+ NewID func() string
+ // View is supplied by moves/Plan 7b, using precisely this fresh observation.
+ View func(context.Context, hostclaim.Observed) (View, error)
+ // Authorize checks phase-specific stop evidence/fences and chore ownership.
+ // It must not manufacture stop evidence from a stopped parent alone.
+ Authorize func(context.Context, hostclaim.Observed, hostclaim.Obligation) error
+ // After is a test crash point, called after durable effects. Nil in callers.
+ After func(string) error
+}
+
+func (e Engine) after(step string) error {
+ if e.After != nil {
+ return e.After(step)
+ }
+ return nil
+}
+func token(s string) bool {
+ return s != "" && s != "." && s != ".." && !strings.ContainsAny(s, "/\\\x00")
+}
+func hex64(s string) bool { _, err := hex.DecodeString(s); return len(s) == 64 && err == nil }
+func find(c *hostclaim.Claim, w Work) (hostclaim.Obligation, bool) {
+ if c == nil {
+ return hostclaim.Obligation{}, false
+ }
+ for _, o := range c.Obligations {
+ if o.ID == w.ID && o.Op == w.Op {
+ return o, true
+ }
+ }
+ return hostclaim.Obligation{}, false
+}
+func validateScope(host string, o hostclaim.Obligation) error {
+ p := o.Park
+ if p.Validate(host) != nil || !token(o.Retirement) || o.Region == "" || o.Owner != "region:"+o.Region {
+ return ErrUnsafe
+ }
+ if o.Kind != "park-replica" && o.Kind != "park-generation" {
+ return ErrUnsafe
+ }
+ if o.Kind == "park-generation" && o.Generation == "" {
+ return ErrUnsafe
+ }
+ return nil
+}
+
+// Bind persists recovery inputs before the first data-bucket effect. Existing
+// Plan 1 chores have no Park field; callers supply real stop-evidence time and
+// a deployment-wide upper bound enforced by every relevant request sender.
+func (e Engine) Bind(ctx context.Context, w Work, p hostclaim.ParkScope) error {
+ obs, err := hostclaim.Read(ctx, e.Claims, w.Host)
+ if err != nil {
+ return err
+ }
+ o, ok := find(obs.Claim, w)
+ if !ok {
+ return ErrChanged
+ }
+ if o.Park != nil {
+ if o.Park.Location == p.Location && o.Park.Incarnation == p.Incarnation && o.Park.RequestLimit == p.RequestLimit && o.Park.EvidenceAt.Equal(p.EvidenceAt) {
+ return nil
+ }
+ return ErrChanged
+ }
+ if obs.Claim.Holder == nil || (p.Location != obs.Claim.Holder.Location && !isMovesScope(w.Host, p.Location)) || p.Incarnation != obs.Claim.Holder.Incarnation {
+ return ErrUnsafe
+ }
+ o.Park = &p
+ if err := validateScope(w.Host, o); err != nil {
+ return err
+ }
+ if e.Authorize == nil {
+ return ErrUnsafe
+ }
+ if err := e.Authorize(ctx, obs, o); err != nil {
+ return err
+ }
+ _, err = hostclaim.Update(ctx, e.Claims, obs, func(c *hostclaim.Claim) error {
+ for i := range c.Obligations {
+ if c.Obligations[i].ID == w.ID && c.Obligations[i].Op == w.Op {
+ c.Obligations[i].Park = &p
+ return nil
+ }
+ }
+ return ErrChanged
+ })
+ return err
+}
+func isMovesScope(host, location string) bool {
+ base := "moves/" + host
+ return location == base || strings.HasPrefix(location, base+"/")
+}
+func prefixes(host, location string) []string {
+ if isMovesScope(host, location) {
+ return []string{location + "/"}
+ }
+ suffix := strings.TrimPrefix(location, "instances/"+host)
+ return []string{location + "/", "moves/" + host + suffix + "/"}
+}
+
+// locationOf partitions legacy level/generation children from hex incarnation
+// children, including objects in moves/. No retired-incarnations lookup.
+func locationOf(host, key string) (string, error) {
+ for _, root := range []string{"instances/", "moves/"} {
+ base := root + host + "/"
+ if !strings.HasPrefix(key, base) {
+ continue
+ }
+ tail := strings.TrimPrefix(key, base)
+ if tail == "" {
+ return "", ErrUnsafe
+ }
+ first, _, child := strings.Cut(tail, "/")
+ if child && hex64(first) {
+ return "instances/" + host + "/" + first, nil
+ }
+ return "instances/" + host, nil
+ }
+ return "", ErrUnsafe
+}
+func scopeContains(host, location, key string) (bool, error) {
+ if isMovesScope(host, location) {
+ return strings.HasPrefix(key, location+"/"), nil
+ }
+ loc, err := locationOf(host, key)
+ return loc == location, err
+}
+func isLock(host, location, key string) bool {
+ for _, p := range prefixes(host, location) {
+ if key == p+"lock.json" {
+ return true
+ }
+ }
+ return false
+}
+func (e Engine) list(ctx context.Context, host, location string) ([]lease.ObjectVersion, error) {
+ var out []lease.ObjectVersion
+ for _, p := range prefixes(host, location) {
+ vs, err := e.Store.ListVersions(ctx, e.Bucket, p)
+ if err != nil {
+ return nil, err
+ }
+ out = append(out, vs...)
+ }
+ seen := map[Ref]bool{}
+ filtered := out[:0]
+ for _, v := range out {
+ inScope, err := scopeContains(host, location, v.Key)
+ if err != nil || v.VersionID == "" || seen[ref(v)] {
+ return nil, ErrUnsafe
+ }
+ seen[ref(v)] = true
+ if inScope && !isLock(host, location, v.Key) {
+ filtered = append(filtered, v)
+ }
+ }
+ sort.Slice(filtered, func(i, j int) bool {
+ a, b := filtered[i], filtered[j]
+ if a.Key == b.Key {
+ return a.VersionID < b.VersionID
+ }
+ return a.Key < b.Key
+ })
+ return filtered, nil
+}
+func archiveBase(host string, o hostclaim.Obligation) string {
+ return "parked/" + o.Park.Incarnation + "/" + host + "/" + o.Retirement + "/"
+}
+func destination(host string, o hostclaim.Obligation, v lease.ObjectVersion) string {
+ for _, p := range prefixes(host, o.Park.Location) {
+ if strings.HasPrefix(v.Key, p) {
+ kind := strings.SplitN(p, "/", 2)[0]
+ return archiveBase(host, o) + kind + "/" + strings.TrimPrefix(v.Key, p) + "@" + url.QueryEscape(v.VersionID)
+ }
+ }
+ return ""
+}
+func (e Engine) current(ctx context.Context, key string) (lease.ObjectVersion, bool, error) {
+ vs, err := e.Store.ListVersions(ctx, e.Bucket, key)
+ if err != nil {
+ return lease.ObjectVersion{}, false, err
+ }
+ var got lease.ObjectVersion
+ ok := false
+ for _, v := range vs {
+ if v.Key == key && v.IsLatest {
+ if ok || v.VersionID == "" {
+ return got, false, ErrUnsafe
+ }
+ got, ok = v, true
+ }
+ }
+ if ok && got.DeleteMarker {
+ return got, false, ErrUnsafe
+ }
+ return got, ok, nil
+}
+func (e Engine) manifest(ctx context.Context, w Work, o hostclaim.Obligation) (*Manifest, error) {
+ v, ok, err := e.current(ctx, archiveBase(w.Host, o)+"manifest.json")
+ if err != nil || !ok {
+ return nil, err
+ }
+ b, err := e.Store.GetVersion(ctx, e.Bucket, v.Key, v.VersionID)
+ if err != nil {
+ return nil, err
+ }
+ var m Manifest
+ d := json.NewDecoder(bytes.NewReader(b))
+ d.DisallowUnknownFields()
+ if err := d.Decode(&m); err != nil {
+ return nil, err
+ }
+ if _, err := d.Token(); err != io.EOF {
+ return nil, ErrUnsafe
+ }
+ if m.V != 1 || m.Host != w.Host || m.Obligation != w.ID || m.Op != w.Op || m.Retirement != o.Retirement || m.Region != o.Region || m.Location != o.Park.Location || m.Incarnation != o.Park.Incarnation || m.Generation != o.Generation {
+ return nil, ErrUnsafe
+ }
+ deletes := map[Ref]lease.ObjectVersion{}
+ for _, v := range m.Delete {
+ inScope, err := scopeContains(w.Host, o.Park.Location, v.Key)
+ if err != nil || !inScope || v.VersionID == "" || isLock(w.Host, o.Park.Location, v.Key) {
+ return nil, ErrUnsafe
+ }
+ if _, ok := deletes[ref(v)]; ok {
+ return nil, ErrUnsafe
+ }
+ deletes[ref(v)] = v
+ }
+ copies := map[Ref]bool{}
+ for _, a := range m.Archive {
+ v, ok := deletes[ref(a.Source)]
+ if !ok || v.DeleteMarker || !reflect.DeepEqual(v, a.Source) || copies[ref(v)] || a.Destination.Key != destination(w.Host, o, v) || a.Destination.VersionID == "" {
+ return nil, ErrUnsafe
+ }
+ copies[ref(v)] = true
+ }
+ for r, v := range deletes {
+ if !v.DeleteMarker && !copies[r] {
+ return nil, ErrUnsafe
+ }
+ }
+ return &m, nil
+}
+func (e Engine) observe(ctx context.Context, w Work, held HeldLease, expected hostclaim.Obligation) (hostclaim.Observed, View, error) {
+ if err := held.Check(ctx); err != nil {
+ return hostclaim.Observed{}, View{}, err
+ }
+ obs, err := hostclaim.Read(ctx, e.Claims, w.Host)
+ if err != nil {
+ return obs, View{}, err
+ }
+ o, ok := find(obs.Claim, w)
+ if !ok || !reflect.DeepEqual(o, expected) {
+ return obs, View{}, ErrChanged
+ }
+ if err := e.Authorize(ctx, obs, o); err != nil {
+ return obs, View{}, err
+ }
+ v, err := e.View(ctx, obs)
+ if err != nil {
+ return obs, v, err
+ }
+ if v.ClaimETag != obs.ETag || v.Region != e.Region {
+ return obs, v, ErrChanged
+ }
+ return obs, v, nil
+}
+func (e Engine) selected(ctx context.Context, w Work, o hostclaim.Obligation, held HeldLease, vs []lease.ObjectVersion) (hostclaim.Observed, View, []lease.ObjectVersion, error) {
+ obs, v, err := e.observe(ctx, w, held, o)
+ if err != nil {
+ return obs, v, nil, err
+ }
+ live, err := protected(obs.Claim, v, o.Park.Location)
+ if err != nil {
+ return obs, v, nil, err
+ }
+ out := make([]lease.ObjectVersion, 0, len(vs))
+ for _, x := range vs {
+ inScope, err := scopeContains(w.Host, o.Park.Location, x.Key)
+ if err != nil || !inScope || x.VersionID == "" {
+ return obs, v, nil, ErrUnsafe
+ }
+ if isLock(w.Host, o.Park.Location, x.Key) {
+ return obs, v, nil, ErrUnsafe
+ }
+ if obs.Claim.Holder != nil && obs.Claim.Op.Kind != hostclaim.KindParking && o.Park.Location == obs.Claim.Holder.Location && v.Region == v.AuthoritativeRegion {
+ if x.Key == v.Marker.Key {
+ if o.Kind == "park-generation" {
+ continue
+ }
+ return obs, v, nil, ErrUnsafe
+ }
+ g, ok := v.GenerationOf(ref(x))
+ if !ok {
+ return obs, v, nil, ErrUnsafe
+ }
+ g = generation(g)
+ if o.Kind == "park-generation" && g != o.Generation {
+ continue
+ }
+ if live[g] || v.Pinned[ref(x)] || ref(x) == v.Marker {
+ return obs, v, nil, ErrUnsafe
+ }
+ } else if o.Kind == "park-generation" {
+ if v.GenerationOf == nil {
+ return obs, v, nil, ErrUnsafe
+ }
+ g, ok := v.GenerationOf(ref(x))
+ if !ok {
+ return obs, v, nil, ErrUnsafe
+ }
+ if generation(g) != o.Generation {
+ continue
+ }
+ }
+ out = append(out, x)
+ }
+ return obs, v, out, nil
+}
+
+// Run performs one resumable event. ErrPending is a scheduler retry, never an
+// acknowledgement. A newly rotated event is persisted before returning it.
+func (e Engine) Run(ctx context.Context, w Work) error {
+ if e.Store == nil || e.Claims == nil || e.Leases == nil || e.Now == nil || e.NewID == nil || e.View == nil || e.Authorize == nil || e.Bucket == "" || e.Region == "" {
+ return ErrUnsafe
+ }
+ initial, err := hostclaim.Read(ctx, e.Claims, w.Host)
+ if err != nil {
+ return err
+ }
+ if initial.Absent {
+ return ErrChanged
+ }
+ o, ok := find(initial.Claim, w)
+ if !ok {
+ return nil
+ }
+ if err := validateScope(w.Host, o); err != nil {
+ return err
+ }
+ if o.Region != e.Region {
+ return ErrUnsafe
+ }
+ held, err := e.Leases.Acquire(ctx, e.Bucket, o.Park.Location)
+ if err != nil {
+ return err
+ }
+ defer held.Close()
+ if _, _, err = e.observe(ctx, w, held, o); err != nil {
+ return err
+ }
+ m, err := e.manifest(ctx, w, o)
+ if err != nil {
+ return err
+ }
+ if m == nil {
+ vs, err := e.list(ctx, w.Host, o.Park.Location)
+ if err != nil {
+ return err
+ }
+ _, _, vs, err = e.selected(ctx, w, o, held, vs)
+ if err != nil {
+ return err
+ }
+ m = &Manifest{V: 1, Host: w.Host, Obligation: w.ID, Op: w.Op, Retirement: o.Retirement, Region: o.Region, Location: o.Park.Location, Incarnation: o.Park.Incarnation, Generation: o.Generation, Delete: vs}
+ for _, v := range vs {
+ if v.DeleteMarker {
+ continue
+ }
+ if _, _, _, err := e.selected(ctx, w, o, held, []lease.ObjectVersion{v}); err != nil {
+ return err
+ }
+ dst := destination(w.Host, o, v)
+ id, _, err := e.Store.CopyVersion(ctx, e.Bucket, v.Key, v.VersionID, e.Bucket, dst, true)
+ if errors.Is(err, lease.ErrDestinationExists) {
+ got, ok, getErr := e.current(ctx, dst)
+ if getErr != nil {
+ return getErr
+ }
+ if !ok || got.Size != v.Size {
+ return ErrUnsafe
+ }
+ id, err = got.VersionID, nil
+ }
+ if err != nil {
+ return err
+ }
+ if id == "" {
+ return ErrUnsafe
+ }
+ m.Archive = append(m.Archive, Copy{v, Ref{dst, id}})
+ if err := e.after("copy"); err != nil {
+ return err
+ }
+ }
+ if _, _, _, err = e.selected(ctx, w, o, held, m.Delete); err != nil {
+ return err
+ }
+ raw, err := json.Marshal(m)
+ if err != nil {
+ return err
+ }
+ _, err = e.Store.PutIfAbsent(ctx, e.Bucket, archiveBase(w.Host, o)+"manifest.json", raw)
+ if errors.Is(err, lease.ErrExists) {
+ m, err = e.manifest(ctx, w, o)
+ }
+ if err != nil {
+ return err
+ }
+ if m == nil {
+ return ErrUnsafe
+ }
+ if err := e.after("manifest"); err != nil {
+ return err
+ }
+ }
+ // Existing manifests go directly to their recorded deletes: never re-copy
+ // vanished source versions. The live-set admission check still applies.
+ for _, v := range m.Delete {
+ _, _, selected, err := e.selected(ctx, w, o, held, []lease.ObjectVersion{v})
+ if err != nil {
+ return err
+ }
+ if len(selected) != 1 {
+ return ErrUnsafe
+ }
+ if err := e.Store.DeleteVersion(ctx, e.Bucket, v.Key, v.VersionID); err != nil {
+ return err
+ }
+ if err := e.after("delete"); err != nil {
+ return err
+ }
+ }
+ started := e.Now()
+ vs, err := e.list(ctx, w.Host, o.Park.Location)
+ if err != nil {
+ return err
+ }
+ obs, view, vs, err := e.selected(ctx, w, o, held, vs)
+ if err != nil {
+ return err
+ }
+ if err := e.after("verify"); err != nil {
+ return err
+ }
+ deadline := o.Park.EvidenceAt.Add(SignatureAge + o.Park.RequestLimit)
+ if !started.After(deadline) {
+ return ErrPending
+ }
+ generations, err := parkedGenerations(obs.Claim, view, []hostclaim.Obligation{o}, m.Delete)
+ if err != nil {
+ return err
+ }
+ cover := func(c *hostclaim.Claim) error {
+ if c.Holder == nil {
+ return nil
+ }
+ return hostclaim.CoverParkedPredecessors(obs.Claim.Op.ID, obs.Claim.Op.Phase, o.Region, generations)(c)
+ }
+ if len(vs) > 0 {
+ next := e.NewID()
+ if !token(next) || next == o.Retirement {
+ return ErrUnsafe
+ }
+ _, err = hostclaim.Update(ctx, e.Claims, obs, func(c *hostclaim.Claim) error {
+ if err := cover(c); err != nil {
+ return err
+ }
+ for i := range c.Obligations {
+ if c.Obligations[i].ID == w.ID && c.Obligations[i].Op == w.Op {
+ c.Obligations[i].Retirement = next
+ return nil
+ }
+ }
+ return ErrChanged
+ })
+ if err != nil {
+ return err
+ }
+ return ErrPending
+ }
+ if err := e.after("before-completion"); err != nil {
+ return err
+ }
+ if err := held.Check(ctx); err != nil {
+ return err
+ }
+ _, err = hostclaim.Update(ctx, e.Claims, obs, func(c *hostclaim.Claim) error {
+ if err := cover(c); err != nil {
+ return err
+ }
+ if c.QuietAfter == nil {
+ c.QuietAfter = map[string]time.Time{}
+ }
+ if c.QuietAfter[o.Region].Before(deadline) {
+ c.QuietAfter[o.Region] = deadline
+ }
+ if err := hostclaim.CompleteObligation(w.Op, w.ID)(c); err != nil {
+ return err
+ }
+ if c.Holder == nil && len(c.Obligations) == 0 && c.Op.Kind == hostclaim.KindParking && c.Op.Intent["park_sweep"] == "v1" {
+ return hostclaim.FreeAfterPark(c.Op.ID)(c)
+ }
+ return nil
+ })
+ if err != nil {
+ return err
+ }
+ if err := e.after("completion"); err != nil {
+ return err
+ }
+ return held.Release(ctx)
+}
diff --git a/internal/park/remote_test.go b/internal/park/remote_test.go
new file mode 100644
index 000000000..9d4f4c4a4
--- /dev/null
+++ b/internal/park/remote_test.go
@@ -0,0 +1,574 @@
+package park
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "reflect"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/lease"
+ "github.com/carlosframework/platform/internal/lease/leasetest"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+type testLease struct {
+ released int
+ failed bool
+}
+
+func (l *testLease) Check(context.Context) error {
+ if l.failed {
+ return ErrChanged
+ }
+ return nil
+}
+func (l *testLease) Release(context.Context) error { l.released++; return nil }
+func (*testLease) Close() {}
+
+type testLeases struct{ held *testLease }
+
+func (l testLeases) Acquire(context.Context, string, string) (HeldLease, error) { return l.held, nil }
+
+type fixture struct {
+ e Engine
+ b *leasetest.VersionedBucket
+ w Work
+ now time.Time
+ ob hostclaim.Obligation
+ held *testLease
+}
+
+func newFixture(t *testing.T) *fixture {
+ t.Helper()
+ ctx := t.Context()
+ now := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
+ host := "shop.bab.example.com"
+ inc, err := hostclaim.Incarnation("bab", "jam", host, now)
+ if err != nil {
+ t.Fatal(err)
+ }
+ b := leasetest.NewVersionedBucket()
+ b.AddBucket("data")
+ b.PageSize = 1
+ f := &fixture{b: b, w: Work{host, "ob", "op"}, now: now.Add(16 * time.Minute), held: &testLease{}}
+ f.ob = hostclaim.Obligation{ID: "ob", Op: "op", Kind: "park-replica", Region: "ie", Retirement: "rt1", Owner: "region:ie", Park: &hostclaim.ParkScope{Location: "instances/" + host + "/" + inc, Incarnation: inc, EvidenceAt: now, RequestLimit: time.Second}}
+ st := release.NewDirStore(t.TempDir())
+ _, err = hostclaim.Update(ctx, st, hostclaim.Observed{Host: host, Absent: true}, func(c *hostclaim.Claim) error {
+ c.Holder = &hostclaim.Holder{Family: "instance", Key: "apps/bab/jam/instances/" + host + ".json", Account: "bab", App: "jam", Incarnation: inc, CreatedAt: now, Location: f.ob.Park.Location, Stamp: 1}
+ c.Op = hostclaim.Op{Kind: hostclaim.KindParking, ID: "op"}
+ c.Obligations = []hostclaim.Obligation{f.ob}
+ c.Touched = hostclaim.Touched{Complete: true, Regions: []string{"ie"}}
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ serial := 1
+ f.e = Engine{Store: b, Claims: st, Bucket: "data", Region: "ie", Leases: testLeases{f.held}, Now: func() time.Time { return f.now }, NewID: func() string { serial++; return fmt.Sprintf("rt%d", serial) }, Authorize: func(context.Context, hostclaim.Observed, hostclaim.Obligation) error { return nil }, View: func(_ context.Context, o hostclaim.Observed) (View, error) {
+ return View{ClaimETag: o.ETag, Region: "ie", AuthoritativeRegion: "uk"}, nil
+ }}
+ return f
+}
+func (f *fixture) seed() []lease.ObjectVersion {
+ p := f.ob.Park.Location + "/"
+ f.b.Put("data", p+"0000/a", []byte("old"))
+ f.b.Put("data", p+"0000/a", []byte("new"))
+ f.b.BareDelete("data", p+"0000/a")
+ f.b.Put("data", p+"lock.json", []byte("lease"))
+ vs, _ := f.e.list(context.Background(), f.w.Host, f.ob.Park.Location)
+ return vs
+}
+func (f *fixture) done(t *testing.T) {
+ t.Helper()
+ obs, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(obs.Claim.Obligations) != 0 {
+ t.Fatal("obligation still open")
+ }
+ if got := obs.Claim.QuietAfter["ie"]; !got.Equal(f.ob.Park.EvidenceAt.Add(SignatureAge + time.Second)) {
+ t.Fatal(got)
+ }
+ vs, err := f.e.list(t.Context(), f.w.Host, f.ob.Park.Location)
+ if err != nil || len(vs) != 0 {
+ t.Fatalf("residue: %v %v", vs, err)
+ }
+}
+func TestRemoteCrashResume(t *testing.T) {
+ for _, tc := range []struct {
+ step string
+ n int
+ }{{"copy", 1}, {"copy", 2}, {"manifest", 1}, {"delete", 1}, {"delete", 2}, {"delete", 3}, {"verify", 1}, {"before-completion", 1}, {"completion", 1}} {
+ t.Run(fmt.Sprintf("%s-%d", tc.step, tc.n), func(t *testing.T) {
+ f := newFixture(t)
+ source := f.seed()
+ stop := errors.New("crash")
+ n := 0
+ f.e.After = func(s string) error {
+ if s == tc.step {
+ n++
+ if n == tc.n {
+ return stop
+ }
+ }
+ return nil
+ }
+ if err := f.e.Run(t.Context(), f.w); !errors.Is(err, stop) {
+ t.Fatalf("crash: %v", err)
+ }
+ if tc.step != "completion" && f.held.released != 0 {
+ t.Fatal("released before ack")
+ }
+ f.e.After = nil
+ if err := f.e.Run(t.Context(), f.w); err != nil {
+ t.Fatal(err)
+ }
+ f.done(t)
+ m, err := f.e.manifest(t.Context(), f.w, f.ob)
+ if err != nil || m == nil || len(m.Delete) != len(source) || len(m.Archive) != 2 {
+ t.Fatalf("manifest: %#v %v", m, err)
+ }
+ for _, a := range m.Archive {
+ body, err := f.b.GetVersion(t.Context(), "data", a.Destination.Key, a.Destination.VersionID)
+ if err != nil || int64(len(body)) != a.Source.Size {
+ t.Fatalf("copy: %q %v", body, err)
+ }
+ }
+ if f.b.LatestID("data", f.ob.Park.Location+"/lock.json") == "" {
+ t.Fatal("lock deleted")
+ }
+ })
+ }
+}
+func TestStorageFailuresKeepObligation(t *testing.T) {
+ for _, op := range []string{"list", "get", "copy", "put", "delete"} {
+ t.Run(op, func(t *testing.T) {
+ f := newFixture(t)
+ f.seed()
+ if op == "get" {
+ f.e.After = func(s string) error {
+ if s == "manifest" {
+ return ErrPending
+ }
+ return nil
+ }
+ _ = f.e.Run(t.Context(), f.w)
+ f.e.After = nil
+ }
+ failure := errors.New("injected")
+ f.b.FailOn = func(got, key string) error {
+ if got == op {
+ return failure
+ }
+ return nil
+ }
+ if err := f.e.Run(t.Context(), f.w); err == nil {
+ t.Fatal("failure acknowledged")
+ }
+ obs, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil || len(obs.Claim.Obligations) != 1 || len(obs.Claim.QuietAfter) != 0 {
+ t.Fatalf("claim: %v %v", obs, err)
+ }
+ f.b.FailOn = nil
+ if err := f.e.Run(t.Context(), f.w); err != nil {
+ t.Fatal(err)
+ }
+ f.done(t)
+ })
+ }
+}
+func TestPageFailureDoesNotAct(t *testing.T) {
+ f := newFixture(t)
+ f.seed()
+ before := f.b.Snapshot()
+ f.b.FailListAfterPages = 1
+ if err := f.e.Run(t.Context(), f.w); err == nil {
+ t.Fatal("short listing accepted")
+ }
+ if f.b.Snapshot() != before {
+ t.Fatal("mutated on listing failure")
+ }
+}
+func TestHorizonLatePut(t *testing.T) {
+ f := newFixture(t)
+ f.seed()
+ deadline := f.ob.Park.EvidenceAt.Add(SignatureAge + time.Second)
+ f.now = deadline
+ if err := f.e.Run(t.Context(), f.w); !errors.Is(err, ErrPending) {
+ t.Fatal(err)
+ }
+ late := f.b.Put("data", f.ob.Park.Location+"/0000/late", []byte("late"))
+ f.now = deadline.Add(time.Nanosecond)
+ if err := f.e.Run(t.Context(), f.w); !errors.Is(err, ErrPending) {
+ t.Fatal(err)
+ }
+ obs, _ := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if obs.Claim.Obligations[0].Retirement == f.ob.Retirement {
+ t.Fatal("manifest reused")
+ }
+ if err := f.e.Run(t.Context(), f.w); err != nil {
+ t.Fatal(err)
+ }
+ f.done(t)
+ if _, err := f.b.GetVersion(t.Context(), "data", f.ob.Park.Location+"/0000/late", late); !errors.Is(err, lease.ErrMirrorNotFound) {
+ t.Fatal(err)
+ }
+}
+func TestVerificationListingStartsAfterHorizon(t *testing.T) {
+ f := newFixture(t)
+ f.seed()
+ deadline := f.ob.Park.EvidenceAt.Add(SignatureAge + time.Second)
+ f.now = deadline
+ f.e.After = func(s string) error {
+ if s == "verify" {
+ f.now = deadline.Add(time.Hour)
+ }
+ return nil
+ }
+ if err := f.e.Run(t.Context(), f.w); !errors.Is(err, ErrPending) {
+ t.Fatal(err)
+ }
+}
+func TestManifestRetryDoesNotCopy(t *testing.T) {
+ f := newFixture(t)
+ f.seed()
+ f.e.After = func(s string) error {
+ if s == "delete" {
+ return ErrPending
+ }
+ return nil
+ }
+ _ = f.e.Run(t.Context(), f.w)
+ f.e.After = nil
+ f.b.FailOn = func(op, key string) error {
+ if op == "copy" {
+ t.Error("retry copied")
+ return ErrUnsafe
+ }
+ return nil
+ }
+ if err := f.e.Run(t.Context(), f.w); err != nil {
+ t.Fatal(err)
+ }
+}
+func TestCASConflictDoesNotAcknowledge(t *testing.T) {
+ f := newFixture(t)
+ f.seed()
+ f.e.After = func(s string) error {
+ if s == "before-completion" {
+ obs, _ := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ _, err := hostclaim.Update(t.Context(), f.e.Claims, obs, func(c *hostclaim.Claim) error { c.Lost = append(c.Lost, "other-box"); return nil })
+ return err
+ }
+ return nil
+ }
+ if err := f.e.Run(t.Context(), f.w); !errors.Is(err, hostclaim.ErrConflict) {
+ t.Fatal(err)
+ }
+ if f.held.released != 0 {
+ t.Fatal("released unacknowledged lease")
+ }
+ f.e.After = nil
+ if err := f.e.Run(t.Context(), f.w); err != nil {
+ t.Fatal(err)
+ }
+}
+func TestLiveGenerationCannotBeParked(t *testing.T) {
+ f := newFixture(t)
+ obs, _ := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ _, err := hostclaim.Update(t.Context(), f.e.Claims, obs, func(c *hostclaim.Claim) error {
+ c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box"}
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ f.seed()
+ f.e.View = func(_ context.Context, o hostclaim.Observed) (View, error) {
+ return View{ClaimETag: o.ETag, Region: "ie", AuthoritativeRegion: "ie", MarkerKnown: true, MarkerGeneration: hostclaim.RootGeneration, GenerationOf: func(Ref) (string, bool) { return hostclaim.RootGeneration, true }}, nil
+ }
+ before := f.b.Snapshot()
+ if err := f.e.Run(t.Context(), f.w); !errors.Is(err, ErrUnsafe) {
+ t.Fatal(err)
+ }
+ if before != f.b.Snapshot() {
+ t.Fatal("live data changed")
+ }
+}
+
+func TestRemoteMarkerDisagreementIsLocationScoped(t *testing.T) {
+ for _, scenario := range []string{"old-incarnation", "live-location", "ancestor-scope", "ancestor-holder", "unknown-region", "unknown-marker", "unknown-generations", "unknown-pinned-generation", "view-error", "authorization-error"} {
+ t.Run(scenario, func(t *testing.T) {
+ f, obs, view, _ := sweepFixture(t)
+ view.MarkerGeneration = "unrelated"
+ f.ob.Op = "session:3"
+ f.w.Op = f.ob.Op
+ if scenario != "live-location" {
+ f.ob.Park.Incarnation = strings.Repeat("a", 64)
+ f.ob.Park.Location = "instances/" + f.w.Host + "/" + f.ob.Park.Incarnation
+ }
+ if scenario == "ancestor-scope" {
+ f.ob.Park.Location = "instances/" + f.w.Host
+ }
+ _, err := hostclaim.Update(t.Context(), f.e.Claims, obs, func(c *hostclaim.Claim) error {
+ c.Obligations = []hostclaim.Obligation{f.ob}
+ if scenario == "ancestor-holder" {
+ c.Holder.Location = "instances/" + f.w.Host
+ }
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ source := f.seed()
+ before := f.b.Snapshot()
+ liveBefore, err := f.e.list(t.Context(), f.w.Host, obs.Claim.Holder.Location)
+ if err != nil {
+ t.Fatal(err)
+ }
+ switch scenario {
+ case "unknown-region":
+ view.AuthoritativeRegion = ""
+ case "unknown-marker":
+ view.MarkerKnown = false
+ case "unknown-generations":
+ view.GenerationOf = nil
+ case "unknown-pinned-generation":
+ view.GenerationOf = func(Ref) (string, bool) { return "", false }
+ }
+ f.e.View = func(_ context.Context, o hostclaim.Observed) (View, error) {
+ v := view
+ v.ClaimETag = o.ETag
+ if scenario == "view-error" {
+ return v, ErrUnsafe
+ }
+ return v, nil
+ }
+ authorized := false
+ f.e.Authorize = func(_ context.Context, _ hostclaim.Observed, o hostclaim.Obligation) error {
+ authorized = true
+ if !reflect.DeepEqual(o, f.ob) || scenario == "authorization-error" {
+ return ErrUnsafe
+ }
+ return nil
+ }
+ err = f.e.Run(t.Context(), f.w)
+ if !authorized {
+ t.Fatal("scope was not authorized")
+ }
+ if scenario != "old-incarnation" {
+ if !errors.Is(err, ErrUnsafe) {
+ t.Fatalf("Run = %v, want refusal", err)
+ }
+ if f.b.Snapshot() != before {
+ t.Fatal("refused work changed storage")
+ }
+ pending, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(pending.Claim.Obligations) != 1 || len(pending.Claim.QuietAfter) != 0 || f.held.released != 0 {
+ t.Fatal("refused work acknowledged")
+ }
+ return
+ }
+ if err != nil {
+ t.Fatalf("old incarnation refused by live marker disagreement: %v", err)
+ }
+ f.done(t)
+ m, err := f.e.manifest(t.Context(), f.w, f.ob)
+ if err != nil || m == nil || !reflect.DeepEqual(m.Delete, source) || len(m.Archive) != 2 {
+ t.Fatalf("old incarnation manifest: %+v %v", m, err)
+ }
+ for _, copy := range m.Archive {
+ body, err := f.b.GetVersion(t.Context(), "data", copy.Destination.Key, copy.Destination.VersionID)
+ if err != nil || (string(body) != "old" && string(body) != "new") {
+ t.Fatalf("archive: %q %v", body, err)
+ }
+ }
+ liveAfter, err := f.e.list(t.Context(), f.w.Host, obs.Claim.Holder.Location)
+ if err != nil || !reflect.DeepEqual(liveBefore, liveAfter) {
+ t.Fatalf("live location changed: %v", err)
+ }
+ if f.held.released != 1 {
+ t.Fatal("completed work did not release its lease")
+ }
+ })
+ }
+}
+
+func TestBoundScopeAndQuietAfter(t *testing.T) {
+ f := newFixture(t)
+ obs, _ := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ later := f.now.Add(time.Hour)
+ _, err := hostclaim.Update(t.Context(), f.e.Claims, obs, func(c *hostclaim.Claim) error {
+ c.Obligations[0].Park = nil
+ c.QuietAfter = map[string]time.Time{"ie": later}
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := f.e.Bind(t.Context(), f.w, *f.ob.Park); err != nil {
+ t.Fatal(err)
+ }
+ bad := *f.ob.Park
+ bad.EvidenceAt = bad.EvidenceAt.Add(time.Second)
+ if err := f.e.Bind(t.Context(), f.w, bad); !errors.Is(err, ErrChanged) {
+ t.Fatal(err)
+ }
+ if err := f.e.Run(t.Context(), f.w); err != nil {
+ t.Fatal(err)
+ }
+ obs, _ = hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if !obs.Claim.QuietAfter["ie"].Equal(later) {
+ t.Fatal("quiet_after regressed")
+ }
+}
+func TestBindSurvivesSerialization(t *testing.T) {
+ f := newFixture(t)
+ obs, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = hostclaim.Update(t.Context(), f.e.Claims, obs, func(c *hostclaim.Claim) error {
+ c.Obligations[0].Park = nil
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ scope := *f.ob.Park
+ scope.EvidenceAt = time.Now() // Carries a monotonic reading that JSON drops.
+ if err := f.e.Bind(t.Context(), f.w, scope); err != nil {
+ t.Fatal(err)
+ }
+ before, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, at := range []time.Time{scope.EvidenceAt, scope.EvidenceAt.UTC()} {
+ same := scope
+ same.EvidenceAt = at
+ if err := f.e.Bind(t.Context(), f.w, same); err != nil {
+ t.Fatal(err)
+ }
+ }
+ for _, change := range []func(*hostclaim.ParkScope){
+ func(p *hostclaim.ParkScope) { p.Location += "/other" },
+ func(p *hostclaim.ParkScope) { p.Incarnation = strings.Repeat("f", 64) },
+ func(p *hostclaim.ParkScope) { p.RequestLimit += time.Second },
+ func(p *hostclaim.ParkScope) { p.EvidenceAt = p.EvidenceAt.Add(time.Nanosecond) },
+ } {
+ bad := scope
+ change(&bad)
+ if err := f.e.Bind(t.Context(), f.w, bad); !errors.Is(err, ErrChanged) {
+ t.Fatal(err)
+ }
+ }
+ after, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil || after.ETag != before.ETag {
+ t.Fatalf("bind changed persisted scope: %v", err)
+ }
+}
+
+func TestLegacyLocationDoesNotIncludeIncarnationChildren(t *testing.T) {
+ f := newFixture(t)
+ key := f.ob.Park.Location + "/0000/data"
+ f.b.Put("data", key, []byte("new incarnation"))
+ vs, err := f.e.list(t.Context(), f.w.Host, "instances/"+f.w.Host)
+ if err != nil || len(vs) != 0 {
+ t.Fatalf("%v %v", vs, err)
+ }
+ for _, v := range []string{"a/b", "a@b", "a+b", "a b", "a?b"} {
+ key := destination(f.w.Host, f.ob, lease.ObjectVersion{Key: key, VersionID: v})
+ if strings.HasSuffix(key, "@"+v) {
+ t.Errorf("version not escaped: %s", v)
+ }
+ }
+}
+
+func TestDeleteParksStoppedAuthoritativeLocation(t *testing.T) {
+ f := newFixture(t)
+ f.seed()
+ f.e.View = func(_ context.Context, o hostclaim.Observed) (View, error) {
+ return View{ClaimETag: o.ETag, Region: "ie", AuthoritativeRegion: "ie"}, nil
+ }
+ if err := f.e.Run(t.Context(), f.w); err != nil {
+ t.Fatal(err)
+ }
+ f.done(t)
+}
+func TestForeignRootGenerationRetirement(t *testing.T) {
+ f := newFixture(t)
+ f.seed()
+ obs, _ := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ _, err := hostclaim.Update(t.Context(), f.e.Claims, obs, func(c *hostclaim.Claim) error {
+ c.Obligations[0].Kind = "park-generation"
+ c.Obligations[0].Generation = hostclaim.RootGeneration
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ f.e.View = func(_ context.Context, o hostclaim.Observed) (View, error) {
+ return View{ClaimETag: o.ETag, Region: "ie", AuthoritativeRegion: "uk", GenerationOf: func(Ref) (string, bool) { return "", true }}, nil
+ }
+ if err := f.e.Run(t.Context(), f.w); err != nil {
+ t.Fatal(err)
+ }
+ f.done(t)
+}
+
+func TestLeaseLossAndAuthorizationRefuse(t *testing.T) {
+ for _, phase := range []string{"before-first-effect", "before-completion", "authorize"} {
+ t.Run(phase, func(t *testing.T) {
+ f := newFixture(t)
+ f.seed()
+ before := f.b.Snapshot()
+ switch phase {
+ case "before-first-effect":
+ f.held.failed = true
+ case "before-completion":
+ f.e.After = func(s string) error {
+ if s == phase {
+ f.held.failed = true
+ }
+ return nil
+ }
+ case "authorize":
+ f.e.Authorize = func(context.Context, hostclaim.Observed, hostclaim.Obligation) error { return ErrChanged }
+ }
+ if err := f.e.Run(t.Context(), f.w); !errors.Is(err, ErrChanged) {
+ t.Fatal(err)
+ }
+ obs, _ := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if len(obs.Claim.Obligations) != 1 || f.held.released != 0 {
+ t.Fatal("acknowledged without authority")
+ }
+ if phase != "before-completion" && f.b.Snapshot() != before {
+ t.Fatal("acted without authority")
+ }
+ })
+ }
+}
+func TestMalformedManifestNeverDeletes(t *testing.T) {
+ f := newFixture(t)
+ f.seed()
+ _, err := f.b.PutIfAbsent(t.Context(), "data", archiveBase(f.w.Host, f.ob)+"manifest.json", []byte(`{"V":1,"Host":"other.example"}`))
+ if err != nil {
+ t.Fatal(err)
+ }
+ before := f.b.Snapshot()
+ if err := f.e.Run(t.Context(), f.w); err == nil {
+ t.Fatal("accepted malformed manifest")
+ }
+ if before != f.b.Snapshot() {
+ t.Fatal("deleted from bad manifest")
+ }
+}
diff --git a/internal/park/sweep.go b/internal/park/sweep.go
new file mode 100644
index 000000000..8acf33fb7
--- /dev/null
+++ b/internal/park/sweep.go
@@ -0,0 +1,319 @@
+package park
+
+import (
+ "context"
+ "fmt"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/lease"
+)
+
+// View is a moves-owned, completely read snapshot bound to one claim ETag.
+// GenerationOf is a pure classifier for any exact reference, including a
+// version already deleted by a previous attempt. It does no storage I/O.
+// Pinned includes the exact noncurrent versions needed by the installed root.
+// MarkerKnown means its read succeeded; an absent marker may denote @root.
+type View struct {
+ ClaimETag, Region, AuthoritativeRegion string
+ MarkerKnown bool
+ MarkerGeneration string
+ Marker Ref
+ Pinned map[Ref]bool
+ GenerationOf func(Ref) (string, bool)
+}
+type Alert struct{ Location, Reason string }
+type Candidate struct{ Location, Incarnation, Generation string }
+type Decision struct {
+ Candidates []Candidate
+ Alerts []Alert
+}
+
+func generation(s string) string {
+ if s == "" {
+ return hostclaim.RootGeneration
+ }
+ return s
+}
+
+func protected(c *hostclaim.Claim, v View, location string) (map[string]bool, error) {
+ live := map[string]bool{}
+ if c.Holder == nil || c.Op.Kind == hostclaim.KindParking {
+ return live, nil
+ }
+ if v.AuthoritativeRegion == "" {
+ return nil, ErrUnsafe
+ }
+ if v.Region != v.AuthoritativeRegion {
+ return live, nil
+ }
+ if !v.MarkerKnown || v.GenerationOf == nil {
+ return nil, ErrUnsafe
+ }
+ for r, on := range v.Pinned {
+ if on {
+ g, ok := v.GenerationOf(r)
+ if !ok {
+ return nil, ErrUnsafe
+ }
+ live[generation(g)] = true
+ }
+ }
+ h := c.Holder
+ if location != "" && location != h.Location && !strings.HasPrefix(location, h.Location+"/") && !strings.HasPrefix(h.Location, location+"/") {
+ return live, nil
+ }
+ current := generation(h.Generation)
+ marker := generation(v.MarkerGeneration)
+ live[current] = true
+ live[marker] = true
+ if current == marker {
+ return live, nil
+ }
+ // Walk the recorded chronological predecessor chain backwards. Warm
+ // admissions do not form cold unfinished links and cannot bridge a gap.
+ want, region := current, v.AuthoritativeRegion
+ for i := len(h.Admissions) - 1; i >= 0; i-- {
+ a := h.Admissions[i]
+ if generation(a.Generation) != want || a.Region != region {
+ continue
+ }
+ if a.Mode != "cold" {
+ return nil, ErrUnsafe
+ }
+ pred := generation(a.Predecessor)
+ if pred == want {
+ return nil, ErrUnsafe
+ }
+ want = pred
+ region = a.PredecessorRegion
+ if region == "" || region == v.AuthoritativeRegion {
+ if a.Covered {
+ return nil, ErrUnsafe
+ }
+ live[want] = true
+ }
+ if want == marker {
+ if region != "" && region != v.AuthoritativeRegion {
+ return nil, ErrUnsafe
+ }
+ return live, nil
+ }
+ }
+ return nil, ErrUnsafe
+}
+
+// Decide uses a complete namespace version listing supplied by Discover.
+// LegacyIncarnation is a caller-supplied, evidenced archive label only. It is
+// never used to discover locations, and its absence refuses legacy cleanup.
+func Decide(obs hostclaim.Observed, v View, versions []lease.ObjectVersion, legacyIncarnation string) (Decision, error) {
+ var d Decision
+ if obs.Absent || obs.Claim == nil || obs.ETag == "" || v.ClaimETag != obs.ETag || v.Region == "" {
+ return d, ErrUnsafe
+ }
+ c := obs.Claim
+ if err := c.Validate(); err != nil {
+ return d, err
+ }
+ joining := c.Holder == nil && c.Op.Kind == hostclaim.KindParking && c.Op.Intent["park_sweep"] == "v1"
+ if !c.Free() && c.Op.Kind != hostclaim.KindServing && !joining {
+ return d, nil
+ }
+ for _, o := range c.Obligations {
+ if o.Region == v.Region {
+ return d, nil
+ }
+ }
+ live, liveErr := protected(c, v, "")
+ if liveErr != nil && c.Holder != nil {
+ d.Alerts = append(d.Alerts, Alert{c.Holder.Location, "marker does not match the claim or an unfinished admission run"})
+ }
+ groups := map[Candidate]bool{}
+ seen := map[Ref]bool{}
+ for _, x := range versions {
+ loc, err := locationOf(c.Host, x.Key)
+ if err != nil || x.VersionID == "" || seen[ref(x)] {
+ return Decision{}, ErrUnsafe
+ }
+ if strings.HasPrefix(x.Key, "moves/"+c.Host+"/") {
+ loc = "moves/" + c.Host
+ }
+ seen[ref(x)] = true
+ if isLock(c.Host, loc, x.Key) {
+ continue
+ }
+ candidate := Candidate{Location: loc}
+ if tail := strings.TrimPrefix(loc, "instances/"+c.Host+"/"); tail != loc {
+ candidate.Incarnation = tail
+ } else {
+ candidate.Incarnation = legacyIncarnation
+ }
+ if c.Holder != nil && loc == c.Holder.Location && v.Region == v.AuthoritativeRegion {
+ if liveErr != nil {
+ continue
+ }
+ if x.Key == v.Marker.Key || v.Pinned[ref(x)] {
+ continue
+ }
+ g, ok := v.GenerationOf(ref(x))
+ if !ok {
+ return Decision{}, ErrUnsafe
+ }
+ g = generation(g)
+ if live[g] {
+ continue
+ }
+ candidate.Generation = g
+ candidate.Incarnation = c.Holder.Incarnation
+ }
+ if !hex64(candidate.Incarnation) {
+ return Decision{}, ErrUnsafe
+ }
+ groups[candidate] = true
+ }
+ for c := range groups {
+ d.Candidates = append(d.Candidates, c)
+ }
+ sort.Slice(d.Candidates, func(i, j int) bool {
+ a, b := d.Candidates[i], d.Candidates[j]
+ if a.Location == b.Location {
+ return a.Generation < b.Generation
+ }
+ return a.Location < b.Location
+ })
+ return d, nil
+}
+
+// Discover expands both namespaces across every version page. This is a
+// conservative superset of delimiter discovery: deriving first child prefixes
+// from the complete version listing also discovers marker-only locations.
+func (e Engine) Discover(ctx context.Context, host string) ([]lease.ObjectVersion, error) {
+ var out []lease.ObjectVersion
+ for _, p := range []string{"instances/" + host + "/", "moves/" + host + "/"} {
+ vs, err := e.Store.ListVersions(ctx, e.Bucket, p)
+ if err != nil {
+ return nil, err
+ }
+ out = append(out, vs...)
+ }
+ return out, nil
+}
+
+func parkedGenerations(c *hostclaim.Claim, v View, chores []hostclaim.Obligation, versions []lease.ObjectVersion) ([]string, error) {
+ if c.Holder == nil || len(c.Holder.Admissions) == 0 {
+ return nil, nil
+ }
+ seen := map[string]bool{}
+ for _, o := range chores {
+ if o.Park.Location != c.Holder.Location || o.Park.Incarnation != c.Holder.Incarnation {
+ continue
+ }
+ for _, x := range versions {
+ if !strings.HasPrefix(x.Key, c.Holder.Location+"/") || isLock(c.Host, o.Park.Location, x.Key) || x.Key == v.Marker.Key {
+ continue
+ }
+ inScope, err := scopeContains(c.Host, o.Park.Location, x.Key)
+ if err != nil {
+ return nil, err
+ }
+ if !inScope {
+ continue
+ }
+ if v.GenerationOf == nil {
+ return nil, ErrUnsafe
+ }
+ g, ok := v.GenerationOf(ref(x))
+ if !ok {
+ return nil, ErrUnsafe
+ }
+ g = generation(g)
+ if o.Kind != "park-generation" || o.Generation == g {
+ seen[g] = true
+ }
+ }
+ }
+ var generations []string
+ for g := range seen {
+ generations = append(generations, g)
+ }
+ sort.Strings(generations)
+ return generations, nil
+}
+
+// Sweep queues every decision in one claim CAS; Run is the common executor.
+// EvidenceAt is genuine prior stop/fence evidence for the residue. A sweep
+// must not substitute now for evidence, even on a free host.
+func (e Engine) Sweep(ctx context.Context, host, legacyIncarnation string, evidenceAt time.Time, requestLimit time.Duration) (Decision, error) {
+ var empty Decision
+ if e.View == nil || e.NewID == nil || e.Authorize == nil {
+ return empty, ErrUnsafe
+ }
+ obs, err := hostclaim.Read(ctx, e.Claims, host)
+ if err != nil {
+ return empty, err
+ }
+ v, err := e.View(ctx, obs)
+ if err != nil {
+ return empty, err
+ }
+ vs, err := e.Discover(ctx, host)
+ if err != nil {
+ return empty, err
+ }
+ d, err := Decide(obs, v, vs, legacyIncarnation)
+ if err != nil || len(d.Candidates) == 0 {
+ return d, err
+ }
+ var op string
+ if obs.Claim.Holder != nil {
+ op = fmt.Sprintf("session:%d", obs.Claim.Holder.Session)
+ } else if obs.Claim.Op.Kind == hostclaim.KindParking {
+ op = obs.Claim.Op.ID
+ } else {
+ op = e.NewID()
+ if !token(op) {
+ return d, ErrUnsafe
+ }
+ }
+ chores := make([]hostclaim.Obligation, 0, len(d.Candidates))
+ for _, x := range d.Candidates {
+ o := hostclaim.Obligation{ID: e.NewID(), Op: op, Kind: "park-replica", Region: e.Region, Generation: x.Generation, Retirement: e.NewID(), Owner: "region:" + e.Region, Park: &hostclaim.ParkScope{Location: x.Location, Incarnation: x.Incarnation, EvidenceAt: evidenceAt, RequestLimit: requestLimit}}
+ if x.Generation != "" {
+ o.Kind = "park-generation"
+ }
+ if !token(o.ID) || validateScope(host, o) != nil {
+ return d, ErrUnsafe
+ }
+ if err := e.Authorize(ctx, obs, o); err != nil {
+ return d, err
+ }
+ chores = append(chores, o)
+ }
+ generations, err := parkedGenerations(obs.Claim, v, chores, vs)
+ if err != nil {
+ return d, err
+ }
+ _, err = hostclaim.Update(ctx, e.Claims, obs, func(c *hostclaim.Claim) error {
+ if c.Holder != nil {
+ if err := hostclaim.CoverParkedPredecessors(obs.Claim.Op.ID, obs.Claim.Op.Phase, e.Region, generations)(c); err != nil {
+ return err
+ }
+ }
+ if c.Free() {
+ c.Op = hostclaim.Op{Kind: hostclaim.KindParking, ID: op, Intent: map[string]string{"park_sweep": "v1"}}
+ }
+ c.Obligations = append(c.Obligations, chores...)
+ return nil
+ })
+ if err != nil {
+ return d, err
+ }
+ for _, o := range chores {
+ if err := e.Run(ctx, Work{host, o.ID, o.Op}); err != nil {
+ return d, fmt.Errorf("sweep %s: %w", o.ID, err)
+ }
+ }
+ return d, nil
+}
diff --git a/internal/park/sweep_coverage_test.go b/internal/park/sweep_coverage_test.go
new file mode 100644
index 000000000..6fc63a407
--- /dev/null
+++ b/internal/park/sweep_coverage_test.go
@@ -0,0 +1,249 @@
+package park
+
+import (
+ "context"
+ "errors"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/hostclaim"
+)
+
+func coverageFixture(t *testing.T, foreign bool) *fixture {
+ t.Helper()
+ f := newFixture(t)
+ obs, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = hostclaim.Update(t.Context(), f.e.Claims, obs, func(c *hostclaim.Claim) error {
+ c.Obligations = nil
+ c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box"}
+ c.Holder.Generation = "live"
+ c.Holder.Session = 3
+ c.Holder.Admissions = []hostclaim.AdmissionRecord{
+ {Region: "uk", PredecessorRegion: "ie", Predecessor: "dead"},
+ {Region: "ie", PredecessorRegion: "uk", Predecessor: "dead"},
+ {Region: "ie", PredecessorRegion: "ie", Predecessor: "unseen"},
+ {Region: "ie", PredecessorRegion: "ie", Predecessor: "stage-only"},
+ {Region: "ie", PredecessorRegion: "ie", Predecessor: "old-only"},
+ {Region: "uk", PredecessorRegion: "ie", Predecessor: ""},
+ }
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ base := f.ob.Park.Location + "/"
+ for _, key := range []string{
+ base + "dead/data", base + "0000/data",
+ "moves/" + f.w.Host + "/stage-only/data",
+ "instances/" + f.w.Host + "/" + strings.Repeat("a", 64) + "/old-only/data",
+ } {
+ f.b.Put("data", key, []byte(key))
+ }
+ f.e.View = func(_ context.Context, o hostclaim.Observed) (View, error) {
+ authority := "ie"
+ if foreign {
+ authority = "uk"
+ }
+ return View{ClaimETag: o.ETag, Region: "ie", AuthoritativeRegion: authority,
+ MarkerKnown: true, MarkerGeneration: "live", GenerationOf: func(r Ref) (string, bool) {
+ parts := strings.Split(r.Key, "/")
+ g := parts[len(parts)-2]
+ if g == "0000" {
+ g = ""
+ }
+ return g, true
+ }}, nil
+ }
+ return f
+}
+
+func TestSweepQueueCoversOnlyQueuedPredecessors(t *testing.T) {
+ for _, foreign := range []bool{false, true} {
+ name := "generation"
+ if foreign {
+ name = "replica"
+ }
+ t.Run(name, func(t *testing.T) {
+ f := coverageFixture(t, foreign)
+ before := f.b.Snapshot()
+ f.held.failed = true
+ if _, err := f.e.Sweep(t.Context(), f.w.Host, f.ob.Park.Incarnation, f.ob.Park.EvidenceAt, time.Second); !errors.Is(err, ErrChanged) {
+ t.Fatal(err)
+ }
+ obs, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(obs.Claim.Obligations) == 0 || f.b.Snapshot() != before {
+ t.Fatal("queue boundary did not preserve pending work and storage")
+ }
+ for i, a := range obs.Claim.Holder.Admissions {
+ if want := i == 0 || i == 5; a.Covered != want {
+ t.Errorf("queue admission %d covered=%v, want %v", i, a.Covered, want)
+ }
+ }
+ })
+ }
+}
+
+func TestSweepAcknowledgementCoversOnlyRetiredPredecessors(t *testing.T) {
+ for _, scenario := range []string{"generation", "replica", "empty", "old-incarnation", "staging"} {
+ t.Run(scenario, func(t *testing.T) {
+ f := coverageFixture(t, scenario == "replica")
+ f.held.failed = true
+ if _, err := f.e.Sweep(t.Context(), f.w.Host, f.ob.Park.Incarnation, f.ob.Park.EvidenceAt, time.Second); !errors.Is(err, ErrChanged) {
+ t.Fatal(err)
+ }
+ obs, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var chore hostclaim.Obligation
+ for _, o := range obs.Claim.Obligations {
+ match := o.Park.Location == f.ob.Park.Location && (o.Generation == "dead" || scenario == "replica")
+ if scenario == "old-incarnation" {
+ match = strings.HasSuffix(o.Park.Location, strings.Repeat("a", 64))
+ } else if scenario == "staging" {
+ match = strings.HasPrefix(o.Park.Location, "moves/")
+ }
+ if match {
+ chore = o
+ break
+ }
+ }
+ if chore.ID == "" {
+ t.Fatal("target chore was not queued")
+ }
+ if scenario == "empty" {
+ versions, err := f.e.list(t.Context(), f.w.Host, chore.Park.Location)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, x := range versions {
+ if strings.Contains(x.Key, "/dead/") {
+ if err := f.b.DeleteVersion(t.Context(), "data", x.Key, x.VersionID); err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+ }
+ _, err = hostclaim.Update(t.Context(), f.e.Claims, obs, func(c *hostclaim.Claim) error {
+ later := append([]hostclaim.AdmissionRecord(nil), c.Holder.Admissions...)
+ for i := range later {
+ later[i].Covered = false
+ }
+ c.Holder.Admissions = append(c.Holder.Admissions, later...)
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ f.held.failed = false
+ f.e.After = func(step string) error {
+ if step == "before-completion" {
+ return ErrPending
+ }
+ return nil
+ }
+ w := Work{f.w.Host, chore.ID, chore.Op}
+ if err := f.e.Run(t.Context(), w); !errors.Is(err, ErrPending) {
+ t.Fatal(err)
+ }
+ pending, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, a := range pending.Claim.Holder.Admissions[6:] {
+ if a.Covered {
+ t.Fatal("later admission covered before acknowledgement")
+ }
+ }
+ f.e.After = nil
+ f.b.FailOn = func(op, key string) error {
+ if op == "copy" {
+ t.Error("completion retry copied again")
+ return ErrUnsafe
+ }
+ return nil
+ }
+ if err := f.e.Run(t.Context(), w); err != nil {
+ t.Fatal(err)
+ }
+ completed, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, present := find(completed.Claim, w); present || f.held.released != 1 || completed.Claim.QuietAfter["ie"].IsZero() {
+ t.Fatal("retirement was not acknowledged")
+ }
+ for i, a := range completed.Claim.Holder.Admissions[6:] {
+ want := (i == 0 && (scenario == "generation" || scenario == "replica")) || (i == 5 && scenario == "replica")
+ if a.Covered != want {
+ t.Errorf("ack admission %d covered=%v, want %v", i, a.Covered, want)
+ }
+ }
+ })
+ }
+}
+
+func TestSweepCoverageSurvivesRetirementRotation(t *testing.T) {
+ f := coverageFixture(t, true)
+ obs, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = hostclaim.Update(t.Context(), f.e.Claims, obs, func(c *hostclaim.Claim) error {
+ c.Holder.Admissions = append(c.Holder.Admissions, hostclaim.AdmissionRecord{Region: "uk", PredecessorRegion: "ie", Predecessor: "late"})
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ f.held.failed = true
+ if _, err := f.e.Sweep(t.Context(), f.w.Host, f.ob.Park.Incarnation, f.ob.Park.EvidenceAt, time.Second); !errors.Is(err, ErrChanged) {
+ t.Fatal(err)
+ }
+ obs, err = hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var w Work
+ for _, o := range obs.Claim.Obligations {
+ if o.Park.Location == f.ob.Park.Location {
+ w = Work{f.w.Host, o.ID, o.Op}
+ }
+ }
+ if w.ID == "" || obs.Claim.Holder.Admissions[6].Covered {
+ t.Fatal("late generation was queued or covered without an object")
+ }
+ f.held.failed = false
+ for _, next := range []string{"late", "last"} {
+ f.e.After = func(step string) error {
+ if step == "manifest" {
+ f.b.Put("data", f.ob.Park.Location+"/"+next+"/data", []byte(next))
+ }
+ return nil
+ }
+ if err := f.e.Run(t.Context(), w); !errors.Is(err, ErrPending) {
+ t.Fatalf("retirement rotation: %v", err)
+ }
+ }
+ f.e.After = nil
+ if err := f.e.Run(t.Context(), w); err != nil {
+ t.Fatal(err)
+ }
+ completed, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !completed.Claim.Holder.Admissions[6].Covered || completed.Claim.Holder.Admissions[2].Covered {
+ t.Fatalf("coverage lost a retired generation or covered unseen data: %+v", completed.Claim.Holder.Admissions)
+ }
+ if _, present := find(completed.Claim, w); present {
+ t.Fatal("rotated retirement did not complete")
+ }
+}
diff --git a/internal/park/sweep_test.go b/internal/park/sweep_test.go
new file mode 100644
index 000000000..0bcd3aaeb
--- /dev/null
+++ b/internal/park/sweep_test.go
@@ -0,0 +1,390 @@
+package park
+
+import (
+ "context"
+ "errors"
+ "reflect"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/lease"
+ "github.com/carlosframework/platform/internal/lease/leasetest"
+)
+
+func sweepFixture(t *testing.T) (*fixture, hostclaim.Observed, View, []lease.ObjectVersion) {
+ f := newFixture(t)
+ obs, _ := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ obs, err := hostclaim.Update(t.Context(), f.e.Claims, obs, func(c *hostclaim.Claim) error {
+ c.Obligations = nil
+ c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box"}
+ c.Holder.Generation = "g3"
+ c.Holder.Session = 3
+ c.Holder.Admissions = []hostclaim.AdmissionRecord{
+ {Session: 2, Generation: "g2", Predecessor: "g1", Mode: "cold", Region: "ie", PredecessorRegion: "ie"},
+ {Session: 3, Generation: "g3", Predecessor: "g2", Mode: "cold", Region: "ie", PredecessorRegion: "ie"},
+ }
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ p := f.ob.Park.Location + "/"
+ for _, g := range []string{"g0", "g1", "g2", "g3"} {
+ f.b.Put("data", p+g+"/data", []byte(g))
+ }
+ f.b.Put("data", p+"generation.json", []byte("older"))
+ markerID := f.b.Put("data", p+"generation.json", []byte("g1"))
+ pinned := f.b.Put("data", p+"pinned/data", []byte("baseline"))
+ f.b.Put("data", p+"pinned/data", []byte("newer"))
+ classify := func(r Ref) (string, bool) {
+ s := strings.TrimPrefix(r.Key, p)
+ g, _, ok := strings.Cut(s, "/")
+ if r.Key == p+"generation.json" {
+ return "@marker", true
+ }
+ return g, ok
+ }
+ v := View{ClaimETag: obs.ETag, Region: "ie", AuthoritativeRegion: "ie", MarkerKnown: true, MarkerGeneration: "g1", Marker: Ref{p + "generation.json", markerID}, Pinned: map[Ref]bool{{p + "pinned/data", pinned}: true}, GenerationOf: classify}
+ vs, err := f.e.Discover(t.Context(), f.w.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return f, obs, v, vs
+}
+
+func TestSweepMovesNamespaceIsSeparateFromLiveInstance(t *testing.T) {
+ f := newFixture(t)
+ obs, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = hostclaim.Update(t.Context(), f.e.Claims, obs, func(c *hostclaim.Claim) error {
+ c.Obligations = nil
+ c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box"}
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ liveKey := f.ob.Park.Location + "/0000/live"
+ liveID := f.b.Put("data", liveKey, []byte("live"))
+ for _, op := range []string{"move-1", f.ob.Park.Incarnation} {
+ f.b.Put("data", "moves/"+f.w.Host+"/"+op+"/staging", []byte(op))
+ }
+ f.e.View = func(_ context.Context, o hostclaim.Observed) (View, error) {
+ return View{ClaimETag: o.ETag, Region: "ie", AuthoritativeRegion: "ie", MarkerKnown: true, MarkerGeneration: hostclaim.RootGeneration, GenerationOf: func(Ref) (string, bool) { return hostclaim.RootGeneration, true }}, nil
+ }
+ d, err := f.e.Sweep(t.Context(), f.w.Host, f.ob.Park.Incarnation, f.ob.Park.EvidenceAt, f.ob.Park.RequestLimit)
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := []Candidate{{Location: "moves/" + f.w.Host, Incarnation: f.ob.Park.Incarnation}}
+ if !reflect.DeepEqual(d.Candidates, want) {
+ t.Fatalf("staging candidates=%+v, want %+v", d.Candidates, want)
+ }
+ vs, err := f.b.ListVersions(t.Context(), "data", "moves/"+f.w.Host+"/")
+ if err != nil || len(vs) != 0 {
+ t.Fatalf("staging residue: %+v %v", vs, err)
+ }
+ body, err := f.b.GetVersion(t.Context(), "data", liveKey, liveID)
+ if err != nil || string(body) != "live" {
+ t.Fatalf("live instance changed: %q %v", body, err)
+ }
+}
+
+func TestSweepJoinsRunningHolderlessOperation(t *testing.T) {
+ f := newFixture(t)
+ f.seed()
+ obs, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = hostclaim.Update(t.Context(), f.e.Claims, obs, func(c *hostclaim.Claim) error {
+ c.Obligations = nil
+ return hostclaim.FreeAfterPark("op")(c)
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ f.held.failed = true
+ if _, err := f.e.Sweep(t.Context(), f.w.Host, f.ob.Park.Incarnation, f.ob.Park.EvidenceAt, f.ob.Park.RequestLimit); !errors.Is(err, ErrChanged) {
+ t.Fatalf("first region queue: %v", err)
+ }
+ obs, err = hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ first := obs.Claim.Obligations[0]
+ second := f.e
+ second.Region = "uk"
+ b := leasetest.NewVersionedBucket()
+ b.AddBucket("data")
+ b.Put("data", f.ob.Park.Location+"/0000/data", []byte("second region"))
+ second.Store = b
+ second.View = func(_ context.Context, o hostclaim.Observed) (View, error) {
+ return View{ClaimETag: o.ETag, Region: "uk"}, nil
+ }
+ second.Authorize = func(_ context.Context, _ hostclaim.Observed, o hostclaim.Obligation) error {
+ if o.Op != first.Op || o.Region != "uk" {
+ return ErrUnsafe
+ }
+ return nil
+ }
+ if _, err := second.Sweep(t.Context(), f.w.Host, f.ob.Park.Incarnation, f.ob.Park.EvidenceAt, f.ob.Park.RequestLimit); !errors.Is(err, ErrChanged) {
+ t.Fatalf("second region did not join the running sweep: %v", err)
+ }
+ obs, err = hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if obs.Claim.Holder != nil || obs.Claim.Op.ID != first.Op || len(obs.Claim.Obligations) != 2 || !reflect.DeepEqual(obs.Claim.Obligations[0], first) {
+ t.Fatalf("joined sweep: %+v", obs.Claim)
+ }
+ joined := obs.Claim.Obligations[1]
+ f.held.failed = false
+ if err := f.e.Run(t.Context(), Work{f.w.Host, first.ID, first.Op}); err != nil {
+ t.Fatal(err)
+ }
+ obs, err = hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil || obs.Claim.Free() || len(obs.Claim.Obligations) != 1 {
+ t.Fatalf("first completion freed an outstanding sweep: %+v %v", obs.Claim, err)
+ }
+ if err := second.Run(t.Context(), Work{f.w.Host, joined.ID, joined.Op}); err != nil {
+ t.Fatal(err)
+ }
+ obs, err = hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil || !obs.Claim.Free() {
+ t.Fatalf("last completion did not free the sweep: %+v %v", obs.Claim, err)
+ }
+ deadline := f.ob.Park.EvidenceAt.Add(SignatureAge + f.ob.Park.RequestLimit)
+ for _, region := range []string{"ie", "uk"} {
+ if !obs.Claim.QuietAfter[region].Equal(deadline) {
+ t.Fatalf("quiet_after[%s]=%v, want %v", region, obs.Claim.QuietAfter[region], deadline)
+ }
+ }
+}
+func TestSweepProtectsUnfinishedRunAndPins(t *testing.T) {
+ f, obs, v, vs := sweepFixture(t)
+ d, err := Decide(obs, v, vs, f.ob.Park.Incarnation)
+ if err != nil || len(d.Alerts) != 0 || len(d.Candidates) != 1 || d.Candidates[0].Generation != "g0" {
+ t.Fatalf("%+v %v", d, err)
+ }
+ f.e.View = func(_ context.Context, o hostclaim.Observed) (View, error) {
+ x := v
+ x.ClaimETag = o.ETag
+ return x, nil
+ }
+ if _, err := f.e.Sweep(t.Context(), f.w.Host, f.ob.Park.Incarnation, f.ob.Park.EvidenceAt, time.Second); err != nil {
+ t.Fatal(err)
+ }
+ for _, x := range vs {
+ if strings.Contains(x.Key, "/g0/") {
+ continue
+ }
+ if _, err := f.b.GetVersion(t.Context(), "data", x.Key, x.VersionID); err != nil {
+ t.Fatalf("live version lost: %+v %v", x, err)
+ }
+ }
+}
+
+func TestSweepUnfinishedRunRegionalCoverage(t *testing.T) {
+ for _, scenario := range []string{"covered-foreign", "uncovered-foreign", "covered-local"} {
+ t.Run(scenario, func(t *testing.T) {
+ f, obs, v, _ := sweepFixture(t)
+ _, err := hostclaim.Update(t.Context(), f.e.Claims, obs, func(c *hostclaim.Claim) error {
+ c.Holder.Admissions = []hostclaim.AdmissionRecord{
+ {Generation: "T", Predecessor: "g1", Region: "us", PredecessorRegion: "ie", Mode: "cold", Covered: scenario == "covered-local"},
+ {Generation: "U", Predecessor: "T", Region: "uk", PredecessorRegion: "us", Mode: "cold", Covered: scenario == "covered-foreign"},
+ {Generation: "g3", Predecessor: "U", Region: "ie", PredecessorRegion: "uk", Mode: "cold", Covered: scenario == "covered-foreign"},
+ }
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, g := range []string{"T", "U"} {
+ f.b.Put("data", f.ob.Park.Location+"/"+g+"/data", []byte(g))
+ }
+ before, err := f.e.Discover(t.Context(), f.w.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ f.e.View = func(_ context.Context, o hostclaim.Observed) (View, error) {
+ x := v
+ x.ClaimETag = o.ETag
+ return x, nil
+ }
+ snapshot := f.b.Snapshot()
+ d, err := f.e.Sweep(t.Context(), f.w.Host, f.ob.Park.Incarnation, f.ob.Park.EvidenceAt, time.Second)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if scenario == "covered-local" {
+ if len(d.Alerts) != 1 || len(d.Candidates) != 0 || f.b.Snapshot() != snapshot {
+ t.Fatalf("covered local baseline did not refuse unchanged: %+v", d)
+ }
+ return
+ }
+ var got []string
+ for _, c := range d.Candidates {
+ got = append(got, c.Generation)
+ }
+ if len(d.Alerts) != 0 || !reflect.DeepEqual(got, []string{"T", "U", "g0", "g2"}) {
+ t.Fatalf("foreign links blocked sweep or were protected: %+v", d)
+ }
+ for _, x := range before {
+ g, _ := v.GenerationOf(ref(x))
+ _, err := f.b.GetVersion(t.Context(), "data", x.Key, x.VersionID)
+ if g == "T" || g == "U" || g == "g0" || g == "g2" {
+ if !errors.Is(err, lease.ErrMirrorNotFound) {
+ t.Fatalf("dead version retained: %+v %v", x, err)
+ }
+ } else if err != nil {
+ t.Fatalf("live version lost: %+v %v", x, err)
+ }
+ }
+ completed, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil || len(completed.Claim.Obligations) != 0 {
+ t.Fatalf("sweep did not complete: %+v %v", completed, err)
+ }
+ })
+ }
+}
+func TestSweepDisagreementAndBusyRefuse(t *testing.T) {
+ f, obs, v, vs := sweepFixture(t)
+ v.MarkerGeneration = "unrelated"
+ d, err := Decide(obs, v, vs, f.ob.Park.Incarnation)
+ if err != nil || len(d.Alerts) != 1 || len(d.Candidates) != 0 {
+ t.Fatalf("%+v %v", d, err)
+ }
+ obs.Claim.Op.Kind = hostclaim.KindMoving
+ obs.Claim.Op.ID = "moving"
+ d, err = Decide(obs, v, vs, f.ob.Park.Incarnation)
+ if err != nil || len(d.Candidates) != 0 {
+ t.Fatalf("%+v %v", d, err)
+ }
+}
+func TestSweepDiscoversOldNamespaceAndMoves(t *testing.T) {
+ f, obs, v, vs := sweepFixture(t)
+ old := strings.Repeat("a", 64)
+ other := strings.Repeat("b", 64)
+ for _, key := range []string{"instances/" + f.w.Host + "/" + old + "/0000/a", "moves/" + f.w.Host + "/" + other + "/g-op/a", "instances/" + f.w.Host + "/0000/legacy", "moves/" + f.w.Host + "/g-old/staging"} {
+ id := f.b.Put("data", key, []byte("old"))
+ vs = append(vs, lease.ObjectVersion{Key: key, VersionID: id})
+ }
+ d, err := Decide(obs, v, vs, f.ob.Park.Incarnation)
+ if err != nil || len(d.Candidates) != 4 {
+ t.Fatalf("%+v %v", d, err)
+ }
+ // Capped retired history is irrelevant to discovery.
+ obs.Claim.RetiredIncarnations = nil
+ v.Region = "us"
+ d, err = Decide(obs, v, vs, f.ob.Park.Incarnation)
+ if err != nil || len(d.Candidates) != 4 {
+ t.Fatalf("foreign location not residue: %+v %v", d, err)
+ }
+}
+func TestSweepFreeHostQueuesAndAcknowledges(t *testing.T) {
+ f := newFixture(t)
+ f.seed()
+ obs, _ := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ _, err := hostclaim.Update(t.Context(), f.e.Claims, obs, func(c *hostclaim.Claim) error { c.Obligations = nil; return hostclaim.FreeAfterPark("op")(c) })
+ if err != nil {
+ t.Fatal(err)
+ }
+ d, err := f.e.Sweep(t.Context(), f.w.Host, f.ob.Park.Incarnation, f.ob.Park.EvidenceAt, time.Second)
+ if err != nil || len(d.Candidates) != 1 {
+ t.Fatalf("%+v %v", d, err)
+ }
+ obs, err = hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil || !obs.Claim.Free() || len(obs.Claim.Obligations) != 0 {
+ t.Fatalf("%+v %v", obs, err)
+ }
+}
+func TestSweepOperationOwnership(t *testing.T) {
+ for _, serving := range []bool{true, false} {
+ name := "holderless"
+ if serving {
+ name = "serving"
+ }
+ t.Run(name, func(t *testing.T) {
+ f, obs, v, _ := sweepFixture(t)
+ if !serving {
+ _, err := hostclaim.Update(t.Context(), f.e.Claims, obs, func(c *hostclaim.Claim) error {
+ c.Op = hostclaim.Op{Kind: hostclaim.KindParking, ID: "free"}
+ return hostclaim.FreeAfterPark("free")(c)
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+ f.e.View = func(_ context.Context, o hostclaim.Observed) (View, error) {
+ x := v
+ x.ClaimETag = o.ETag
+ return x, nil
+ }
+ ids := []string{"generated-op", "chore", "retirement"}
+ wantOp, wantCalls := "generated-op", 3
+ if serving {
+ ids, wantOp, wantCalls = ids[1:], "session:3", 2
+ }
+ calls := 0
+ f.e.NewID = func() string {
+ if calls >= len(ids) {
+ t.Fatal("unexpected generated operation id")
+ }
+ id := ids[calls]
+ calls++
+ return id
+ }
+ authorized := false
+ f.e.Authorize = func(_ context.Context, _ hostclaim.Observed, o hostclaim.Obligation) error {
+ authorized = true
+ if o.Op != wantOp {
+ return ErrUnsafe
+ }
+ return nil
+ }
+ // Stop after queueing, before Run can acknowledge the persisted chore.
+ f.held.failed = true
+ if _, err := f.e.Sweep(t.Context(), f.w.Host, f.ob.Park.Incarnation, f.ob.Park.EvidenceAt, time.Second); !errors.Is(err, ErrChanged) {
+ t.Fatal(err)
+ }
+ if !authorized || calls != wantCalls {
+ t.Fatalf("authorized=%v ids=%d", authorized, calls)
+ }
+ queued, err := hostclaim.Read(t.Context(), f.e.Claims, f.w.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(queued.Claim.Obligations) != 1 || queued.Claim.Obligations[0].Op != wantOp {
+ t.Fatalf("wrong chore owner: %+v", queued.Claim)
+ }
+ if serving {
+ if queued.Claim.Op.Kind != hostclaim.KindServing || queued.Claim.Holder.Session != 3 {
+ t.Fatal("serving state changed")
+ }
+ } else if queued.Claim.Holder != nil || queued.Claim.Op.ID != wantOp || queued.Claim.Op.Intent["park_sweep"] != "v1" {
+ t.Fatal("wrong holderless owner")
+ }
+ })
+ }
+}
+
+func TestSweepPageFailureAndStaleView(t *testing.T) {
+ f, obs, v, vs := sweepFixture(t)
+ before := f.b.Snapshot()
+ f.b.FailListAfterPages = 1
+ if _, err := f.e.Sweep(t.Context(), f.w.Host, f.ob.Park.Incarnation, f.ob.Park.EvidenceAt, time.Second); err == nil {
+ t.Fatal("short sweep acted")
+ }
+ if before != f.b.Snapshot() {
+ t.Fatal("mutation")
+ }
+ v.ClaimETag = "old"
+ if _, err := Decide(obs, v, vs, f.ob.Park.Incarnation); !errors.Is(err, ErrUnsafe) {
+ t.Fatal(err)
+ }
+}