| new file mode 100644 |
| index 000000000..b6c3153b0 |
| --- /dev/null |
| +++ b/internal/host/movesevidence_test.go |
| @@ -0,0 +1,897 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "fmt" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/hostbox" |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + "github.com/carlosframework/platform/internal/hostops" |
| + "github.com/carlosframework/platform/internal/lease" |
| + "github.com/carlosframework/platform/internal/lease/leasetest" |
| + "github.com/carlosframework/platform/internal/park" |
| + "github.com/carlosframework/platform/internal/provision" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// The moves retirement evidence provider (Plan 5 Task 5, rulings K10, M7, |
| +// M8). The fake bucket's clock starts at 2026-10-05T00:00:00Z and ticks a |
| +// second per write, so every object a fixture writes is older than evAt. |
| +var ( |
| + evCreated = time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) |
| + evAt = time.Date(2026, 10, 5, 1, 0, 0, 0, time.UTC) // LastMove.At, the close |
| + evNow = time.Date(2026, 10, 5, 2, 0, 0, 0, time.UTC) // the sweep's clock |
| +) |
| + |
| +const ( |
| + evHost = "ev.example" |
| + evAcct = "acct" |
| + evApp = "app" |
| + evMoveID = "mv_retired" |
| +) |
| + |
| +type evFixture struct { |
| + inc, location string |
| + claims release.Store |
| + bucket *leasetest.VersionedBucket |
| + rec *provision.Record |
| + reads int |
| +} |
| + |
| +// newEvFixture is a serving claim released by evMoveID, whose record closed |
| +// that move at evAt and carries no live move. |
| +func newEvFixture(t *testing.T) *evFixture { |
| + t.Helper() |
| + inc, err := hostclaim.Incarnation(evAcct, evApp, evHost, evCreated) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + f := &evFixture{inc: inc, location: "instances/" + evHost + "/" + inc, claims: release.NewDirStore(t.TempDir()), bucket: leasetest.NewVersionedBucket()} |
| + f.bucket.AddBucket("data") |
| + f.rec = &provision.Record{Schema: 1, Host: evHost, App: evApp, Account: evAcct, CreatedAt: evCreated, |
| + LastMove: &provision.MoveOutcome{ID: evMoveID, Outcome: "done", At: evAt}} |
| + f.setClaim(t, func(c *hostclaim.Claim) { |
| + c.Holder = &hostclaim.Holder{Family: "instance", Key: provision.RecordKey(evAcct, evApp, evHost), Account: evAcct, App: evApp, Incarnation: inc, CreatedAt: evCreated, Location: f.location, Stamp: 2, Session: 2, WriterBox: "box", Generation: evMoveID, |
| + Admissions: evAdmissions(evMoveID)} |
| + c.Op = servingReleasedBy(evMoveID) |
| + c.Touched = hostclaim.Touched{Complete: true, Regions: []string{"ie"}} |
| + }) |
| + return f |
| +} |
| + |
| +// evAdmissions is the holder's admission history: an origin generation |
| +// "old", then the move's prepared landing, which admits the generation |
| +// named after the move (movesprobeowners.go), then any later generations, |
| +// each the next one's predecessor. The last is the holder's current. |
| +func evAdmissions(gens ...string) []hostclaim.AdmissionRecord { |
| + out := []hostclaim.AdmissionRecord{{Session: 1, Box: "box", Generation: "old", Mode: "cold", Region: "ie"}} |
| + for i, g := range gens { |
| + out = append(out, hostclaim.AdmissionRecord{Session: uint64(i + 2), Box: "box", Generation: g, Predecessor: out[i].Generation, Mode: "cold", Region: "ie", PredecessorRegion: "ie"}) |
| + } |
| + return out |
| +} |
| + |
| +// genKey is an object of generation gen at the fixture's live location. |
| +func (f *evFixture) genKey(gen string) string { return f.location + "/g-" + gen + "/0000/db" } |
| + |
| +// servingReleasedBy is the serving op hostclaim.ReleaseMove leaves. |
| +func servingReleasedBy(id string) hostclaim.Op { |
| + return hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box", Journal: map[string]string{"released_move": id}} |
| +} |
| + |
| +func (f *evFixture) setClaim(t *testing.T, mut func(*hostclaim.Claim)) { |
| + t.Helper() |
| + obs, err := hostclaim.Read(t.Context(), f.claims, evHost) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := hostclaim.Update(t.Context(), f.claims, obs, func(c *hostclaim.Claim) error { mut(c); return nil }); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +func (f *evFixture) observe(t *testing.T) hostclaim.Observed { |
| + t.Helper() |
| + obs, err := hostclaim.Read(t.Context(), f.claims, evHost) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return obs |
| +} |
| + |
| +func (f *evFixture) census(t *testing.T) []lease.ObjectVersion { |
| + t.Helper() |
| + var out []lease.ObjectVersion |
| + for _, p := range []string{"instances/" + evHost + "/", "moves/" + evHost + "/"} { |
| + vs, err := f.bucket.ListVersions(t.Context(), "data", p) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + out = append(out, vs...) |
| + } |
| + return out |
| +} |
| + |
| +func (f *evFixture) provider() movesRetirementEvidence { |
| + return movesRetirementEvidence{ |
| + Records: func(_ context.Context, acct, app, host string) (*provision.Record, error) { |
| + f.reads++ |
| + if acct != evAcct || app != evApp || host != evHost { |
| + return nil, fmt.Errorf("read %s/%s/%s", acct, app, host) |
| + } |
| + if f.rec == nil { |
| + return nil, errors.New("no record") |
| + } |
| + cp := *f.rec |
| + return &cp, nil |
| + }, |
| + Now: func() time.Time { return evNow }, |
| + } |
| +} |
| + |
| +// lane writes one object of a move's staging lane. |
| +func (f *evFixture) lane(op, name string) { |
| + f.bucket.Put("data", "moves/"+evHost+"/"+op+"/"+name, []byte(op)) |
| +} |
| + |
| +func wantRefusal(t *testing.T, err error, anomaly bool) { |
| + t.Helper() |
| + var r *MovesSweepRefusal |
| + if !errors.As(err, &r) { |
| + t.Fatalf("err = %v, want a *MovesSweepRefusal", err) |
| + } |
| + if r.Host != evHost || r.Reason == "" || r.Anomaly != anomaly { |
| + t.Fatalf("refusal = %+v, want host %s, a reason and anomaly %v", r, evHost, anomaly) |
| + } |
| +} |
| + |
| +// bothRefuse asserts the two methods refuse alike: VerifyRetirement's |
| +// label is the proof's, so neither may answer where the other refuses. |
| +func bothRefuse(t *testing.T, f *evFixture, anomaly bool) { |
| + t.Helper() |
| + obs, census := f.observe(t), f.census(t) |
| + if _, err := f.provider().VerifyMovesSweep(t.Context(), obs, census); err == nil { |
| + t.Fatal("VerifyMovesSweep answered") |
| + } else { |
| + wantRefusal(t, err, anomaly) |
| + } |
| + inc, at, err := f.provider().VerifyRetirement(t.Context(), obs, census) |
| + if err == nil || inc != "" || !at.IsZero() { |
| + t.Fatalf("VerifyRetirement = %q, %v, %v; want a refusal and no evidence", inc, at, err) |
| + } |
| + wantRefusal(t, err, anomaly) |
| +} |
| + |
| +// TestMovesSweepProofRetiredAtIsTheClose (M8): the proof binds the exact |
| +// claim observation, names LastMove's id, the holder's incarnation and the |
| +// close's time, and the engine's Check accepts it only for a listing that |
| +// started after that close. |
| +func TestMovesSweepProofRetiredAtIsTheClose(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.lane(evMoveID, "move.json") |
| + f.lane(evMoveID, "replica/0000/db") |
| + f.bucket.Put("data", "moves/"+evHost+"/lock.json", []byte("lease")) |
| + obs := f.observe(t) |
| + proof, err := f.provider().VerifyMovesSweep(t.Context(), obs, f.census(t)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + want := park.MoveSweepProof{Host: evHost, ClaimETag: obs.ETag, MoveID: evMoveID, Incarnation: f.inc, RetiredAt: evAt} |
| + if proof != want { |
| + t.Fatalf("proof = %+v\nwant %+v", proof, want) |
| + } |
| + if err := proof.Check(obs, f.inc, evNow); err != nil { |
| + t.Fatalf("a listing after the close: %v", err) |
| + } |
| + for _, listing := range []time.Time{evAt, evAt.Add(-time.Second)} { |
| + if err := proof.Check(obs, f.inc, listing); !errors.Is(err, park.ErrUnsafe) { |
| + t.Fatalf("a listing at %v, not after the close %v: %v", listing, evAt, err) |
| + } |
| + } |
| +} |
| + |
| +// TestVerifyRetirementLabelsMovesResidueWithTheProofsIncarnation (M7): |
| +// Decide labels every moves/<host> candidate with VerifyRetirement's |
| +// answer, and Check requires the proof's incarnation to equal it, so the |
| +// two answers come from one source. The evidence time is the close's, never |
| +// zero and never the clock's. |
| +func TestVerifyRetirementLabelsMovesResidueWithTheProofsIncarnation(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.lane(evMoveID, "move.json") |
| + f.bucket.Put("data", f.genKey(evMoveID), []byte("live")) |
| + obs, census := f.observe(t), f.census(t) |
| + inc, at, err := f.provider().VerifyRetirement(t.Context(), obs, census) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if inc != f.inc || !at.Equal(evAt) { |
| + t.Fatalf("VerifyRetirement = %q, %v; want %q, %v", inc, at, f.inc, evAt) |
| + } |
| + proof, err := f.provider().VerifyMovesSweep(t.Context(), obs, census) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if proof.Incarnation != inc { |
| + t.Fatalf("label %q, proof %q", inc, proof.Incarnation) |
| + } |
| + if err := proof.Check(obs, inc, evNow); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +// TestVerifyRetirementRefusesLegacyResidue (M7): residue under |
| +// instances/<host> outside the holder's live location has no moves-owned |
| +// archived-incarnation history. VerifyRetirement refuses it with a typed |
| +// dependency error (the legacy census is claim Plan 12's), which skips the |
| +// whole at-rest sweep. The moves proof itself is unaffected. |
| +func TestVerifyRetirementRefusesLegacyResidue(t *testing.T) { |
| + for name, key := range map[string]string{ |
| + "a legacy level object": "instances/" + evHost + "/0000/db", |
| + "a legacy generation object": "instances/" + evHost + "/g-old/0000/db", |
| + "an older incarnation": "instances/" + evHost + "/" + strings.Repeat("ab", 32) + "/g-old/0000/db", |
| + } { |
| + t.Run(name, func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.lane(evMoveID, "move.json") |
| + f.bucket.Put("data", f.genKey(evMoveID), []byte("live")) |
| + f.bucket.Put("data", key, []byte("legacy")) |
| + obs, census := f.observe(t), f.census(t) |
| + inc, at, err := f.provider().VerifyRetirement(t.Context(), obs, census) |
| + var dep *ClaimDependencyError |
| + if !errors.As(err, &dep) || inc != "" || !at.IsZero() { |
| + t.Fatalf("VerifyRetirement = %q, %v, %v; want a *ClaimDependencyError and no evidence", inc, at, err) |
| + } |
| + if _, err := f.provider().VerifyMovesSweep(t.Context(), obs, census); err != nil { |
| + t.Fatalf("the moves proof does not depend on instances residue: %v", err) |
| + } |
| + }) |
| + } |
| + t.Run("an incarnation directory beside a bare holder", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.bare(t) |
| + f.bucket.Put("data", f.genKey(evMoveID), []byte("live")) |
| + f.bucket.Put("data", "instances/"+evHost+"/"+f.inc+"/g-old/0000/db", []byte("scoped")) |
| + _, _, err := f.provider().VerifyRetirement(t.Context(), f.observe(t), f.census(t)) |
| + var dep *ClaimDependencyError |
| + if !errors.As(err, &dep) { |
| + t.Fatalf("err = %v, want a *ClaimDependencyError", err) |
| + } |
| + }) |
| +} |
| + |
| +// bare puts the holder at the activator's own lease location, |
| +// instances/<host>: the only holder a move cuts (ruling T5-lease, |
| +// movescut.go writerLease), so every moved host's holder. |
| +func (f *evFixture) bare(t *testing.T) { |
| + t.Helper() |
| + f.location = "instances/" + evHost |
| + f.setClaim(t, func(c *hostclaim.Claim) { c.Holder.Location = f.location }) |
| +} |
| + |
| +// TestMovedHostAtABareLocationIsSwept (controller finding, Plan 12 review): |
| +// a moved host's holder is at the bare location, and its objects there are |
| +// its live location's, not residue outside it. The provider answers, and |
| +// the worker queues the moves/<host> lane chore. |
| +func TestMovedHostAtABareLocationIsSwept(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.bare(t) |
| + f.lane(evMoveID, "move.json") |
| + f.lane(evMoveID, "replica/0000/db") |
| + f.bucket.Put("data", f.genKey(evMoveID), []byte("live")) |
| + obs, census := f.observe(t), f.census(t) |
| + inc, at, err := f.provider().VerifyRetirement(t.Context(), obs, census) |
| + if err != nil || inc != f.inc || !at.Equal(evAt) { |
| + t.Fatalf("VerifyRetirement = %q, %v, %v; want %q, %v", inc, at, err, f.inc, evAt) |
| + } |
| + proof, err := f.provider().VerifyMovesSweep(t.Context(), obs, census) |
| + if err != nil || proof.Incarnation != f.inc || !proof.RetiredAt.Equal(evAt) { |
| + t.Fatalf("proof = %+v, %v", proof, err) |
| + } |
| + if err := f.worker(t, evNow).Run(t.Context()); !errors.Is(err, park.ErrPending) { |
| + t.Fatalf("the lane chore waits for its horizon: %v", err) |
| + } |
| + moves, ok := f.chores(t)["moves/"+evHost] |
| + if !ok || moves.Incarnation != f.inc || !moves.EvidenceAt.Equal(evNow) { |
| + t.Fatalf("chores = %+v; want the moves/<host> chore at the listing's end", f.chores(t)) |
| + } |
| +} |
| + |
| +// TestVerifyRetirementVouchesOnlyForWhatTheMoveRetired (fix round 1, |
| +// ruling 1): the close is evidence only for what the move retired. At the |
| +// live location that is a generation superseded at or before the move's |
| +// own admission, every version of it older than the close. A generation |
| +// admitted from the move's admission on was written past the close, so the |
| +// provider has no evidence for it and says so with a typed error; the |
| +// moves lane proof is unaffected. |
| +func TestVerifyRetirementVouchesOnlyForWhatTheMoveRetired(t *testing.T) { |
| + answers := func(t *testing.T, f *evFixture) { |
| + t.Helper() |
| + inc, at, err := f.provider().VerifyRetirement(t.Context(), f.observe(t), f.census(t)) |
| + if err != nil || inc != f.inc || !at.Equal(evAt) { |
| + t.Fatalf("VerifyRetirement = %q, %v, %v; want %q, %v", inc, at, err, f.inc, evAt) |
| + } |
| + } |
| + notEvidence := func(t *testing.T, f *evFixture, gen string) { |
| + t.Helper() |
| + obs, census := f.observe(t), f.census(t) |
| + inc, at, err := f.provider().VerifyRetirement(t.Context(), obs, census) |
| + var ne *NotMovesEvidence |
| + if !errors.As(err, &ne) || inc != "" || !at.IsZero() { |
| + t.Fatalf("VerifyRetirement = %q, %v, %v; want a *NotMovesEvidence and no evidence", inc, at, err) |
| + } |
| + if ne.Host != evHost || ne.Generation != gen || ne.Reason == "" { |
| + t.Fatalf("refusal = %+v, want generation %q", ne, gen) |
| + } |
| + if _, err := f.provider().VerifyMovesSweep(t.Context(), obs, census); err != nil { |
| + t.Fatalf("the lane proof does not depend on the generations: %v", err) |
| + } |
| + } |
| + t.Run("the current generation and the marker", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.bucket.Put("data", f.genKey(evMoveID), []byte("live")) |
| + f.bucket.Put("data", f.location+"/generation.json", []byte("marker")) |
| + answers(t, f) |
| + }) |
| + t.Run("a generation the move's admission superseded", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.bucket.Put("data", f.genKey("old"), []byte("old")) |
| + f.bucket.Put("data", f.location+"/files/g-old/a.txt", []byte("old")) |
| + answers(t, f) |
| + }) |
| + t.Run("the move's generation superseded after the close", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.setClaim(t, func(c *hostclaim.Claim) { |
| + c.Holder.Admissions, c.Holder.Generation, c.Holder.Session, c.Holder.Stamp = evAdmissions(evMoveID, "later"), "later", 3, 3 |
| + }) |
| + f.bucket.Put("data", f.genKey(evMoveID), []byte("superseded after the close")) |
| + f.bucket.Put("data", f.genKey("later"), []byte("live")) |
| + notEvidence(t, f, evMoveID) |
| + }) |
| + t.Run("a later generation superseded after the close", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.setClaim(t, func(c *hostclaim.Claim) { |
| + c.Holder.Admissions, c.Holder.Generation, c.Holder.Session, c.Holder.Stamp = evAdmissions(evMoveID, "later", "latest"), "latest", 4, 4 |
| + }) |
| + f.bucket.Put("data", f.genKey("later"), []byte("superseded after the close")) |
| + notEvidence(t, f, "later") |
| + }) |
| + t.Run("no admission of the move in the claim", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.setClaim(t, func(c *hostclaim.Claim) { |
| + c.Holder.Admissions, c.Holder.Generation = evAdmissions("fresh"), "fresh" |
| + }) |
| + f.bucket.Put("data", f.genKey("old"), []byte("old")) |
| + notEvidence(t, f, "old") |
| + }) |
| + t.Run("a retired generation written after the close", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.rec.LastMove.At = time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC) |
| + f.bucket.Put("data", f.genKey("old"), []byte("late")) |
| + notEvidence(t, f, "old") |
| + }) |
| + t.Run("an object no generation owns", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.bucket.Put("data", f.location+"/stray.bin", []byte("?")) |
| + notEvidence(t, f, "") |
| + }) |
| +} |
| + |
| +// TestMovesSweepProofRefusesALiveMove (K10, M8): no proof while a move op |
| +// is on the claim, while the record carries a live move, or while the |
| +// claim was released by a move the record has not closed. |
| +func TestMovesSweepProofRefusesALiveMove(t *testing.T) { |
| + t.Run("the record carries a live move", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.rec.Move = &provision.MoveState{ID: "mv_next", Incarnation: f.inc, To: "uk", State: provision.MovePreparing} |
| + f.lane("mv_next", "move.json") |
| + bothRefuse(t, f, false) |
| + }) |
| + for _, op := range []hostclaim.Op{ |
| + {Kind: hostclaim.KindMoving, ID: "mv_next", Phase: hostclaim.PhaseMoving, Exec: "box"}, |
| + {Kind: hostclaim.KindLanding, ID: evMoveID, Phase: hostclaim.PhaseCommitted, Exec: "box"}, |
| + {Kind: hostclaim.KindRestoring, ID: evMoveID, Phase: hostclaim.PhaseOpened, Exec: "box"}, |
| + {Kind: hostclaim.KindReverting, ID: evMoveID, Phase: hostclaim.PhaseOpened, Exec: "box"}, |
| + } { |
| + // The last three are the record-first close's window (M4): the |
| + // record closed the move, the claim has not released it yet. |
| + t.Run("the claim holds "+op.Kind, func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.setClaim(t, func(c *hostclaim.Claim) { c.Op = op }) |
| + bothRefuse(t, f, false) |
| + }) |
| + } |
| + t.Run("the record names no closed move", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.rec.LastMove = nil |
| + bothRefuse(t, f, false) |
| + }) |
| +} |
| + |
| +// TestMovesProofSurvivesAMetadataPlacement (fix round 1, ruling 2): the |
| +// durable proof is the record's LastMove, bound to the holder's |
| +// incarnation, with no move op on the claim. A later operation that |
| +// replaces the serving op (here CompleteMetadataPlacement, which drops the |
| +// released_move journal) leaves cleanup running, and so does a serving op |
| +// released by a move whose record never carried it. |
| +func TestMovesProofSurvivesAMetadataPlacement(t *testing.T) { |
| + t.Run("a metadata placement", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.lane(evMoveID, "move.json") |
| + f.bucket.Put("data", f.genKey(evMoveID), []byte("live")) |
| + f.setClaim(t, func(c *hostclaim.Claim) { |
| + if err := hostops.BeginPlacement("pl_1", "op@example", "meta", evNow, []string{"ie"}, []string{"box"})(c); err != nil { |
| + t.Fatal(err) |
| + } |
| + c.Op.Intent["app_id"] = "app_1" |
| + if err := hostops.CompleteMetadataPlacement("pl_1", "app_1")(c); err != nil { |
| + t.Fatal(err) |
| + } |
| + }) |
| + obs := f.observe(t) |
| + if obs.Claim.Op.Kind != hostclaim.KindServing || obs.Claim.Op.Journal["released_move"] != "" { |
| + t.Fatalf("the placement left op %+v", obs.Claim.Op) |
| + } |
| + census := f.census(t) |
| + proof, err := f.provider().VerifyMovesSweep(t.Context(), obs, census) |
| + if err != nil || proof.MoveID != evMoveID || !proof.RetiredAt.Equal(evAt) { |
| + t.Fatalf("proof = %+v, %v", proof, err) |
| + } |
| + if inc, at, err := f.provider().VerifyRetirement(t.Context(), obs, census); err != nil || inc != f.inc || !at.Equal(evAt) { |
| + t.Fatalf("VerifyRetirement = %q, %v, %v", inc, at, err) |
| + } |
| + if err := f.worker(t, evNow).Run(t.Context()); !errors.Is(err, park.ErrPending) { |
| + t.Fatalf("the lane chore waits for its horizon: %v", err) |
| + } |
| + if _, ok := f.chores(t)["moves/"+evHost]; !ok { |
| + t.Fatalf("chores = %+v; want the moves/<host> chore", f.chores(t)) |
| + } |
| + }) |
| + t.Run("a release by a move the record never carried", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.setClaim(t, func(c *hostclaim.Claim) { c.Op = servingReleasedBy("mv_orphan") }) |
| + if _, err := f.provider().VerifyMovesSweep(t.Context(), f.observe(t), f.census(t)); err != nil { |
| + t.Fatalf("no lane of the orphan, so nothing unproven: %v", err) |
| + } |
| + f.bucket.Put("data", "moves/"+evHost+"/mv_orphan/move.json", []byte("orphan")) |
| + f.rec.LastMove.At = time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC) |
| + bothRefuse(t, f, true) |
| + }) |
| +} |
| + |
| +// TestMovesEvidenceRefusesAZeroTime (M7, M8): a zero time is never |
| +// evidence: not the close's, and not a lane object's. |
| +func TestMovesEvidenceRefusesAZeroTime(t *testing.T) { |
| + t.Run("a zero close", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.rec.LastMove.At = time.Time{} |
| + bothRefuse(t, f, true) |
| + }) |
| + t.Run("a close not before the provider's clock", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.rec.LastMove.At = evNow |
| + bothRefuse(t, f, true) |
| + }) |
| + t.Run("an older lane object with no time", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + obs := f.observe(t) |
| + census := []lease.ObjectVersion{{Key: "moves/" + evHost + "/mv_older/move.json", VersionID: "v1", IsLatest: true}} |
| + if _, err := f.provider().VerifyMovesSweep(t.Context(), obs, census); err == nil { |
| + t.Fatal("a zero LastModified counted as before the close") |
| + } else { |
| + wantRefusal(t, err, true) |
| + } |
| + if _, _, err := f.provider().VerifyRetirement(t.Context(), obs, census); err == nil { |
| + t.Fatal("VerifyRetirement answered over a lane with no time") |
| + } |
| + }) |
| +} |
| + |
| +// TestMovesSweepProofRefusesAnUnknownLane (M8): every moves/<host>/<op>/ |
| +// object is the closed move's own, or older than its close. An object of |
| +// another op written after the close has an unknown writer. |
| +func TestMovesSweepProofRefusesAnUnknownLane(t *testing.T) { |
| + t.Run("an older lane is covered by the close", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.lane("mv_older", "move.json") |
| + f.lane("rb_return", "replica/0000/db") |
| + if _, err := f.provider().VerifyMovesSweep(t.Context(), f.observe(t), f.census(t)); err != nil { |
| + t.Fatal(err) |
| + } |
| + }) |
| + t.Run("a lane written after the close", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.rec.LastMove.At = time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC) // before every write |
| + f.lane(evMoveID, "move.json") |
| + if _, err := f.provider().VerifyMovesSweep(t.Context(), f.observe(t), f.census(t)); err != nil { |
| + t.Fatalf("the closed move's own lane is its own: %v", err) |
| + } |
| + f.lane("mv_unknown", "move.json") |
| + bothRefuse(t, f, true) |
| + }) |
| + t.Run("an object outside any lane", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.bucket.Put("data", "moves/"+evHost+"/stray.json", []byte("?")) |
| + bothRefuse(t, f, true) |
| + }) |
| + t.Run("an object outside the host's namespaces", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + census := []lease.ObjectVersion{{Key: "moves/other.example/" + evMoveID + "/move.json", VersionID: "v1", LastModified: evAt.Add(-time.Hour)}} |
| + if _, err := f.provider().VerifyMovesSweep(t.Context(), f.observe(t), census); err == nil { |
| + t.Fatal("another host's object counted") |
| + } |
| + }) |
| +} |
| + |
| +// TestMovesEvidenceRefusesAnotherIdentity: the record must be the holder's |
| +// exact record, and a holderless claim has no record to prove from. |
| +func TestMovesEvidenceRefusesAnotherIdentity(t *testing.T) { |
| + t.Run("a record of another incarnation", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.rec.CreatedAt = evCreated.Add(time.Hour) |
| + bothRefuse(t, f, true) |
| + }) |
| + t.Run("a stale observation", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + obs := f.observe(t) |
| + obs.ETag = "" |
| + if _, err := f.provider().VerifyMovesSweep(t.Context(), obs, f.census(t)); err == nil { |
| + t.Fatal("answered without an etag") |
| + } |
| + }) |
| + t.Run("a free claim", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.setClaim(t, func(c *hostclaim.Claim) { c.Holder, c.Op = nil, hostclaim.Op{} }) |
| + obs, census := f.observe(t), f.census(t) |
| + _, err := f.provider().VerifyMovesSweep(t.Context(), obs, census) |
| + var dep *ClaimDependencyError |
| + if !errors.As(err, &dep) { |
| + t.Fatalf("err = %v, want a *ClaimDependencyError", err) |
| + } |
| + if _, _, err := f.provider().VerifyRetirement(t.Context(), obs, census); !errors.As(err, &dep) { |
| + t.Fatalf("err = %v, want a *ClaimDependencyError", err) |
| + } |
| + if f.reads != 0 { |
| + t.Fatalf("read %d records with no holder to name one", f.reads) |
| + } |
| + }) |
| + t.Run("no providers", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + if _, err := (movesRetirementEvidence{}).VerifyMovesSweep(t.Context(), f.observe(t), nil); err == nil { |
| + t.Fatal("answered with no record reader") |
| + } |
| + if _, _, err := (movesRetirementEvidence{Records: f.provider().Records}).VerifyRetirement(t.Context(), f.observe(t), nil); err == nil { |
| + t.Fatal("answered with no clock") |
| + } |
| + }) |
| +} |
| + |
| +// The provider satisfies both slots RegisterProductionClaimMoveParks reads: |
| +// the regional evidence and, by type assertion, the sweep verifier. |
| +var ( |
| + _ hostbox.RegionalEvidence = movesRetirementEvidence{} |
| + _ park.MoveSweepVerifier = movesRetirementEvidence{} |
| +) |
| + |
| +// --- against the claim's regional worker, with a fake store --- |
| + |
| +type evLease struct{} |
| + |
| +func (evLease) Check(context.Context) error { return nil } |
| +func (evLease) Release(context.Context) error { return nil } |
| +func (evLease) Close() {} |
| + |
| +type evLeases struct{} |
| + |
| +func (evLeases) Acquire(context.Context, string, string) (park.HeldLease, error) { |
| + return evLease{}, nil |
| +} |
| + |
| +type evEnrollment struct{} |
| + |
| +func (evEnrollment) VerifyRegional(context.Context, string, string, time.Duration) error { return nil } |
| + |
| +// worker is hostbox.RegionalWorker over park.Engine, wired as the claim's |
| +// assembly wires them (host/claimassembly.go): the provider is the worker's |
| +// Evidence and the engine's VerifyMovesSweep. This region (ie) is not the |
| +// authoritative one, so the holder's location here is residue too. |
| +func (f *evFixture) worker(t *testing.T, listing time.Time) hostbox.RegionalWorker { |
| + t.Helper() |
| + seq := 0 |
| + p := f.provider() |
| + location := f.location |
| + engine := park.Engine{ |
| + Store: f.bucket, Claims: f.claims, Bucket: "data", Region: "ie", Leases: evLeases{}, |
| + Now: func() time.Time { return listing }, |
| + NewID: func() string { seq++; return fmt.Sprintf("id%d", seq) }, |
| + View: func(_ context.Context, o hostclaim.Observed) (park.View, error) { |
| + return park.View{ClaimETag: o.ETag, Region: "ie", AuthoritativeRegion: "uk", |
| + GenerationOf: func(r park.Ref) (string, bool) { return landingGenerationOf(location, r.Key) }}, nil |
| + }, |
| + Authorize: func(context.Context, hostclaim.Observed, hostclaim.Obligation) error { return nil }, |
| + VerifyMovesSweep: p.VerifyMovesSweep, |
| + } |
| + return hostbox.RegionalWorker{ |
| + Regional: hostbox.Regional{Engine: engine, Owner: func(context.Context, hostclaim.Observed) error { return nil }}, |
| + Enrollment: evEnrollment{}, |
| + Evidence: p, |
| + } |
| +} |
| + |
| +func (f *evFixture) chores(t *testing.T) map[string]hostclaim.ParkScope { |
| + t.Helper() |
| + out := map[string]hostclaim.ParkScope{} |
| + for _, o := range f.observe(t).Claim.Obligations { |
| + if o.Park == nil { |
| + t.Fatalf("an unscoped chore: %+v", o) |
| + } |
| + out[o.Park.Location] = *o.Park |
| + } |
| + return out |
| +} |
| + |
| +// TestAllowedSweepQueuesTheMovesChore: with the proof, the worker queues the |
| +// moves/<host> chore under the serving session, labelled with the holder's |
| +// incarnation, its evidence the listing's end, so it waits for its horizon. |
| +// The holder location's residue in this non-authoritative region takes the |
| +// close as its evidence: that horizon has passed, so the same pass parks it. |
| +func TestAllowedSweepQueuesTheMovesChore(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.lane(evMoveID, "move.json") |
| + f.lane("mv_older", "replica/0000/db") |
| + left := f.genKey("old") // superseded by the move's own admission |
| + f.bucket.Put("data", left, []byte("left behind")) |
| + err := f.worker(t, evNow).Run(t.Context()) |
| + if !errors.Is(err, park.ErrPending) { |
| + t.Fatalf("a new sweep waits for its horizon: %v", err) |
| + } |
| + chores := f.chores(t) |
| + moves, ok := chores["moves/"+evHost] |
| + if !ok || len(chores) != 1 { |
| + t.Fatalf("chores = %+v; want the moves chore alone", chores) |
| + } |
| + if moves.Incarnation != f.inc || !moves.EvidenceAt.Equal(evNow) || moves.RequestLimit != hostbox.RequestLimit { |
| + t.Fatalf("moves chore = %+v; want incarnation %s, evidence %v", moves, f.inc, evNow) |
| + } |
| + if vs, err := f.bucket.ListVersions(t.Context(), "data", left); err != nil || len(vs) != 0 { |
| + t.Fatalf("the location's residue was not parked: %v, %v", vs, err) |
| + } |
| + if vs, err := f.bucket.ListVersions(t.Context(), "data", "parked/"+f.inc+"/"+evHost+"/"); err != nil || len(vs) == 0 { |
| + t.Fatalf("no archive under the holder's incarnation: %v, %v", vs, err) |
| + } |
| + for _, o := range f.observe(t).Claim.Obligations { |
| + if o.Op != "session:2" || o.Owner != "region:ie" { |
| + t.Fatalf("chore %+v is not the serving session's regional chore", o) |
| + } |
| + } |
| + if c := f.observe(t).Claim; c.Op.Kind != hostclaim.KindServing || c.Op.Journal["released_move"] != evMoveID { |
| + t.Fatalf("the sweep changed the serving op: %+v", c.Op) |
| + } |
| +} |
| + |
| +// TestSweepSkipsWhileAMoveStages (Review Focus 3, K10): while a move op is |
| +// on the claim the worker never asks for evidence; while the record carries |
| +// a live move under a serving claim the provider refuses and nothing is |
| +// queued. |
| +func TestSweepSkipsWhileAMoveStages(t *testing.T) { |
| + t.Run("a moving claim", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.setClaim(t, func(c *hostclaim.Claim) { |
| + c.Op = hostclaim.Op{Kind: hostclaim.KindMoving, ID: "mv_next", Phase: hostclaim.PhaseMoving, Exec: "box"} |
| + }) |
| + f.rec.Move = &provision.MoveState{ID: "mv_next", Incarnation: f.inc, To: "uk", State: provision.MovePreparing} |
| + f.lane("mv_next", "replica/0000/db") |
| + if err := f.worker(t, evNow).Run(t.Context()); err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(f.chores(t)) != 0 || f.reads != 0 { |
| + t.Fatalf("chores %+v, record reads %d", f.chores(t), f.reads) |
| + } |
| + }) |
| + t.Run("a live move on the record", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.rec.Move = &provision.MoveState{ID: "mv_next", Incarnation: f.inc, To: "uk", State: provision.MovePreparing} |
| + f.lane("mv_next", "replica/0000/db") |
| + err := f.worker(t, evNow).Run(t.Context()) |
| + var r *MovesSweepRefusal |
| + if !errors.As(err, &r) { |
| + t.Fatalf("err = %v, want the provider's refusal", err) |
| + } |
| + if len(f.chores(t)) != 0 { |
| + t.Fatalf("chores %+v", f.chores(t)) |
| + } |
| + }) |
| +} |
| + |
| +// TestSweepRefusedUnlessTheCloseIsBeforeTheListing (K10): the worker's own |
| +// Check drops the moves/<host> candidate when the newest retired move's |
| +// close is not earlier than the listing; the rest of the decision stands. |
| +func TestSweepRefusedUnlessTheCloseIsBeforeTheListing(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.lane(evMoveID, "move.json") |
| + f.bucket.Put("data", f.genKey("old"), []byte("left behind")) |
| + err := f.worker(t, evAt).Run(t.Context()) |
| + if !errors.Is(err, park.ErrPending) { |
| + t.Fatalf("the location chore still queues: %v", err) |
| + } |
| + chores := f.chores(t) |
| + if _, ok := chores["moves/"+evHost]; ok || len(chores) != 1 { |
| + t.Fatalf("chores = %+v; want only the location's", chores) |
| + } |
| + if loc := chores[f.location]; loc.Incarnation != f.inc || !loc.EvidenceAt.Equal(evAt) { |
| + t.Fatalf("location chore = %+v; want the close %v as its evidence", loc, evAt) |
| + } |
| +} |
| + |
| +// TestSweepScopeMembership (witness wSweepScopeMembership, M7): residue of |
| +// an older incarnation is never the moves/<host> chore's. Without its own |
| +// evidence the provider refuses the whole sweep, so nothing is queued. |
| +func TestSweepScopeMembership(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.lane(evMoveID, "move.json") |
| + f.bucket.Put("data", "instances/"+evHost+"/"+strings.Repeat("cd", 32)+"/g-old/0000/db", []byte("late")) |
| + err := f.worker(t, evNow).Run(t.Context()) |
| + var dep *ClaimDependencyError |
| + if !errors.As(err, &dep) { |
| + t.Fatalf("err = %v, want the legacy census dependency", err) |
| + } |
| + if len(f.chores(t)) != 0 { |
| + t.Fatalf("chores %+v", f.chores(t)) |
| + } |
| +} |
| + |
| +// TestFreeHostLaneProofIsClaimPlan12s (fix round 1, ruling 3): a free claim |
| +// names no record, so moves can prove no closed move and the claim stays |
| +// free. The free-host lane proof is transferred to claim Plan 12 (confirmed |
| +// by platform-1e); this test pins the refusal until then. |
| +func TestFreeHostLaneProofIsClaimPlan12s(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.setClaim(t, func(c *hostclaim.Claim) { c.Holder, c.Op, c.Touched = nil, hostclaim.Op{}, hostclaim.Touched{} }) |
| + f.lane(evMoveID, "move.json") |
| + err := f.worker(t, evNow).Run(t.Context()) |
| + var dep *ClaimDependencyError |
| + if !errors.As(err, &dep) { |
| + t.Fatalf("err = %v, want a *ClaimDependencyError", err) |
| + } |
| + if c := f.observe(t).Claim; !c.Free() || len(c.Obligations) != 0 { |
| + t.Fatalf("claim = %+v", c) |
| + } |
| +} |
| + |
| +// authoritative makes ie the authoritative region, with the marker at the |
| +// holder's current generation and objects classified by the landing's |
| +// layout, as Task 9a's view will. |
| +func (f *evFixture) authoritative(t *testing.T, w *hostbox.RegionalWorker, current string) { |
| + t.Helper() |
| + marker := f.location + "/generation.json" |
| + ref := park.Ref{Key: marker, VersionID: f.bucket.LatestID("data", marker)} |
| + location := f.location |
| + w.Regional.Engine.View = func(_ context.Context, o hostclaim.Observed) (park.View, error) { |
| + return park.View{ClaimETag: o.ETag, Region: "ie", AuthoritativeRegion: "ie", MarkerKnown: true, MarkerGeneration: current, Marker: ref, |
| + GenerationOf: func(r park.Ref) (string, bool) { return landingGenerationOf(location, r.Key) }}, nil |
| + } |
| +} |
| + |
| +// TestAuthoritativeSweepStampsTheCloseOnlyOnWhatTheMoveRetired (fix round 1, |
| +// ruling 1): in the authoritative region the engine makes superseded |
| +// generations park-generation candidates and binds them to |
| +// VerifyRetirement's time. A generation the move's admission superseded |
| +// gets the close; when the move's own generation was superseded after the |
| +// close, the provider refuses and nothing is queued. |
| +func TestAuthoritativeSweepStampsTheCloseOnlyOnWhatTheMoveRetired(t *testing.T) { |
| + listing := evAt.Add(time.Minute) // before the close's horizon, so chores stay queued |
| + t.Run("superseded by the move", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.lane(evMoveID, "move.json") |
| + f.bucket.Put("data", f.genKey(evMoveID), []byte("live")) |
| + f.bucket.Put("data", f.location+"/generation.json", []byte("marker")) |
| + f.bucket.Put("data", f.genKey("old"), []byte("superseded by the move")) |
| + w := f.worker(t, listing) |
| + f.authoritative(t, &w, evMoveID) |
| + if err := w.Run(t.Context()); !errors.Is(err, park.ErrPending) { |
| + t.Fatalf("the chores wait for their horizons: %v", err) |
| + } |
| + var gens []hostclaim.Obligation |
| + for _, o := range f.observe(t).Claim.Obligations { |
| + if o.Kind == "park-generation" { |
| + gens = append(gens, o) |
| + } |
| + } |
| + if len(gens) != 1 || gens[0].Generation != "old" || gens[0].Park.Location != f.location || !gens[0].Park.EvidenceAt.Equal(evAt) { |
| + t.Fatalf("generation chores = %+v; want one for old at the close %v", gens, evAt) |
| + } |
| + if moves, ok := f.chores(t)["moves/"+evHost]; !ok || !moves.EvidenceAt.Equal(listing) { |
| + t.Fatalf("chores = %+v; want the moves chore at the listing", f.chores(t)) |
| + } |
| + }) |
| + t.Run("superseded after the close", func(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.setClaim(t, func(c *hostclaim.Claim) { |
| + c.Holder.Admissions, c.Holder.Generation, c.Holder.Session, c.Holder.Stamp = evAdmissions(evMoveID, "later"), "later", 3, 3 |
| + }) |
| + f.lane(evMoveID, "move.json") |
| + f.bucket.Put("data", f.genKey("later"), []byte("live")) |
| + f.bucket.Put("data", f.location+"/generation.json", []byte("marker")) |
| + f.bucket.Put("data", f.genKey(evMoveID), []byte("written until after the close")) |
| + w := f.worker(t, listing) |
| + f.authoritative(t, &w, "later") |
| + err := w.Run(t.Context()) |
| + var ne *NotMovesEvidence |
| + if !errors.As(err, &ne) || ne.Generation != evMoveID { |
| + t.Fatalf("err = %v, want a *NotMovesEvidence for %s", err, evMoveID) |
| + } |
| + if len(f.observe(t).Claim.Obligations) != 0 { |
| + t.Fatalf("chores %+v", f.observe(t).Claim.Obligations) |
| + } |
| + }) |
| +} |
| + |
| +// TestMovesParkLockMatchesTheEngine (fix round 2): the provider mirrors the |
| +// park engine's unexported lock predicate (park's isLock over locationOf), |
| +// so it skips exactly the lease objects Decide skips. park.Decide is the |
| +// oracle: over a one-key census in a non-authoritative region, where every |
| +// non-lock object is a candidate, it decides nothing exactly for a lock. |
| +func TestMovesParkLockMatchesTheEngine(t *testing.T) { |
| + f := newEvFixture(t) |
| + obs := f.observe(t) |
| + view := park.View{ClaimETag: obs.ETag, Region: "ie", AuthoritativeRegion: "uk", |
| + GenerationOf: func(r park.Ref) (string, bool) { return landingGenerationOf(f.location, r.Key) }} |
| + other := strings.Repeat("ab", 32) |
| + for _, key := range []string{ |
| + "instances/" + evHost + "/lock.json", |
| + "instances/" + evHost + "/" + f.inc + "/lock.json", |
| + "instances/" + evHost + "/" + other + "/lock.json", |
| + "moves/" + evHost + "/lock.json", |
| + "moves/" + evHost + "/" + evMoveID + "/lock.json", |
| + "instances/" + evHost + "/g-old/lock.json", |
| + "instances/" + evHost + "/" + f.inc + "/g-old/lock.json", |
| + "instances/" + evHost + "/lock.json.old", |
| + "instances/" + evHost + "/0000/db", |
| + } { |
| + d, err := park.Decide(obs, view, []lease.ObjectVersion{{Key: key, VersionID: "v1", LastModified: evAt.Add(-time.Hour)}}, f.inc) |
| + if err != nil { |
| + t.Fatalf("%s: %v", key, err) |
| + } |
| + if engine, mirror := len(d.Candidates) == 0, movesParkLock(evHost, key); engine != mirror { |
| + t.Errorf("%s: the engine skips it %v, the provider's mirror says lock %v", key, engine, mirror) |
| + } |
| + } |
| +} |
| + |
| +// TestLiveLocationLeaseIsSweptNormally (fix round 2): the writer's lease, |
| +// <holder.location>/lock.json (lease.Acquire), stands at every live |
| +// location. It is the engine's to skip, never residue the close must |
| +// vouch for, so the sweep proceeds, scoped and bare alike. |
| +func TestLiveLocationLeaseIsSweptNormally(t *testing.T) { |
| + for name, bare := range map[string]bool{"scoped": false, "bare": true} { |
| + t.Run(name, func(t *testing.T) { |
| + f := newEvFixture(t) |
| + if bare { |
| + f.bare(t) |
| + } |
| + f.lane(evMoveID, "move.json") |
| + f.bucket.Put("data", f.genKey(evMoveID), []byte("live")) |
| + f.bucket.Put("data", f.location+"/generation.json", []byte("marker")) |
| + f.bucket.Put("data", f.location+"/lock.json", []byte("writer lease")) |
| + if _, _, err := f.provider().VerifyRetirement(t.Context(), f.observe(t), f.census(t)); err != nil { |
| + t.Fatalf("VerifyRetirement refused the writer's lease: %v", err) |
| + } |
| + listing := evAt.Add(time.Minute) |
| + w := f.worker(t, listing) |
| + f.authoritative(t, &w, evMoveID) |
| + if err := w.Run(t.Context()); !errors.Is(err, park.ErrPending) { |
| + t.Fatalf("the lane chore waits for its horizon: %v", err) |
| + } |
| + chores := f.chores(t) |
| + if moves, ok := chores["moves/"+evHost]; !ok || len(chores) != 1 || !moves.EvidenceAt.Equal(listing) { |
| + t.Fatalf("chores = %+v; want the moves chore alone", chores) |
| + } |
| + if vs, err := f.bucket.ListVersions(t.Context(), "data", f.location+"/lock.json"); err != nil || len(vs) != 1 { |
| + t.Fatalf("the lease was touched: %v, %v", vs, err) |
| + } |
| + }) |
| + } |
| +} |
amadan