amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package host
2
3import (
4 "context"
5 "errors"
6 "fmt"
7 "strings"
8 "testing"
9 "time"
10
11 "github.com/carlosframework/platform/internal/hostbox"
12 "github.com/carlosframework/platform/internal/hostclaim"
13 "github.com/carlosframework/platform/internal/hostops"
14 "github.com/carlosframework/platform/internal/lease"
15 "github.com/carlosframework/platform/internal/lease/leasetest"
16 "github.com/carlosframework/platform/internal/park"
17 "github.com/carlosframework/platform/internal/provision"
18 "github.com/carlosframework/platform/internal/release"
19)
20
21// The moves retirement evidence provider (Plan 5 Task 5, rulings K10, M7,
22// M8). The fake bucket's clock starts at 2026-10-05T00:00:00Z and ticks a
23// second per write, so every object a fixture writes is older than evAt.
24var (
25 evCreated = time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
26 evAt = time.Date(2026, 10, 5, 1, 0, 0, 0, time.UTC) // LastMove.At, the close
27 evNow = time.Date(2026, 10, 5, 2, 0, 0, 0, time.UTC) // the sweep's clock
28)
29
30const (
31 evHost = "ev.example"
32 evAcct = "acct"
33 evApp = "app"
34 evMoveID = "mv_retired"
35)
36
37type evFixture struct {
38 inc, location string
39 claims release.Store
40 bucket *leasetest.VersionedBucket
41 rec *provision.Record
42 reads int
43}
44
45// newEvFixture is a serving claim released by evMoveID, whose record closed
46// that move at evAt and carries no live move.
47func newEvFixture(t *testing.T) *evFixture {
48 t.Helper()
49 inc, err := hostclaim.Incarnation(evAcct, evApp, evHost, evCreated)
50 if err != nil {
51 t.Fatal(err)
52 }
53 f := &evFixture{inc: inc, location: "instances/" + evHost + "/" + inc, claims: release.NewDirStore(t.TempDir()), bucket: leasetest.NewVersionedBucket()}
54 f.bucket.AddBucket("data")
55 f.rec = &provision.Record{Schema: 1, Host: evHost, App: evApp, Account: evAcct, CreatedAt: evCreated,
56 LastMove: &provision.MoveOutcome{ID: evMoveID, Outcome: "done", At: evAt}}
57 f.setClaim(t, func(c *hostclaim.Claim) {
58 c.Holder = &hostclaim.Holder{Family: "instance", Key: provision.RecordKey(evAcct, evApp, evHost), Account: evAcct, App: evApp, Incarnation: inc, CreatedAt: evCreated, Location: f.location, Stamp: 2, Session: 2, WriterBox: "box", Generation: evMoveID,
59 Admissions: evAdmissions(evMoveID)}
60 c.Op = servingReleasedBy(evMoveID)
61 c.Touched = hostclaim.Touched{Complete: true, Regions: []string{"ie"}}
62 })
63 return f
64}
65
66// evAdmissions is the holder's admission history: an origin generation
67// "old", then the move's prepared landing, which admits the generation
68// named after the move (movesprobeowners.go), then any later generations,
69// each the next one's predecessor. The last is the holder's current.
70func evAdmissions(gens ...string) []hostclaim.AdmissionRecord {
71 out := []hostclaim.AdmissionRecord{{Session: 1, Box: "box", Generation: "old", Mode: "cold", Region: "ie"}}
72 for i, g := range gens {
73 out = append(out, hostclaim.AdmissionRecord{Session: uint64(i + 2), Box: "box", Generation: g, Predecessor: out[i].Generation, Mode: "cold", Region: "ie", PredecessorRegion: "ie"})
74 }
75 return out
76}
77
78// genKey is an object of generation gen at the fixture's live location.
79func (f *evFixture) genKey(gen string) string { return f.location + "/g-" + gen + "/0000/db" }
80
81// servingReleasedBy is the serving op hostclaim.ReleaseMove leaves.
82func servingReleasedBy(id string) hostclaim.Op {
83 return hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box", Journal: map[string]string{"released_move": id}}
84}
85
86func (f *evFixture) setClaim(t *testing.T, mut func(*hostclaim.Claim)) {
87 t.Helper()
88 obs, err := hostclaim.Read(t.Context(), f.claims, evHost)
89 if err != nil {
90 t.Fatal(err)
91 }
92 if _, err := hostclaim.Update(t.Context(), f.claims, obs, func(c *hostclaim.Claim) error { mut(c); return nil }); err != nil {
93 t.Fatal(err)
94 }
95}
96
97func (f *evFixture) observe(t *testing.T) hostclaim.Observed {
98 t.Helper()
99 obs, err := hostclaim.Read(t.Context(), f.claims, evHost)
100 if err != nil {
101 t.Fatal(err)
102 }
103 return obs
104}
105
106func (f *evFixture) census(t *testing.T) []lease.ObjectVersion {
107 t.Helper()
108 var out []lease.ObjectVersion
109 for _, p := range []string{"instances/" + evHost + "/", "moves/" + evHost + "/"} {
110 vs, err := f.bucket.ListVersions(t.Context(), "data", p)
111 if err != nil {
112 t.Fatal(err)
113 }
114 out = append(out, vs...)
115 }
116 return out
117}
118
119func (f *evFixture) provider() movesRetirementEvidence {
120 return movesRetirementEvidence{
121 Records: func(_ context.Context, acct, app, host string) (*provision.Record, error) {
122 f.reads++
123 if acct != evAcct || app != evApp || host != evHost {
124 return nil, fmt.Errorf("read %s/%s/%s", acct, app, host)
125 }
126 if f.rec == nil {
127 return nil, errors.New("no record")
128 }
129 cp := *f.rec
130 return &cp, nil
131 },
132 Now: func() time.Time { return evNow },
133 }
134}
135
136// lane writes one object of a move's staging lane.
137func (f *evFixture) lane(op, name string) {
138 f.bucket.Put("data", "moves/"+evHost+"/"+op+"/"+name, []byte(op))
139}
140
141func wantRefusal(t *testing.T, err error, anomaly bool) {
142 t.Helper()
143 var r *MovesSweepRefusal
144 if !errors.As(err, &r) {
145 t.Fatalf("err = %v, want a *MovesSweepRefusal", err)
146 }
147 if r.Host != evHost || r.Reason == "" || r.Anomaly != anomaly {
148 t.Fatalf("refusal = %+v, want host %s, a reason and anomaly %v", r, evHost, anomaly)
149 }
150}
151
152// bothRefuse asserts the two methods refuse alike: VerifyRetirement's
153// label is the proof's, so neither may answer where the other refuses.
154func bothRefuse(t *testing.T, f *evFixture, anomaly bool) {
155 t.Helper()
156 obs, census := f.observe(t), f.census(t)
157 if _, err := f.provider().VerifyMovesSweep(t.Context(), obs, census); err == nil {
158 t.Fatal("VerifyMovesSweep answered")
159 } else {
160 wantRefusal(t, err, anomaly)
161 }
162 inc, at, err := f.provider().VerifyRetirement(t.Context(), obs, census)
163 if err == nil || inc != "" || !at.IsZero() {
164 t.Fatalf("VerifyRetirement = %q, %v, %v; want a refusal and no evidence", inc, at, err)
165 }
166 wantRefusal(t, err, anomaly)
167}
168
169// TestMovesSweepProofRetiredAtIsTheClose (M8): the proof binds the exact
170// claim observation, names LastMove's id, the holder's incarnation and the
171// close's time, and the engine's Check accepts it only for a listing that
172// started after that close.
173func TestMovesSweepProofRetiredAtIsTheClose(t *testing.T) {
174 f := newEvFixture(t)
175 f.lane(evMoveID, "move.json")
176 f.lane(evMoveID, "replica/0000/db")
177 f.bucket.Put("data", "moves/"+evHost+"/lock.json", []byte("lease"))
178 obs := f.observe(t)
179 proof, err := f.provider().VerifyMovesSweep(t.Context(), obs, f.census(t))
180 if err != nil {
181 t.Fatal(err)
182 }
183 want := park.MoveSweepProof{Host: evHost, ClaimETag: obs.ETag, MoveID: evMoveID, Incarnation: f.inc, RetiredAt: evAt}
184 if proof != want {
185 t.Fatalf("proof = %+v\nwant %+v", proof, want)
186 }
187 if err := proof.Check(obs, f.inc, evNow); err != nil {
188 t.Fatalf("a listing after the close: %v", err)
189 }
190 for _, listing := range []time.Time{evAt, evAt.Add(-time.Second)} {
191 if err := proof.Check(obs, f.inc, listing); !errors.Is(err, park.ErrUnsafe) {
192 t.Fatalf("a listing at %v, not after the close %v: %v", listing, evAt, err)
193 }
194 }
195}
196
197// TestVerifyRetirementLabelsMovesResidueWithTheProofsIncarnation (M7):
198// Decide labels every moves/<host> candidate with VerifyRetirement's
199// answer, and Check requires the proof's incarnation to equal it, so the
200// two answers come from one source. The evidence time is the close's, never
201// zero and never the clock's.
202func TestVerifyRetirementLabelsMovesResidueWithTheProofsIncarnation(t *testing.T) {
203 f := newEvFixture(t)
204 f.lane(evMoveID, "move.json")
205 f.bucket.Put("data", f.genKey(evMoveID), []byte("live"))
206 obs, census := f.observe(t), f.census(t)
207 inc, at, err := f.provider().VerifyRetirement(t.Context(), obs, census)
208 if err != nil {
209 t.Fatal(err)
210 }
211 if inc != f.inc || !at.Equal(evAt) {
212 t.Fatalf("VerifyRetirement = %q, %v; want %q, %v", inc, at, f.inc, evAt)
213 }
214 proof, err := f.provider().VerifyMovesSweep(t.Context(), obs, census)
215 if err != nil {
216 t.Fatal(err)
217 }
218 if proof.Incarnation != inc {
219 t.Fatalf("label %q, proof %q", inc, proof.Incarnation)
220 }
221 if err := proof.Check(obs, inc, evNow); err != nil {
222 t.Fatal(err)
223 }
224}
225
226// retirement runs the claim's retirement answer, the one the regional worker
227// binds to every instances/<host> candidate, over the real census with this
228// fixture's provider as the claim's moves evidence.
229func (f *evFixture) retirement(t *testing.T, evidence hostbox.RegionalEvidence) (string, time.Time, error) {
230 t.Helper()
231 p := productionClaimRegional{Now: func() time.Time { return evNow }, Parks: &claimMoveParkSources{Evidence: evidence}}
232 return p.retirement(t.Context(), f.observe(t), f.census(t))
233}
234
235// TestRetirementLeavesQualifiedResidueToTheCensus (Q4): residue under
236// instances/<host> outside the holder's live location is the claim's census
237// to place, not the moves provider's to refuse. The census labels a
238// qualified path by its own identity (ClaimRetirementCensus) and the claim's
239// view refuses unmapped bare history per location, so the provider answers
240// with the move's close and the moves/<host> lane sweep goes ahead.
241func TestRetirementLeavesQualifiedResidueToTheCensus(t *testing.T) {
242 older := strings.Repeat("ab", 32)
243 for name, key := range map[string]string{
244 "a legacy level object": "instances/" + evHost + "/0000/db",
245 "a legacy generation object": "instances/" + evHost + "/g-old/0000/db",
246 "an older incarnation": "instances/" + evHost + "/" + older + "/g-old/0000/db",
247 } {
248 t.Run(name, func(t *testing.T) {
249 f := newEvFixture(t)
250 f.lane(evMoveID, "move.json")
251 f.bucket.Put("data", f.genKey(evMoveID), []byte("live"))
252 f.bucket.Put("data", key, []byte("residue"))
253 inc, at, err := f.retirement(t, f.provider())
254 if err != nil || inc != f.inc || !at.Equal(evAt) {
255 t.Fatalf("retirement = %q, %v, %v; want %q, %v and no refusal", inc, at, err, f.inc, evAt)
256 }
257 if _, err := f.provider().VerifyMovesSweep(t.Context(), f.observe(t), f.census(t)); err != nil {
258 t.Fatalf("the moves proof does not depend on instances residue: %v", err)
259 }
260 })
261 }
262 t.Run("the census labels a qualified path by its own identity", func(t *testing.T) {
263 f := newEvFixture(t)
264 f.bucket.Put("data", f.genKey(evMoveID), []byte("live"))
265 f.bucket.Put("data", "instances/"+evHost+"/"+older+"/g-old/0000/db", []byte("older"))
266 labels, err := ClaimRetirementCensus(f.observe(t), f.census(t))
267 if err != nil || labels["instances/"+evHost+"/"+older] != older {
268 t.Fatalf("labels = %v, %v", labels, err)
269 }
270 if _, _, err := f.retirement(t, f.provider()); err != nil {
271 t.Fatal(err)
272 }
273 })
274 t.Run("an incarnation directory beside a bare holder", func(t *testing.T) {
275 f := newEvFixture(t)
276 f.bare(t)
277 f.bucket.Put("data", f.genKey(evMoveID), []byte("live"))
278 f.bucket.Put("data", "instances/"+evHost+"/"+f.inc+"/g-old/0000/db", []byte("scoped"))
279 inc, at, err := f.retirement(t, f.provider())
280 if err != nil || inc != f.inc || !at.Equal(evAt) {
281 t.Fatalf("retirement = %q, %v, %v", inc, at, err)
282 }
283 })
284}
285
286// TestRetirementStillRefusesAGenerationTheMoveDidNotRetire (Q4): the close
287// is the evidence time for every instances/<host> candidate the worker binds
288// it to, so at the live location it must be true. retiredGenerations stays:
289// a generation admitted from the move's own admission on, or written after
290// the close, is refused with *NotMovesEvidence. A generation the move did
291// retire passes beside it.
292func TestRetirementStillRefusesAGenerationTheMoveDidNotRetire(t *testing.T) {
293 refused := func(t *testing.T, f *evFixture, gen string) {
294 t.Helper()
295 inc, at, err := f.retirement(t, f.provider())
296 var n *NotMovesEvidence
297 if !errors.As(err, &n) || n.Generation != gen || inc != "" || !at.IsZero() {
298 t.Fatalf("retirement = %q, %v, %v; want *NotMovesEvidence for %q and no evidence", inc, at, err, gen)
299 }
300 if !errors.Is(err, ErrNotMovesEvidence) {
301 t.Fatalf("err does not unwrap to ErrNotMovesEvidence: %v", err)
302 }
303 }
304 t.Run("a generation the move retired passes", func(t *testing.T) {
305 f := newEvFixture(t)
306 f.bucket.Put("data", f.genKey(evMoveID), []byte("live"))
307 f.bucket.Put("data", f.genKey("old"), []byte("superseded by the move"))
308 inc, at, err := f.retirement(t, f.provider())
309 if err != nil || inc != f.inc || !at.Equal(evAt) {
310 t.Fatalf("retirement = %q, %v, %v", inc, at, err)
311 }
312 })
313 t.Run("a generation admitted after the move", func(t *testing.T) {
314 f := newEvFixture(t)
315 f.setClaim(t, func(c *hostclaim.Claim) {
316 c.Holder.Admissions = evAdmissions(evMoveID, "next")
317 c.Holder.Generation = "next"
318 })
319 f.bucket.Put("data", f.genKey("next"), []byte("live"))
320 f.bucket.Put("data", f.genKey(evMoveID), []byte("superseded after the move"))
321 refused(t, f, evMoveID)
322 })
323 t.Run("an object written after the close", func(t *testing.T) {
324 f := newEvFixture(t)
325 f.bucket.Put("data", f.genKey(evMoveID), []byte("live"))
326 f.bucket.Put("data", f.genKey("old"), []byte("late"))
327 f.rec.LastMove.At = time.Date(2026, 10, 5, 0, 0, 0, 0, time.UTC)
328 refused(t, f, "old")
329 })
330 t.Run("a claim that holds no admission of the move", func(t *testing.T) {
331 f := newEvFixture(t)
332 f.setClaim(t, func(c *hostclaim.Claim) { c.Holder.Admissions = evAdmissions()[:1]; c.Holder.Generation = "old" })
333 f.bucket.Put("data", f.genKey("old"), []byte("live"))
334 f.bucket.Put("data", f.genKey("other"), []byte("no admission to date it by"))
335 refused(t, f, "other")
336 })
337}
338
339// TestJoiningClaimRefusesQuietly (Q4): a joining claim (a parking sweep with
340// no holder) and a free one name no record, so the moves provider has no
341// close to prove from. It refuses with a *MovesSweepRefusal that is not an
342// anomaly, never a dependency, and reads no record: the agent logs it once an
343// hour and pages nobody. The claim's own free-host proof answers a free claim
344// before it gets here (claimFreeMoveProof).
345func TestJoiningClaimRefusesQuietly(t *testing.T) {
346 f := newEvFixture(t)
347 f.setClaim(t, func(c *hostclaim.Claim) {
348 c.Holder, c.Touched = nil, hostclaim.Touched{}
349 c.Op = hostclaim.Op{Kind: hostclaim.KindParking, ID: "op", Intent: map[string]string{"park_sweep": "v1"}}
350 c.Obligations = []hostclaim.Obligation{{ID: "chore", Op: "op", Kind: "park-replica", Region: "ie", Owner: "region:ie", Retirement: "ret",
351 Park: &hostclaim.ParkScope{Location: f.location, Incarnation: f.inc, EvidenceAt: evAt, RequestLimit: hostbox.RequestLimit}}}
352 })
353 f.lane(evMoveID, "move.json")
354 var pages pagesSeen
355 l := &refusalLog{now: evNow}
356 r := newMovesRefusals("ie-box", pages.hook(), l.logf, l.clock)
357 for name, evidence := range map[string]hostbox.RegionalEvidence{"the provider": f.provider(), "the quiet wrapper": r.evidence(f.provider())} {
358 t.Run(name, func(t *testing.T) {
359 inc, at, err := f.retirement(t, evidence)
360 var dep *ClaimDependencyError
361 if errors.As(err, &dep) {
362 t.Fatalf("a dependency, not a refusal: %v", err)
363 }
364 wantRefusal(t, err, false)
365 if inc != "" || !at.IsZero() {
366 t.Fatalf("evidence %q, %v with a refusal", inc, at)
367 }
368 })
369 }
370 if f.reads != 0 {
371 t.Fatalf("read %d records with no holder to name one", f.reads)
372 }
373 _, _, err := f.retirement(t, f.provider())
374 r.report(err)
375 r.report(err)
376 if got := l.take(); len(got) != 1 {
377 t.Fatalf("logged %q; want one line within the hour", got)
378 }
379 if len(pages) != 0 {
380 t.Fatalf("a joining claim paged: %+v", pages)
381 }
382}
383
384// bare puts the holder at the activator's own lease location,
385// instances/<host>: the only holder a move cuts (ruling T5-lease,
386// movescut.go writerLease), so every moved host's holder.
387func (f *evFixture) bare(t *testing.T) {
388 t.Helper()
389 f.location = "instances/" + evHost
390 f.setClaim(t, func(c *hostclaim.Claim) { c.Holder.Location = f.location })
391}
392
393// TestMovedHostAtABareLocationIsSwept (controller finding, Plan 12 review):
394// a moved host's holder is at the bare location, and its objects there are
395// its live location's, not residue outside it. The provider answers, and
396// the worker queues the moves/<host> lane chore.
397func TestMovedHostAtABareLocationIsSwept(t *testing.T) {
398 f := newEvFixture(t)
399 f.bare(t)
400 f.lane(evMoveID, "move.json")
401 f.lane(evMoveID, "replica/0000/db")
402 f.bucket.Put("data", f.genKey(evMoveID), []byte("live"))
403 obs, census := f.observe(t), f.census(t)
404 inc, at, err := f.provider().VerifyRetirement(t.Context(), obs, census)
405 if err != nil || inc != f.inc || !at.Equal(evAt) {
406 t.Fatalf("VerifyRetirement = %q, %v, %v; want %q, %v", inc, at, err, f.inc, evAt)
407 }
408 proof, err := f.provider().VerifyMovesSweep(t.Context(), obs, census)
409 if err != nil || proof.Incarnation != f.inc || !proof.RetiredAt.Equal(evAt) {
410 t.Fatalf("proof = %+v, %v", proof, err)
411 }
412 if err := f.worker(t, evNow).Run(t.Context()); !errors.Is(err, park.ErrPending) {
413 t.Fatalf("the lane chore waits for its horizon: %v", err)
414 }
415 moves, ok := f.chores(t)["moves/"+evHost]
416 if !ok || moves.Incarnation != f.inc || !moves.EvidenceAt.Equal(evNow) {
417 t.Fatalf("chores = %+v; want the moves/<host> chore at the listing's end", f.chores(t))
418 }
419}
420
421// TestVerifyRetirementVouchesOnlyForWhatTheMoveRetired (fix round 1,
422// ruling 1): the close is evidence only for what the move retired. At the
423// live location that is a generation superseded at or before the move's
424// own admission, every version of it older than the close. A generation
425// admitted from the move's admission on was written past the close, so the
426// provider has no evidence for it and says so with a typed error; the
427// moves lane proof is unaffected.
428func TestVerifyRetirementVouchesOnlyForWhatTheMoveRetired(t *testing.T) {
429 answers := func(t *testing.T, f *evFixture) {
430 t.Helper()
431 inc, at, err := f.provider().VerifyRetirement(t.Context(), f.observe(t), f.census(t))
432 if err != nil || inc != f.inc || !at.Equal(evAt) {
433 t.Fatalf("VerifyRetirement = %q, %v, %v; want %q, %v", inc, at, err, f.inc, evAt)
434 }
435 }
436 notEvidence := func(t *testing.T, f *evFixture, gen string) {
437 t.Helper()
438 obs, census := f.observe(t), f.census(t)
439 inc, at, err := f.provider().VerifyRetirement(t.Context(), obs, census)
440 var ne *NotMovesEvidence
441 if !errors.As(err, &ne) || inc != "" || !at.IsZero() {
442 t.Fatalf("VerifyRetirement = %q, %v, %v; want a *NotMovesEvidence and no evidence", inc, at, err)
443 }
444 if ne.Host != evHost || ne.Generation != gen || ne.Reason == "" {
445 t.Fatalf("refusal = %+v, want generation %q", ne, gen)
446 }
447 if _, err := f.provider().VerifyMovesSweep(t.Context(), obs, census); err != nil {
448 t.Fatalf("the lane proof does not depend on the generations: %v", err)
449 }
450 }
451 t.Run("the current generation and the marker", func(t *testing.T) {
452 f := newEvFixture(t)
453 f.bucket.Put("data", f.genKey(evMoveID), []byte("live"))
454 f.bucket.Put("data", f.location+"/generation.json", []byte("marker"))
455 answers(t, f)
456 })
457 t.Run("a generation the move's admission superseded", func(t *testing.T) {
458 f := newEvFixture(t)
459 f.bucket.Put("data", f.genKey("old"), []byte("old"))
460 f.bucket.Put("data", f.location+"/files/g-old/a.txt", []byte("old"))
461 answers(t, f)
462 })
463 t.Run("the move's generation superseded after the close", func(t *testing.T) {
464 f := newEvFixture(t)
465 f.setClaim(t, func(c *hostclaim.Claim) {
466 c.Holder.Admissions, c.Holder.Generation, c.Holder.Session, c.Holder.Stamp = evAdmissions(evMoveID, "later"), "later", 3, 3
467 })
468 f.bucket.Put("data", f.genKey(evMoveID), []byte("superseded after the close"))
469 f.bucket.Put("data", f.genKey("later"), []byte("live"))
470 notEvidence(t, f, evMoveID)
471 })
472 t.Run("a later generation superseded after the close", func(t *testing.T) {
473 f := newEvFixture(t)
474 f.setClaim(t, func(c *hostclaim.Claim) {
475 c.Holder.Admissions, c.Holder.Generation, c.Holder.Session, c.Holder.Stamp = evAdmissions(evMoveID, "later", "latest"), "latest", 4, 4
476 })
477 f.bucket.Put("data", f.genKey("later"), []byte("superseded after the close"))
478 notEvidence(t, f, "later")
479 })
480 t.Run("no admission of the move in the claim", func(t *testing.T) {
481 f := newEvFixture(t)
482 f.setClaim(t, func(c *hostclaim.Claim) {
483 c.Holder.Admissions, c.Holder.Generation = evAdmissions("fresh"), "fresh"
484 })
485 f.bucket.Put("data", f.genKey("old"), []byte("old"))
486 notEvidence(t, f, "old")
487 })
488 t.Run("a retired generation written after the close", func(t *testing.T) {
489 f := newEvFixture(t)
490 f.rec.LastMove.At = time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC)
491 f.bucket.Put("data", f.genKey("old"), []byte("late"))
492 notEvidence(t, f, "old")
493 })
494 t.Run("an object no generation owns", func(t *testing.T) {
495 f := newEvFixture(t)
496 f.bucket.Put("data", f.location+"/stray.bin", []byte("?"))
497 notEvidence(t, f, "")
498 })
499}
500
501// TestMovesSweepProofRefusesALiveMove (K10, M8): no proof while a move op
502// is on the claim, while the record carries a live move, or while the
503// claim was released by a move the record has not closed.
504func TestMovesSweepProofRefusesALiveMove(t *testing.T) {
505 t.Run("the record carries a live move", func(t *testing.T) {
506 f := newEvFixture(t)
507 f.rec.Move = &provision.MoveState{ID: "mv_next", Incarnation: f.inc, To: "uk", State: provision.MovePreparing}
508 f.lane("mv_next", "move.json")
509 bothRefuse(t, f, false)
510 })
511 for _, op := range []hostclaim.Op{
512 {Kind: hostclaim.KindMoving, ID: "mv_next", Phase: hostclaim.PhaseMoving, Exec: "box"},
513 {Kind: hostclaim.KindLanding, ID: evMoveID, Phase: hostclaim.PhaseCommitted, Exec: "box"},
514 {Kind: hostclaim.KindRestoring, ID: evMoveID, Phase: hostclaim.PhaseOpened, Exec: "box"},
515 {Kind: hostclaim.KindReverting, ID: evMoveID, Phase: hostclaim.PhaseOpened, Exec: "box"},
516 } {
517 // The last three are the record-first close's window (M4): the
518 // record closed the move, the claim has not released it yet.
519 t.Run("the claim holds "+op.Kind, func(t *testing.T) {
520 f := newEvFixture(t)
521 f.setClaim(t, func(c *hostclaim.Claim) { c.Op = op })
522 bothRefuse(t, f, false)
523 })
524 }
525 t.Run("the record names no closed move", func(t *testing.T) {
526 f := newEvFixture(t)
527 f.rec.LastMove = nil
528 bothRefuse(t, f, false)
529 })
530}
531
532// TestMovesProofSurvivesAMetadataPlacement (fix round 1, ruling 2): the
533// durable proof is the record's LastMove, bound to the holder's
534// incarnation, with no move op on the claim. A later operation that
535// replaces the serving op (here CompleteMetadataPlacement, which drops the
536// released_move journal) leaves cleanup running, and so does a serving op
537// released by a move whose record never carried it.
538func TestMovesProofSurvivesAMetadataPlacement(t *testing.T) {
539 t.Run("a metadata placement", func(t *testing.T) {
540 f := newEvFixture(t)
541 f.lane(evMoveID, "move.json")
542 f.bucket.Put("data", f.genKey(evMoveID), []byte("live"))
543 f.setClaim(t, func(c *hostclaim.Claim) {
544 if err := hostops.BeginPlacement("pl_1", "op@example", "meta", evNow, []string{"ie"}, []string{"box"})(c); err != nil {
545 t.Fatal(err)
546 }
547 c.Op.Intent["app_id"] = "app_1"
548 if err := hostops.CompleteMetadataPlacement("pl_1", "app_1")(c); err != nil {
549 t.Fatal(err)
550 }
551 })
552 obs := f.observe(t)
553 if obs.Claim.Op.Kind != hostclaim.KindServing || obs.Claim.Op.Journal["released_move"] != "" {
554 t.Fatalf("the placement left op %+v", obs.Claim.Op)
555 }
556 census := f.census(t)
557 proof, err := f.provider().VerifyMovesSweep(t.Context(), obs, census)
558 if err != nil || proof.MoveID != evMoveID || !proof.RetiredAt.Equal(evAt) {
559 t.Fatalf("proof = %+v, %v", proof, err)
560 }
561 if inc, at, err := f.provider().VerifyRetirement(t.Context(), obs, census); err != nil || inc != f.inc || !at.Equal(evAt) {
562 t.Fatalf("VerifyRetirement = %q, %v, %v", inc, at, err)
563 }
564 if err := f.worker(t, evNow).Run(t.Context()); !errors.Is(err, park.ErrPending) {
565 t.Fatalf("the lane chore waits for its horizon: %v", err)
566 }
567 if _, ok := f.chores(t)["moves/"+evHost]; !ok {
568 t.Fatalf("chores = %+v; want the moves/<host> chore", f.chores(t))
569 }
570 })
571 t.Run("a release by a move the record never carried", func(t *testing.T) {
572 f := newEvFixture(t)
573 f.setClaim(t, func(c *hostclaim.Claim) { c.Op = servingReleasedBy("mv_orphan") })
574 if _, err := f.provider().VerifyMovesSweep(t.Context(), f.observe(t), f.census(t)); err != nil {
575 t.Fatalf("no lane of the orphan, so nothing unproven: %v", err)
576 }
577 f.bucket.Put("data", "moves/"+evHost+"/mv_orphan/move.json", []byte("orphan"))
578 f.rec.LastMove.At = time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC)
579 bothRefuse(t, f, true)
580 })
581}
582
583// TestMovesEvidenceRefusesAZeroTime (M7, M8): a zero time is never
584// evidence: not the close's, and not a lane object's.
585func TestMovesEvidenceRefusesAZeroTime(t *testing.T) {
586 t.Run("a zero close", func(t *testing.T) {
587 f := newEvFixture(t)
588 f.rec.LastMove.At = time.Time{}
589 bothRefuse(t, f, true)
590 })
591 t.Run("a close not before the provider's clock", func(t *testing.T) {
592 f := newEvFixture(t)
593 f.rec.LastMove.At = evNow
594 bothRefuse(t, f, true)
595 })
596 t.Run("an older lane object with no time", func(t *testing.T) {
597 f := newEvFixture(t)
598 obs := f.observe(t)
599 census := []lease.ObjectVersion{{Key: "moves/" + evHost + "/mv_older/move.json", VersionID: "v1", IsLatest: true}}
600 if _, err := f.provider().VerifyMovesSweep(t.Context(), obs, census); err == nil {
601 t.Fatal("a zero LastModified counted as before the close")
602 } else {
603 wantRefusal(t, err, true)
604 }
605 if _, _, err := f.provider().VerifyRetirement(t.Context(), obs, census); err == nil {
606 t.Fatal("VerifyRetirement answered over a lane with no time")
607 }
608 })
609}
610
611// TestMovesSweepProofRefusesAnUnknownLane (M8): every moves/<host>/<op>/
612// object is the closed move's own, or older than its close. An object of
613// another op written after the close has an unknown writer.
614func TestMovesSweepProofRefusesAnUnknownLane(t *testing.T) {
615 t.Run("an older lane is covered by the close", func(t *testing.T) {
616 f := newEvFixture(t)
617 f.lane("mv_older", "move.json")
618 f.lane("rb_return", "replica/0000/db")
619 if _, err := f.provider().VerifyMovesSweep(t.Context(), f.observe(t), f.census(t)); err != nil {
620 t.Fatal(err)
621 }
622 })
623 t.Run("a lane written after the close", func(t *testing.T) {
624 f := newEvFixture(t)
625 f.rec.LastMove.At = time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC) // before every write
626 f.lane(evMoveID, "move.json")
627 if _, err := f.provider().VerifyMovesSweep(t.Context(), f.observe(t), f.census(t)); err != nil {
628 t.Fatalf("the closed move's own lane is its own: %v", err)
629 }
630 f.lane("mv_unknown", "move.json")
631 bothRefuse(t, f, true)
632 })
633 t.Run("an object outside any lane", func(t *testing.T) {
634 f := newEvFixture(t)
635 f.bucket.Put("data", "moves/"+evHost+"/stray.json", []byte("?"))
636 bothRefuse(t, f, true)
637 })
638 t.Run("an object outside the host's namespaces", func(t *testing.T) {
639 f := newEvFixture(t)
640 census := []lease.ObjectVersion{{Key: "moves/other.example/" + evMoveID + "/move.json", VersionID: "v1", LastModified: evAt.Add(-time.Hour)}}
641 if _, err := f.provider().VerifyMovesSweep(t.Context(), f.observe(t), census); err == nil {
642 t.Fatal("another host's object counted")
643 }
644 })
645}
646
647// TestMovesEvidenceRefusesAnotherIdentity: the record must be the holder's
648// exact record, and a holderless claim has no record to prove from.
649func TestMovesEvidenceRefusesAnotherIdentity(t *testing.T) {
650 t.Run("a record of another incarnation", func(t *testing.T) {
651 f := newEvFixture(t)
652 f.rec.CreatedAt = evCreated.Add(time.Hour)
653 bothRefuse(t, f, true)
654 })
655 t.Run("a stale observation", func(t *testing.T) {
656 f := newEvFixture(t)
657 obs := f.observe(t)
658 obs.ETag = ""
659 if _, err := f.provider().VerifyMovesSweep(t.Context(), obs, f.census(t)); err == nil {
660 t.Fatal("answered without an etag")
661 }
662 })
663 t.Run("a free claim", func(t *testing.T) {
664 f := newEvFixture(t)
665 f.setClaim(t, func(c *hostclaim.Claim) { c.Holder, c.Op = nil, hostclaim.Op{} })
666 obs, census := f.observe(t), f.census(t)
667 _, err := f.provider().VerifyMovesSweep(t.Context(), obs, census)
668 wantRefusal(t, err, false)
669 if _, _, err := f.provider().VerifyRetirement(t.Context(), obs, census); err == nil {
670 t.Fatal("VerifyRetirement answered with no holder")
671 } else {
672 wantRefusal(t, err, false)
673 }
674 if f.reads != 0 {
675 t.Fatalf("read %d records with no holder to name one", f.reads)
676 }
677 })
678 t.Run("no providers", func(t *testing.T) {
679 f := newEvFixture(t)
680 if _, err := (movesRetirementEvidence{}).VerifyMovesSweep(t.Context(), f.observe(t), nil); err == nil {
681 t.Fatal("answered with no record reader")
682 }
683 if _, _, err := (movesRetirementEvidence{Records: f.provider().Records}).VerifyRetirement(t.Context(), f.observe(t), nil); err == nil {
684 t.Fatal("answered with no clock")
685 }
686 })
687}
688
689// The provider satisfies both slots RegisterProductionClaimMoveParks reads:
690// the regional evidence and, by type assertion, the sweep verifier.
691var (
692 _ hostbox.RegionalEvidence = movesRetirementEvidence{}
693 _ park.MoveSweepVerifier = movesRetirementEvidence{}
694)
695
696// --- against the claim's regional worker, with a fake store ---
697
698type evLease struct{}
699
700func (evLease) Check(context.Context) error { return nil }
701func (evLease) Release(context.Context) error { return nil }
702func (evLease) Close() {}
703
704type evLeases struct{}
705
706func (evLeases) Acquire(context.Context, string, string) (park.HeldLease, error) {
707 return evLease{}, nil
708}
709
710type evEnrollment struct{}
711
712func (evEnrollment) VerifyRegional(context.Context, string, string, time.Duration) error { return nil }
713
714// worker is hostbox.RegionalWorker over park.Engine, wired as the claim's
715// assembly wires them (host/claimassembly.go): the provider is the worker's
716// Evidence and the engine's VerifyMovesSweep. This region (ie) is not the
717// authoritative one, so the holder's location here is residue too.
718func (f *evFixture) worker(t *testing.T, listing time.Time) hostbox.RegionalWorker {
719 t.Helper()
720 seq := 0
721 p := f.provider()
722 location := f.location
723 engine := park.Engine{
724 Store: f.bucket, Claims: f.claims, Bucket: "data", Region: "ie", Leases: evLeases{},
725 Now: func() time.Time { return listing },
726 NewID: func() string { seq++; return fmt.Sprintf("id%d", seq) },
727 View: func(_ context.Context, o hostclaim.Observed) (park.View, error) {
728 return park.View{ClaimETag: o.ETag, Region: "ie", AuthoritativeRegion: "uk",
729 GenerationOf: func(r park.Ref) (string, bool) { return landingGenerationOf(location, r.Key) }}, nil
730 },
731 Authorize: func(context.Context, hostclaim.Observed, hostclaim.Obligation) error { return nil },
732 VerifyMovesSweep: p.VerifyMovesSweep,
733 }
734 return hostbox.RegionalWorker{
735 Regional: hostbox.Regional{Engine: engine, Owner: func(context.Context, hostclaim.Observed) error { return nil }},
736 Enrollment: evEnrollment{},
737 Evidence: p,
738 }
739}
740
741func (f *evFixture) chores(t *testing.T) map[string]hostclaim.ParkScope {
742 t.Helper()
743 out := map[string]hostclaim.ParkScope{}
744 for _, o := range f.observe(t).Claim.Obligations {
745 if o.Park == nil {
746 t.Fatalf("an unscoped chore: %+v", o)
747 }
748 out[o.Park.Location] = *o.Park
749 }
750 return out
751}
752
753// TestAllowedSweepQueuesTheMovesChore: with the proof, the worker queues the
754// moves/<host> chore under the serving session, labelled with the holder's
755// incarnation, its evidence the listing's end, so it waits for its horizon.
756// The holder location's residue in this non-authoritative region takes the
757// close as its evidence: that horizon has passed, so the same pass parks it.
758func TestAllowedSweepQueuesTheMovesChore(t *testing.T) {
759 f := newEvFixture(t)
760 f.lane(evMoveID, "move.json")
761 f.lane("mv_older", "replica/0000/db")
762 left := f.genKey("old") // superseded by the move's own admission
763 f.bucket.Put("data", left, []byte("left behind"))
764 err := f.worker(t, evNow).Run(t.Context())
765 if !errors.Is(err, park.ErrPending) {
766 t.Fatalf("a new sweep waits for its horizon: %v", err)
767 }
768 chores := f.chores(t)
769 moves, ok := chores["moves/"+evHost]
770 if !ok || len(chores) != 1 {
771 t.Fatalf("chores = %+v; want the moves chore alone", chores)
772 }
773 if moves.Incarnation != f.inc || !moves.EvidenceAt.Equal(evNow) || moves.RequestLimit != hostbox.RequestLimit {
774 t.Fatalf("moves chore = %+v; want incarnation %s, evidence %v", moves, f.inc, evNow)
775 }
776 if vs, err := f.bucket.ListVersions(t.Context(), "data", left); err != nil || len(vs) != 0 {
777 t.Fatalf("the location's residue was not parked: %v, %v", vs, err)
778 }
779 if vs, err := f.bucket.ListVersions(t.Context(), "data", "parked/"+f.inc+"/"+evHost+"/"); err != nil || len(vs) == 0 {
780 t.Fatalf("no archive under the holder's incarnation: %v, %v", vs, err)
781 }
782 for _, o := range f.observe(t).Claim.Obligations {
783 if o.Op != "session:2" || o.Owner != "region:ie" {
784 t.Fatalf("chore %+v is not the serving session's regional chore", o)
785 }
786 }
787 if c := f.observe(t).Claim; c.Op.Kind != hostclaim.KindServing || c.Op.Journal["released_move"] != evMoveID {
788 t.Fatalf("the sweep changed the serving op: %+v", c.Op)
789 }
790}
791
792// TestSweepSkipsWhileAMoveStages (Review Focus 3, K10): while a move op is
793// on the claim the worker never asks for evidence; while the record carries
794// a live move under a serving claim the provider refuses and nothing is
795// queued.
796func TestSweepSkipsWhileAMoveStages(t *testing.T) {
797 t.Run("a moving claim", func(t *testing.T) {
798 f := newEvFixture(t)
799 f.setClaim(t, func(c *hostclaim.Claim) {
800 c.Op = hostclaim.Op{Kind: hostclaim.KindMoving, ID: "mv_next", Phase: hostclaim.PhaseMoving, Exec: "box"}
801 })
802 f.rec.Move = &provision.MoveState{ID: "mv_next", Incarnation: f.inc, To: "uk", State: provision.MovePreparing}
803 f.lane("mv_next", "replica/0000/db")
804 if err := f.worker(t, evNow).Run(t.Context()); err != nil {
805 t.Fatal(err)
806 }
807 if len(f.chores(t)) != 0 || f.reads != 0 {
808 t.Fatalf("chores %+v, record reads %d", f.chores(t), f.reads)
809 }
810 })
811 t.Run("a live move on the record", func(t *testing.T) {
812 f := newEvFixture(t)
813 f.rec.Move = &provision.MoveState{ID: "mv_next", Incarnation: f.inc, To: "uk", State: provision.MovePreparing}
814 f.lane("mv_next", "replica/0000/db")
815 err := f.worker(t, evNow).Run(t.Context())
816 var r *MovesSweepRefusal
817 if !errors.As(err, &r) {
818 t.Fatalf("err = %v, want the provider's refusal", err)
819 }
820 if len(f.chores(t)) != 0 {
821 t.Fatalf("chores %+v", f.chores(t))
822 }
823 })
824}
825
826// TestSweepRefusedUnlessTheCloseIsBeforeTheListing (K10): the worker's own
827// Check drops the moves/<host> candidate when the newest retired move's
828// close is not earlier than the listing; the rest of the decision stands.
829func TestSweepRefusedUnlessTheCloseIsBeforeTheListing(t *testing.T) {
830 f := newEvFixture(t)
831 f.lane(evMoveID, "move.json")
832 f.bucket.Put("data", f.genKey("old"), []byte("left behind"))
833 err := f.worker(t, evAt).Run(t.Context())
834 if !errors.Is(err, park.ErrPending) {
835 t.Fatalf("the location chore still queues: %v", err)
836 }
837 chores := f.chores(t)
838 if _, ok := chores["moves/"+evHost]; ok || len(chores) != 1 {
839 t.Fatalf("chores = %+v; want only the location's", chores)
840 }
841 if loc := chores[f.location]; loc.Incarnation != f.inc || !loc.EvidenceAt.Equal(evAt) {
842 t.Fatalf("location chore = %+v; want the close %v as its evidence", loc, evAt)
843 }
844}
845
846// TestSweepScopeMembership (witness wSweepScopeMembership, M7): residue of
847// an older incarnation is never the moves/<host> chore's. The provider no
848// longer refuses the sweep for it (Q4): the engine queues it as its own
849// chore under its own qualified identity and parks it, and the moves/<host>
850// chore keeps the holder's.
851func TestSweepScopeMembership(t *testing.T) {
852 f := newEvFixture(t)
853 older := strings.Repeat("cd", 32)
854 f.lane(evMoveID, "move.json")
855 residue := "instances/" + evHost + "/" + older + "/g-old/0000/db"
856 f.bucket.Put("data", residue, []byte("late"))
857 if err := f.worker(t, evNow).Run(t.Context()); !errors.Is(err, park.ErrPending) {
858 t.Fatalf("the chores wait for their horizons: %v", err)
859 }
860 chores := f.chores(t)
861 moves, ok := chores["moves/"+evHost]
862 if !ok || moves.Incarnation != f.inc {
863 t.Fatalf("chores = %+v; want the moves/<host> chore under the holder's incarnation", chores)
864 }
865 if _, ok := chores["instances/"+evHost+"/"+older]; ok || len(chores) != 1 {
866 t.Fatalf("chores = %+v; want only the moves/<host> chore pending", chores)
867 }
868 // The older incarnation's chore ran first, under its own identity, and
869 // parked its residue in the same pass.
870 if _, ok := f.bucket.Current("data", residue); ok {
871 t.Fatal("the older incarnation's residue was not parked")
872 }
873}
874
875// TestFreeHostLanesAreTheClaimsProof (fix round 1, ruling 3; Q4): a free
876// claim names no record, so moves can prove no closed move and the claim
877// stays free. The free-host lane proof is the claim's own
878// (claimFreeMoveProof, asked before the provider); a free claim that
879// reaches the provider gets the quiet sweep refusal, not a dependency.
880func TestFreeHostLanesAreTheClaimsProof(t *testing.T) {
881 f := newEvFixture(t)
882 f.setClaim(t, func(c *hostclaim.Claim) { c.Holder, c.Op, c.Touched = nil, hostclaim.Op{}, hostclaim.Touched{} })
883 f.lane(evMoveID, "move.json")
884 err := f.worker(t, evNow).Run(t.Context())
885 wantRefusal(t, err, false)
886 if c := f.observe(t).Claim; !c.Free() || len(c.Obligations) != 0 {
887 t.Fatalf("claim = %+v", c)
888 }
889}
890
891// authoritative makes ie the authoritative region, with the marker at the
892// holder's current generation and objects classified by the landing's
893// layout, as Task 9a's view will.
894func (f *evFixture) authoritative(t *testing.T, w *hostbox.RegionalWorker, current string) {
895 t.Helper()
896 marker := f.location + "/generation.json"
897 ref := park.Ref{Key: marker, VersionID: f.bucket.LatestID("data", marker)}
898 location := f.location
899 w.Regional.Engine.View = func(_ context.Context, o hostclaim.Observed) (park.View, error) {
900 return park.View{ClaimETag: o.ETag, Region: "ie", AuthoritativeRegion: "ie", MarkerKnown: true, MarkerGeneration: current, Marker: ref,
901 GenerationOf: func(r park.Ref) (string, bool) { return landingGenerationOf(location, r.Key) }}, nil
902 }
903}
904
905// TestAuthoritativeSweepStampsTheCloseOnlyOnWhatTheMoveRetired (fix round 1,
906// ruling 1): in the authoritative region the engine makes superseded
907// generations park-generation candidates and binds them to
908// VerifyRetirement's time. A generation the move's admission superseded
909// gets the close; when the move's own generation was superseded after the
910// close, the provider refuses and nothing is queued.
911func TestAuthoritativeSweepStampsTheCloseOnlyOnWhatTheMoveRetired(t *testing.T) {
912 listing := evAt.Add(time.Minute) // before the close's horizon, so chores stay queued
913 t.Run("superseded by the move", func(t *testing.T) {
914 f := newEvFixture(t)
915 f.lane(evMoveID, "move.json")
916 f.bucket.Put("data", f.genKey(evMoveID), []byte("live"))
917 f.bucket.Put("data", f.location+"/generation.json", []byte("marker"))
918 f.bucket.Put("data", f.genKey("old"), []byte("superseded by the move"))
919 w := f.worker(t, listing)
920 f.authoritative(t, &w, evMoveID)
921 if err := w.Run(t.Context()); !errors.Is(err, park.ErrPending) {
922 t.Fatalf("the chores wait for their horizons: %v", err)
923 }
924 var gens []hostclaim.Obligation
925 for _, o := range f.observe(t).Claim.Obligations {
926 if o.Kind == "park-generation" {
927 gens = append(gens, o)
928 }
929 }
930 if len(gens) != 1 || gens[0].Generation != "old" || gens[0].Park.Location != f.location || !gens[0].Park.EvidenceAt.Equal(evAt) {
931 t.Fatalf("generation chores = %+v; want one for old at the close %v", gens, evAt)
932 }
933 if moves, ok := f.chores(t)["moves/"+evHost]; !ok || !moves.EvidenceAt.Equal(listing) {
934 t.Fatalf("chores = %+v; want the moves chore at the listing", f.chores(t))
935 }
936 })
937 t.Run("superseded after the close", func(t *testing.T) {
938 f := newEvFixture(t)
939 f.setClaim(t, func(c *hostclaim.Claim) {
940 c.Holder.Admissions, c.Holder.Generation, c.Holder.Session, c.Holder.Stamp = evAdmissions(evMoveID, "later"), "later", 3, 3
941 })
942 f.lane(evMoveID, "move.json")
943 f.bucket.Put("data", f.genKey("later"), []byte("live"))
944 f.bucket.Put("data", f.location+"/generation.json", []byte("marker"))
945 f.bucket.Put("data", f.genKey(evMoveID), []byte("written until after the close"))
946 w := f.worker(t, listing)
947 f.authoritative(t, &w, "later")
948 err := w.Run(t.Context())
949 var ne *NotMovesEvidence
950 if !errors.As(err, &ne) || ne.Generation != evMoveID {
951 t.Fatalf("err = %v, want a *NotMovesEvidence for %s", err, evMoveID)
952 }
953 if len(f.observe(t).Claim.Obligations) != 0 {
954 t.Fatalf("chores %+v", f.observe(t).Claim.Obligations)
955 }
956 })
957}
958
959// TestMovesParkLockMatchesTheEngine (fix round 2): the provider mirrors the
960// park engine's unexported lock predicate (park's isLock over locationOf),
961// so it skips exactly the lease objects Decide skips. park.Decide is the
962// oracle: over a one-key census in a non-authoritative region, where every
963// non-lock object is a candidate, it decides nothing exactly for a lock.
964func TestMovesParkLockMatchesTheEngine(t *testing.T) {
965 f := newEvFixture(t)
966 obs := f.observe(t)
967 view := park.View{ClaimETag: obs.ETag, Region: "ie", AuthoritativeRegion: "uk",
968 GenerationOf: func(r park.Ref) (string, bool) { return landingGenerationOf(f.location, r.Key) }}
969 other := strings.Repeat("ab", 32)
970 for _, key := range []string{
971 "instances/" + evHost + "/lock.json",
972 "instances/" + evHost + "/" + f.inc + "/lock.json",
973 "instances/" + evHost + "/" + other + "/lock.json",
974 "moves/" + evHost + "/lock.json",
975 "moves/" + evHost + "/" + evMoveID + "/lock.json",
976 "instances/" + evHost + "/g-old/lock.json",
977 "instances/" + evHost + "/" + f.inc + "/g-old/lock.json",
978 "instances/" + evHost + "/lock.json.old",
979 "instances/" + evHost + "/0000/db",
980 } {
981 d, err := park.Decide(obs, view, []lease.ObjectVersion{{Key: key, VersionID: "v1", LastModified: evAt.Add(-time.Hour)}}, f.inc)
982 if err != nil {
983 t.Fatalf("%s: %v", key, err)
984 }
985 if engine, mirror := len(d.Candidates) == 0, movesParkLock(evHost, key); engine != mirror {
986 t.Errorf("%s: the engine skips it %v, the provider's mirror says lock %v", key, engine, mirror)
987 }
988 }
989}
990
991// TestLiveLocationLeaseIsSweptNormally (fix round 2): the writer's lease,
992// <holder.location>/lock.json (lease.Acquire), stands at every live
993// location. It is the engine's to skip, never residue the close must
994// vouch for, so the sweep proceeds, scoped and bare alike.
995func TestLiveLocationLeaseIsSweptNormally(t *testing.T) {
996 for name, bare := range map[string]bool{"scoped": false, "bare": true} {
997 t.Run(name, func(t *testing.T) {
998 f := newEvFixture(t)
999 if bare {
1000 f.bare(t)
1001 }
1002 f.lane(evMoveID, "move.json")
1003 f.bucket.Put("data", f.genKey(evMoveID), []byte("live"))
1004 f.bucket.Put("data", f.location+"/generation.json", []byte("marker"))
1005 f.bucket.Put("data", f.location+"/lock.json", []byte("writer lease"))
1006 if _, _, err := f.provider().VerifyRetirement(t.Context(), f.observe(t), f.census(t)); err != nil {
1007 t.Fatalf("VerifyRetirement refused the writer's lease: %v", err)
1008 }
1009 listing := evAt.Add(time.Minute)
1010 w := f.worker(t, listing)
1011 f.authoritative(t, &w, evMoveID)
1012 if err := w.Run(t.Context()); !errors.Is(err, park.ErrPending) {
1013 t.Fatalf("the lane chore waits for its horizon: %v", err)
1014 }
1015 chores := f.chores(t)
1016 if moves, ok := chores["moves/"+evHost]; !ok || len(chores) != 1 || !moves.EvidenceAt.Equal(listing) {
1017 t.Fatalf("chores = %+v; want the moves chore alone", chores)
1018 }
1019 if vs, err := f.bucket.ListVersions(t.Context(), "data", f.location+"/lock.json"); err != nil || len(vs) != 1 {
1020 t.Fatalf("the lease was touched: %v, %v", vs, err)
1021 }
1022 })
1023 }
1024}
1025