amadan
host: moves evidence fix round 2 (the writer's lease is not residue)
retiredGenerations classified <holder.location>/lock.json, the writer's lease that stands at every live location, as an object no generation owns, so every live host with a lease was refused. The provider now skips exactly the lock objects the park engine skips: movesParkLock mirrors park's unexported isLock over locationOf (the lock.json at the root of its own park location), and also replaces the moves/<host> root special case. TestMovesParkLockMatchesTheEngine keeps the mirror in step with park.Decide as the oracle; TestLiveLocationLeaseIsSweptNormally sweeps a live location holding the lease, scoped and bare.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2 files changed,
+93
−5
internal/host/movesevidence.go+26 −5internal/host/movesevidence_test.go+67 −0
diff --git a/internal/host/movesevidence.go b/internal/host/movesevidence.go| index 149392519..20d7ee717 100644 |
| --- a/internal/host/movesevidence.go |
| +++ b/internal/host/movesevidence.go |
| @@ -157,7 +157,9 @@ func (e movesRetirementEvidence) VerifyRetirement(ctx context.Context, o hostcla |
| // order and carry no time), and every version of it must predate the close. |
| // A rolled-back or aborted move, or a landing that admitted a fresh name, |
| // leaves no such admission, so no generation is vouched for. Objects are |
| -// classified as the landing's view classifies them (landingGenerationOf). |
| +// classified as the landing's view classifies them (landingGenerationOf), |
| +// after the lock objects the engine itself skips (movesParkLock, fix round |
| +// 2): the writer's lease stands at every live location. |
| func (r movesRetired) retiredGenerations(versions []lease.ObjectVersion) error { |
| h := r.holder |
| refuse := func(gen, format string, args ...any) error { |
| @@ -177,7 +179,7 @@ func (r movesRetired) retiredGenerations(versions []lease.ObjectVersion) error { |
| } |
| } |
| for _, v := range versions { |
| - if loc, ok := movesInstancesLocation(r.host, v.Key); !ok || loc != r.location { |
| + if loc, ok := movesInstancesLocation(r.host, v.Key); !ok || loc != r.location || movesParkLock(r.host, v.Key) { |
| continue |
| } |
| gen, ok := landingGenerationOf(r.location, v.Key) |
| @@ -276,11 +278,11 @@ func (e movesRetirementEvidence) retired(ctx context.Context, o hostclaim.Observ |
| if !ok { |
| return refuse(true, "the census holds %s, outside this host", v.Key) |
| } |
| + if movesParkLock(host, v.Key) { |
| + continue // the moves/<host> park's own lease |
| + } |
| op, _, inLane := strings.Cut(tail, "/") |
| if !inLane { |
| - if tail == "lock.json" { |
| - continue // the moves/<host> park's own lease |
| - } |
| return refuse(true, "%s is outside every lane", v.Key) |
| } |
| if op == last.ID { |
| @@ -292,3 +294,22 @@ func (e movesRetirementEvidence) retired(ctx context.Context, o hostclaim.Observ |
| } |
| return movesRetired{host: host, incarnation: h.Incarnation, location: location, moveID: last.ID, at: last.At.UTC(), holder: *h}, nil |
| } |
| + |
| +// movesParkLock mirrors the park engine's lock predicate (park's isLock over |
| +// locationOf, both unexported): a key is a lock exactly when it is the |
| +// lock.json at the root of its own park location, instances/<host>, |
| +// instances/<host>/<incarnation> or moves/<host>. That is the writer's |
| +// lease at a holder location (lease.Acquire) and a park's own lease. |
| +// Decide never makes it a candidate, so the close need not vouch for it. |
| +// TestMovesParkLockMatchesTheEngine keeps the two in step through |
| +// park.Decide. |
| +func movesParkLock(host, key string) bool { |
| + loc, ok := movesInstancesLocation(host, key) |
| + if !ok { |
| + if !strings.HasPrefix(key, "moves/"+host+"/") { |
| + return false |
| + } |
| + loc = "moves/" + host |
| + } |
| + return key == loc+"/lock.json" |
| +} |
diff --git a/internal/host/movesevidence_test.go b/internal/host/movesevidence_test.go| index 67a9e87e4..b6c3153b0 100644 |
| --- a/internal/host/movesevidence_test.go |
| +++ b/internal/host/movesevidence_test.go |
| @@ -828,3 +828,70 @@ func TestAuthoritativeSweepStampsTheCloseOnlyOnWhatTheMoveRetired(t *testing.T) |
| } |
| }) |
| } |
| + |
| +// TestMovesParkLockMatchesTheEngine (fix round 2): the provider mirrors the |
| +// park engine's unexported lock predicate (park's isLock over locationOf), |
| +// so it skips exactly the lease objects Decide skips. park.Decide is the |
| +// oracle: over a one-key census in a non-authoritative region, where every |
| +// non-lock object is a candidate, it decides nothing exactly for a lock. |
| +func TestMovesParkLockMatchesTheEngine(t *testing.T) { |
| + f := newEvFixture(t) |
| + obs := f.observe(t) |
| + view := park.View{ClaimETag: obs.ETag, Region: "ie", AuthoritativeRegion: "uk", |
| + GenerationOf: func(r park.Ref) (string, bool) { return landingGenerationOf(f.location, r.Key) }} |
| + other := strings.Repeat("ab", 32) |
| + for _, key := range []string{ |
| + "instances/" + evHost + "/lock.json", |
| + "instances/" + evHost + "/" + f.inc + "/lock.json", |
| + "instances/" + evHost + "/" + other + "/lock.json", |
| + "moves/" + evHost + "/lock.json", |
| + "moves/" + evHost + "/" + evMoveID + "/lock.json", |
| + "instances/" + evHost + "/g-old/lock.json", |
| + "instances/" + evHost + "/" + f.inc + "/g-old/lock.json", |
| + "instances/" + evHost + "/lock.json.old", |
| + "instances/" + evHost + "/0000/db", |
| + } { |
| + d, err := park.Decide(obs, view, []lease.ObjectVersion{{Key: key, VersionID: "v1", LastModified: evAt.Add(-time.Hour)}}, f.inc) |
| + if err != nil { |
| + t.Fatalf("%s: %v", key, err) |
| + } |
| + if engine, mirror := len(d.Candidates) == 0, movesParkLock(evHost, key); engine != mirror { |
| + t.Errorf("%s: the engine skips it %v, the provider's mirror says lock %v", key, engine, mirror) |
| + } |
| + } |
| +} |
| + |
| +// TestLiveLocationLeaseIsSweptNormally (fix round 2): the writer's lease, |
| +// <holder.location>/lock.json (lease.Acquire), stands at every live |
| +// location. It is the engine's to skip, never residue the close must |
| +// vouch for, so the sweep proceeds, scoped and bare alike. |
| +func TestLiveLocationLeaseIsSweptNormally(t *testing.T) { |
| + for name, bare := range map[string]bool{"scoped": false, "bare": true} { |
| + t.Run(name, func(t *testing.T) { |
| + f := newEvFixture(t) |
| + if bare { |
| + f.bare(t) |
| + } |
| + f.lane(evMoveID, "move.json") |
| + f.bucket.Put("data", f.genKey(evMoveID), []byte("live")) |
| + f.bucket.Put("data", f.location+"/generation.json", []byte("marker")) |
| + f.bucket.Put("data", f.location+"/lock.json", []byte("writer lease")) |
| + if _, _, err := f.provider().VerifyRetirement(t.Context(), f.observe(t), f.census(t)); err != nil { |
| + t.Fatalf("VerifyRetirement refused the writer's lease: %v", err) |
| + } |
| + listing := evAt.Add(time.Minute) |
| + w := f.worker(t, listing) |
| + f.authoritative(t, &w, evMoveID) |
| + if err := w.Run(t.Context()); !errors.Is(err, park.ErrPending) { |
| + t.Fatalf("the lane chore waits for its horizon: %v", err) |
| + } |
| + chores := f.chores(t) |
| + if moves, ok := chores["moves/"+evHost]; !ok || len(chores) != 1 || !moves.EvidenceAt.Equal(listing) { |
| + t.Fatalf("chores = %+v; want the moves chore alone", chores) |
| + } |
| + if vs, err := f.bucket.ListVersions(t.Context(), "data", f.location+"/lock.json"); err != nil || len(vs) != 1 { |
| + t.Fatalf("the lease was touched: %v, %v", vs, err) |
| + } |
| + }) |
| + } |
| +} |