amadan
moves: the files live set walks the whole live generation chain (Task 9a fix round 1, I2)
The view loaded files manifests only for holder.generation and the marker's generation, but park.protected also protects every generation of the unfinished cold admission chain back to the marker. An object only an intermediate generation's manifest named got no pin and could be swept. - park: LiveGenerations exports protected itself, so readers of the live set enumerate the engine's generations, not a parallel walk. - host: the view reads the manifest of every generation LiveGenerations names for the holder's location, plus the holder's and the marker's (alone when the engine refuses the walk, which refuses every park of the location too). Tests: TestFilesLiveSetFollowsTheLiveChain (engine: g3 -> g2 -> g1 with the marker at g1; an object only g2's manifest names is pinned and survives the sweep's park), TestLiveGenerationsWalksTheUnfinishedChain (park).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
4 files changed,
+124
−2
internal/host/movesfileslive_test.go+46 −0internal/host/movessources.go+25 −2internal/park/live_generations_test.go+39 −0internal/park/sweep.go+14 −0
diff --git a/internal/host/movesfileslive_test.go b/internal/host/movesfileslive_test.go| index 50023ee27..32e22729a 100644 |
| --- a/internal/host/movesfileslive_test.go |
| +++ b/internal/host/movesfileslive_test.go |
| @@ -248,3 +248,49 @@ func TestReturnResidueParkKeepsTheFilesLiveSet(t *testing.T) { |
| t.Fatalf("before the source's admission %+v, %v", v, err) |
| } |
| } |
| + |
| +// TestFilesLiveSetFollowsTheLiveChain (fix round 1, I2): with the marker |
| +// lagging, park.protected treats the whole unfinished chain g3 -> g2 -> g1 |
| +// as live, and the view reads every one of those generations' manifests. An |
| +// object only g2's manifest names, under the older generation g0, is pinned |
| +// and survives the sweep's park. |
| +func TestFilesLiveSetFollowsTheLiveChain(t *testing.T) { |
| + f := newEvFixture(t) |
| + f.setClaim(t, func(c *hostclaim.Claim) { |
| + c.Holder.Generation = "g3" |
| + c.Holder.Admissions = []hostclaim.AdmissionRecord{ |
| + {Session: 1, Box: "box", Generation: "g1", Mode: "cold", Region: "ie"}, |
| + {Session: 2, Box: "box", Generation: "g2", Predecessor: "g1", Mode: "cold", Region: "ie", PredecessorRegion: "ie"}, |
| + {Session: 3, Box: "box", Generation: "g3", Predecessor: "g2", Mode: "cold", Region: "ie", PredecessorRegion: "ie"}, |
| + } |
| + c.Holder.Stamp, c.Holder.Session = 3, 3 |
| + }) |
| + body, err := json.Marshal(map[string]any{"op": "g1", "incarnation": f.inc, "host": evHost, "box": "box", "installed_at": evCreated, "manifest_sha256": "x", "snapshot_txid": 1}) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + f.bucket.Put("data", f.location+"/generation.json", body) |
| + only := f.location + "/files/g-g0/objects/0000000000000001-aa" |
| + ver := f.bucket.Put("data", only, []byte("named by g2 alone")) |
| + other := f.location + "/files/g-g0/objects/0000000000000002-bb" |
| + f.bucket.Put("data", other, []byte("named by nobody")) |
| + filesManifest(t, func(k string, b []byte) { f.bucket.Put("data", k, b) }, f.location, "g2", "a.txt="+only) |
| + seq := 0 |
| + e := park.Engine{Store: f.bucket, Claims: f.claims, Bucket: "data", Region: "ie", Leases: evLeases{}, Now: func() time.Time { return evNow }, |
| + NewID: func() string { seq++; return fmt.Sprintf("id%d", seq) }, |
| + View: movesParkView(f.provider().Records, f.bucket, "data", "ie"), |
| + Authorize: func(context.Context, hostclaim.Observed, hostclaim.Obligation) error { return nil }} |
| + v, err := e.View(t.Context(), f.observe(t)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if !v.Pinned[park.Ref{Key: only, VersionID: ver}] { |
| + t.Fatalf("pins %v; want g2's object %s", v.Pinned, only) |
| + } |
| + if _, err := e.Sweep(t.Context(), evHost, f.inc, evAt, hostbox.RequestLimit); err != nil && !errors.Is(err, park.ErrPending) { |
| + t.Fatal(err) |
| + } |
| + if got, ok := f.bucket.Current("data", only); !ok || string(got) != "named by g2 alone" { |
| + t.Fatalf("g2's object did not survive the park: %q %v", got, ok) |
| + } |
| +} |
diff --git a/internal/host/movessources.go b/internal/host/movessources.go| index cbb19e6fd..c56663ef6 100644 |
| --- a/internal/host/movessources.go |
| +++ b/internal/host/movessources.go |
| @@ -37,6 +37,7 @@ import ( |
| "errors" |
| "fmt" |
| "os" |
| + "sort" |
| "strings" |
| "time" |
| @@ -134,17 +135,39 @@ func movesParkView(records MovesRecordReader, data genselect.Reader, bucket, reg |
| if err := readMarker(ctx, data, bucket, obs.Host, location, *h, listing, &v); err != nil { |
| return park.View{}, err |
| } |
| - if err := pinFilesLiveSet(ctx, data, bucket, location, listing, &v, h.Generation, v.MarkerGeneration); err != nil { |
| + if err := pinFilesLiveSet(ctx, data, bucket, location, listing, &v, liveGenerations(obs.Claim, v)...); err != nil { |
| return park.View{}, err |
| } |
| return v, nil |
| } |
| } |
| +// liveGenerations are the generations whose files manifests are live |
| +// (fix round 1, I2): the holder's and the marker's, and every generation |
| +// park.LiveGenerations, the engine's own protected set, names for the |
| +// holder's location, the unfinished admission chain included. When the |
| +// engine refuses that walk it refuses every park of the location too, so |
| +// the holder's and the marker's are enough then. Sorted, for a stable read |
| +// order. |
| +func liveGenerations(c *hostclaim.Claim, v park.View) []string { |
| + seen := map[string]bool{c.Holder.Generation: true, v.MarkerGeneration: true} |
| + if live, err := park.LiveGenerations(c, v, c.Holder.Location); err == nil { |
| + for g := range live { |
| + seen[g] = true |
| + } |
| + } |
| + out := make([]string, 0, len(seen)) |
| + for g := range seen { |
| + out = append(out, g) |
| + } |
| + sort.Strings(out) |
| + return out |
| +} |
| + |
| // pinFilesLiveSet pins the files live set (J2/F4, Paul's #236 ruling; M10): |
| // every exact version an object a live generation's files manifest names, |
| // read through hostfiles.Load by the version the listing names. The live |
| -// generations are the holder's and the marker's. The rule is M10's: spec |
| +// generations are liveGenerations'. The rule is M10's: spec |
| // §6.6 P6-D3 says a target's manifest never references a source-region |
| // object, so a manifest in this region's bucket names objects in this |
| // region's bucket, and a region pins only what its own live manifests name. |
diff --git a/internal/park/live_generations_test.go b/internal/park/live_generations_test.go| new file mode 100644 |
| index 000000000..055725b8c |
| --- /dev/null |
| +++ b/internal/park/live_generations_test.go |
| @@ -0,0 +1,39 @@ |
| +package park |
| + |
| +import ( |
| + "errors" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| +) |
| + |
| +// TestLiveGenerationsWalksTheUnfinishedChain: the exported live set is the |
| +// engine's own: with the marker lagging at g1 and the holder at g3, the |
| +// unfinished cold chain g3 -> g2 -> g1 in the authoritative region is live; |
| +// a region that is not authoritative has none; a missing claim refuses. |
| +func TestLiveGenerationsWalksTheUnfinishedChain(t *testing.T) { |
| + loc := "instances/h.example" |
| + c := &hostclaim.Claim{Host: "h.example", Op: hostclaim.Op{Kind: hostclaim.KindServing}, Holder: &hostclaim.Holder{Location: loc, Generation: "g3", |
| + Admissions: []hostclaim.AdmissionRecord{ |
| + {Generation: "g1", Mode: "cold", Region: "ie"}, |
| + {Generation: "g2", Predecessor: "g1", Mode: "cold", Region: "ie", PredecessorRegion: "ie"}, |
| + {Generation: "g3", Predecessor: "g2", Mode: "cold", Region: "ie", PredecessorRegion: "ie"}, |
| + }}} |
| + v := View{Region: "ie", AuthoritativeRegion: "ie", MarkerKnown: true, MarkerGeneration: "g1", GenerationOf: func(Ref) (string, bool) { return "", true }} |
| + live, err := LiveGenerations(c, v, loc) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, g := range []string{"g1", "g2", "g3"} { |
| + if !live[g] { |
| + t.Fatalf("live %v, missing %s", live, g) |
| + } |
| + } |
| + v.Region = "uk" |
| + if live, err := LiveGenerations(c, v, loc); err != nil || len(live) != 0 { |
| + t.Fatalf("a non-authoritative region: %v %v", live, err) |
| + } |
| + if _, err := LiveGenerations(nil, v, loc); !errors.Is(err, ErrUnsafe) { |
| + t.Fatalf("no claim: %v", err) |
| + } |
| +} |
diff --git a/internal/park/sweep.go b/internal/park/sweep.go| index 82097b102..92ceb109b 100644 |
| --- a/internal/park/sweep.go |
| +++ b/internal/park/sweep.go |
| @@ -106,6 +106,20 @@ func protected(c *hostclaim.Claim, v View, location string) (map[string]bool, er |
| return nil, ErrUnsafe |
| } |
| +// LiveGenerations is the region-qualified live generation set the engine |
| +// protects at location: the holder's generation, the marker's, every |
| +// generation the pinned versions belong to, and the unfinished cold |
| +// admission chain walked back to the marker in the authoritative region. |
| +// It is protected itself, exported so a view that needs the live set (the |
| +// moves files live set) enumerates the same generations rather than a |
| +// parallel walk. An error is protected's refusal. |
| +func LiveGenerations(c *hostclaim.Claim, v View, location string) (map[string]bool, error) { |
| + if c == nil { |
| + return nil, ErrUnsafe |
| + } |
| + return protected(c, v, location) |
| +} |
| + |
| // Decide uses a complete namespace version listing supplied by Discover. |
| // LegacyIncarnation is a caller-supplied, evidenced archive label only. It is |
| // never used to discover locations, and its absence refuses legacy cleanup. |