amadan

carlos / platform Public

host: recover committed wakes that failed before publishing (Plan 3 final review, ruling 1)

The .files-gen repair also walks the admitted generation's cold
predecessor T back to the published P through cold admissions on this
box and region, none of which published; when .files-gen names P or one
of them and the files match P's recorded digest, .files-gen becomes T,
under the writer's fence. StartCheck is unchanged. P, a failed G1, then
G2 starts and publishes; two failed wakes then G3 starts; a file changed
since P refuses.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude c846354c16e745915ee395b74337585da5ebca6a parent ff6110f
2 files changed, +135 −18
  • internal/host/movesprobeowners.go +68 −18
  • internal/host/movesprobeowners_committed_test.go +67 −0
diff --git a/internal/host/movesprobeowners.go b/internal/host/movesprobeowners.go
index 512a1c360..3ea0cf25e 100644
--- a/internal/host/movesprobeowners.go
+++ b/internal/host/movesprobeowners.go
@@ -1388,53 +1388,103 @@ func pinnedLaunchIdentity(in *provision.MoveInputs) hostwrite.Identity {
return hostwrite.Identity{Config: committedEnvPrefix + in.EnvReport, Release: in.Release}
}
-// repairFilesGen repairs a publication interrupted between its marker write
-// and its .files-gen refresh (fix round 2, item 1). After the admission and
-// before the start check, inside the admitted writer's fence: when the
-// location's published marker names G, G is the admitted generation's cold
-// predecessor (Task 4's rule), the local .files-gen names an older generation
-// of G's own ancestry, and the files on disk are proved G's (the landing's
-// placement and, fix round 3, the content: the plain digest G's install
-// record holds), .files-gen is moved to G. No recorded digest, or anything
-// else, is left alone, for the unchanged start check to decide.
+// repairFilesGen brings .files-gen to the admitted generation's cold
+// predecessor T before the start check, inside the admitted writer's fence,
+// in two cases, each verified by content (the files tree's plain digest
+// against a recorded one, with the landing's placement), never by identity
+// alone:
+//
+// - an interrupted refresh (Task 7 fix round 2): the published marker
+// names T itself, and .files-gen names an older generation of T's
+// ancestry; the files must match T's recorded digest.
+// - unpublished cold admissions (final review, ruling 1): the published
+// marker names an older generation P, and T reaches P through cold
+// admissions on this box and region, none of which published (a
+// publication would have superseded P's marker); .files-gen names P or
+// one of those admissions. The files must match P's recorded digest:
+// every admission since P restored from P's data and its files are P's.
+//
+// Anything else is left alone, for the unchanged start check to decide.
func (m *movesProbeOwners) repairFilesGen(ctx context.Context, o hostclaim.Observed, v probeLanding, work hostwrite.Work, id hostwrite.Identity) error {
h := o.Claim.Holder
marker, err := m.readMarker(ctx, h)
if err != nil || marker == nil {
return err
}
- g := marker.Op
- if g == "" || marker.Incarnation != h.Incarnation || marker.Host != o.Host {
+ published := marker.Op
+ if published == "" || marker.Incarnation != h.Incarnation || marker.Host != o.Host {
return nil
}
- if pred, ok := coldPredecessor(h, h.Generation); !ok || pred != g {
+ t, ok := coldPredecessor(h, h.Generation)
+ if !ok || t == "" {
return nil
}
dir, _ := m.FilesDir(o.Host)
files, _, err := m.localMarkers(o.Host, filepath.Join(dir, ".none"))
- if err != nil || files == nil || files.Op == g || files.Incarnation != h.Incarnation || !ancestorOf(h, g, files.Op) {
+ if err != nil || files == nil || files.Op == t || files.Incarnation != h.Incarnation {
return nil
}
- want, ok := m.recordedFilesDigest(o.Host, v, g)
+ var vouch string // the published generation whose recorded digest the files must match
+ switch {
+ case published == t:
+ if !ancestorOf(h, t, files.Op) {
+ return nil
+ }
+ vouch = t
+ default:
+ path, ok := m.unpublishedPath(h, t, published)
+ if !ok || !containsString(path[1:], files.Op) {
+ return nil
+ }
+ vouch = published
+ }
+ want, ok := m.recordedFilesDigest(o.Host, v, vouch)
if !ok {
- return nil // nothing recorded for G: the start check decides
+ return nil // nothing recorded: the start check decides
}
- body, err := json.Marshal(moves.FilesGen{Op: g, Incarnation: h.Incarnation})
+ body, err := json.Marshal(moves.FilesGen{Op: t, Incarnation: h.Incarnation})
if err != nil {
return err
}
return work.With(ctx, id, func() error {
if m.verifyFilesContent(o.Host, v, want) != nil {
- return nil // not G's files: the start check decides
+ return nil // not the vouching generation's files: the start check decides
}
if err := syncPath(dir); err != nil {
return err
}
- log.Printf("moves: %s: .files-gen repaired from %s to the published %s", o.Host, files.Op, g)
+ log.Printf("moves: %s: .files-gen repaired from %s to %s (vouched by the published %s)", o.Host, files.Op, t, vouch)
return writeFileAtomic(filepath.Join(dir, archive.MarkerName), body, 0o644)
})
}
+// unpublishedPath walks from back to the published generation to through
+// cold admissions, each into the generation walked and on this box and
+// region. It returns the generations from from to to, inclusive, or false
+// when the chain leaves this box or region, breaks, loops, or never
+// reaches to.
+func (m *movesProbeOwners) unpublishedPath(h *hostclaim.Holder, from, to string) ([]string, bool) {
+ path := []string{from}
+ for g := from; g != to; {
+ if len(path) > len(h.Admissions)+1 {
+ return nil, false
+ }
+ var found *hostclaim.AdmissionRecord
+ for i := len(h.Admissions) - 1; i >= 0; i-- {
+ if a := h.Admissions[i]; a.Generation == g && a.Mode == "cold" {
+ found = &h.Admissions[i]
+ break
+ }
+ }
+ if found == nil || found.Box != m.Box || found.Region != m.Region || found.Predecessor == "" {
+ return nil, false
+ }
+ g = found.Predecessor
+ path = append(path, g)
+ }
+ return path, true
+}
+
// coldPredecessor is the predecessor of the newest cold admission into g.
func coldPredecessor(h *hostclaim.Holder, g string) (string, bool) {
for i := len(h.Admissions) - 1; i >= 0; i-- {
diff --git a/internal/host/movesprobeowners_committed_test.go b/internal/host/movesprobeowners_committed_test.go
index a7cf045a4..819064dfc 100644
--- a/internal/host/movesprobeowners_committed_test.go
+++ b/internal/host/movesprobeowners_committed_test.go
@@ -442,3 +442,70 @@ func TestMovesProbeOwnersRepairVerifiesTheFilesContent(t *testing.T) {
})
}
}
+
+// publishedP is the fixture at a published committed generation P: a
+// restarted probe published (P), the commit, .files-gen naming P and P's
+// files digest journalled.
+func publishedP(t *testing.T) (*poFixture, string) {
+ t.Helper()
+ f, restarted, again := restartedProbePublished(t)
+ if err := f.publishFresh(restarted, again); err != nil {
+ t.Fatal(err)
+ }
+ f.commit()
+ return f, restarted.Claim.Holder.Generation
+}
+
+// Final review, ruling 1: a committed wake G1 that failed before its marker
+// leaves .files-gen at the published P while the next admission G2's cold
+// predecessor is G1. The repair walks G1 back to P through unpublished cold
+// admissions on this box and region and, the files matching P's recorded
+// digest, sets .files-gen to G1: G2 starts (StartCheck unchanged) and
+// publishes.
+func TestMovesProbeOwnersRecoverACommittedWakeThatFailedBeforePublishing(t *testing.T) {
+ f, p := publishedP(t)
+ if _, _, err := f.committedWake(f.read()); err != nil { // G1, then it dies before PutMarker
+ t.Fatal(err)
+ }
+ if fg := f.filesGen(); fg.Op != p {
+ t.Fatalf("the fixture: %+v", fg)
+ }
+ g2, in2, err := f.committedWake(f.read())
+ if err != nil {
+ t.Fatalf("G2 after an unpublished G1: %v", err)
+ }
+ if err := f.publishFresh(g2, in2); err != nil {
+ t.Fatal(err)
+ }
+ if fg := f.filesGen(); fg.Op != g2.Claim.Holder.Generation {
+ t.Fatalf(".files-gen after G2's publication: %+v", fg)
+ }
+}
+
+// Ruling 1: two failed wakes, G1 and G2, then G3 starts.
+func TestMovesProbeOwnersRecoverTwoFailedCommittedWakes(t *testing.T) {
+ f, _ := publishedP(t)
+ f.committedWake(f.read()) // G1, dies before PutMarker
+ f.committedWake(f.read()) // G2, dies before PutMarker
+ if _, _, err := f.committedWake(f.read()); err != nil {
+ t.Fatalf("G3 after two unpublished wakes: %v", err)
+ }
+}
+
+// Ruling 1: a file changed after P's publication refuses the walk's repair;
+// the start check refuses G2 and .files-gen stays P.
+func TestMovesProbeOwnersCommittedWalkVerifiesContent(t *testing.T) {
+ f, p := publishedP(t)
+ if _, _, err := f.committedWake(f.read()); err != nil { // G1
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(f.files, "a.txt"), []byte("changed after P"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if _, _, err := f.committedWake(f.read()); !errors.Is(err, moves.ErrStartFiles) {
+ t.Fatalf("G2 over changed files: %v", err)
+ }
+ if fg := f.filesGen(); fg.Op != p {
+ t.Fatalf(".files-gen was repaired over changed files: %+v", fg)
+ }
+}