| index efacab1f1..d16ce6cd2 100644 |
| --- a/internal/host/movesrecoveryowners.go |
| +++ b/internal/host/movesrecoveryowners.go |
| @@ -444,6 +444,11 @@ func (r movesRecoveryOwners) AdmissionEvidence(ctx context.Context, o hostclaim. |
| // call before the admission's CAS and so before the activator restores |
| // it (claimwake.go Restore precedes Materialize): a crash after the |
| // restore leaves the pin a restart must match (fix round 2). |
| + // So is this admission's link in the recovery's lineage: the |
| + // generation it admits and the one it observed (fix round 3). |
| + if err := r.recordLineage(o.Host, v, in.NewGeneration); err != nil { |
| + return err |
| + } |
| return r.recordSourceBaseline(o.Host, v, baseline) |
| } |
| |
| @@ -506,8 +511,9 @@ func (r movesRecoveryOwners) Materialize(ctx context.Context, o hostclaim.Observ |
| return recoveryInputsChanged("launch environment") |
| } |
| m.recordFence(o.Host, h.Session, work) |
| - // The tree the recovery restored, as placed, before anything runs on it. |
| - if err := r.recordPlacement(ctx, o.Host, v, work, in.Identity); err != nil { |
| + // The start gate: the tree is the one the recovery restored before its |
| + // first authorized start, and the recovery's own after it. |
| + if err := r.startGate(ctx, o, v, work, in.Identity); err != nil { |
| return err |
| } |
| if v.dirty { |
| @@ -585,10 +591,9 @@ func (r movesRecoveryOwners) PublishFenced(ctx context.Context, o hostclaim.Obse |
| if work == nil { |
| return ClaimPublicationReceipt{}, fmt.Errorf("%w: no admitted writer's fence for session %d", hostbox.ErrChanged, h.Session) |
| } |
| - // The files on disk must be the tree the recovery placed before anything |
| - // is published, and a tree whose label the publication will move must |
| - // still hold the content journalled when it was placed (fix round 2): |
| - // the relabel is vouched for by that digest, never one sampled now. |
| + // The files on disk must be the tree the recovery placed, held by this |
| + // recovery's admitted lineage, before anything is published (fix round |
| + // 3): its own writes since its first start are authoritative. |
| placed, ok, err := r.placement(o.Host, v) |
| if err != nil { |
| return ClaimPublicationReceipt{}, err |
| @@ -597,24 +602,30 @@ func (r movesRecoveryOwners) PublishFenced(ctx context.Context, o hostclaim.Obse |
| return ClaimPublicationReceipt{}, fmt.Errorf("%w: the recovery recorded no placed tree", hostbox.ErrChanged) |
| } |
| if err := work.With(ctx, in.Identity, func() error { |
| - _, err := r.vouchFiles(o.Host, h, placed) |
| + _, err := r.vouchFiles(o.Host, v, placed) |
| return err |
| }); err != nil { |
| return ClaimPublicationReceipt{}, err |
| } |
| - return m.publishFresh(ctx, o.Host, h, pub, func() error { return r.refreshFilesGen(ctx, o.Host, h, placed, work, in.Identity) }) |
| + return m.publishFresh(ctx, o.Host, h, pub, func() error { return r.refreshFilesGen(ctx, o.Host, v, placed, work, in.Identity) }) |
| } |
| |
| -// vouchFiles requires the files on disk to be the placed tree (identity) |
| -// and reports whether .files-gen will move to holder.generation: it names |
| -// that generation's cold predecessor for this incarnation. When it will, |
| -// the tree must also hold the journalled placement digest. A label that |
| -// will not move needs no content evidence: nothing is relabelled. |
| -func (r movesRecoveryOwners) vouchFiles(host string, h *hostclaim.Holder, placed recoveryPlacement) (bool, error) { |
| +// vouchFiles requires the files on disk to be the placed tree (identity: a |
| +// replaced tree is refused) and the holder's session to be this recovery's |
| +// own through its admitted lineage, and reports whether .files-gen will |
| +// move to holder.generation: it names that generation's cold predecessor |
| +// for this incarnation. It never checks content: since the first start the |
| +// tree is the recovery's, and its writes are authoritative (fix round 3, |
| +// spec §7.8). The caller runs it under the session's fence. |
| +func (r movesRecoveryOwners) vouchFiles(host string, v recoveryProbe, placed recoveryPlacement) (bool, error) { |
| m := r.m |
| + h := v.holder |
| if err := r.verifyPlaced(host, placed); err != nil { |
| return false, err |
| } |
| + if err := r.lineage(host, v); err != nil { |
| + return false, err |
| + } |
| dir, _ := m.FilesDir(host) |
| files, _, err := m.localMarkers(host, filepath.Join(dir, ".none")) |
| if err != nil || files == nil || files.Incarnation != h.Incarnation || files.Op == h.Generation { |
| @@ -623,22 +634,23 @@ func (r movesRecoveryOwners) vouchFiles(host string, h *hostclaim.Holder, placed |
| if pred, ok := coldPredecessor(h, h.Generation); !ok || pred == "" || files.Op != pred { |
| return false, nil |
| } |
| - return true, r.verifyContent(host, placed, placed.Plain) |
| + return true, nil |
| } |
| |
| // refreshFilesGen points .files-gen at the published generation, inside the |
| // admitted writer's fence, after the files on disk are proved the tree the |
| -// recovery placed, by identity and by the journalled placement digest |
| -// (vouchFiles): never by identity alone, and a mismatch fails the |
| -// publication and relabels nothing. It relabels only a .files-gen naming the |
| -// generation's cold predecessor for this incarnation (the files the start |
| -// check accepted for this session); absent, or anything else, is left for |
| -// the start check to decide. |
| -func (r movesRecoveryOwners) refreshFilesGen(ctx context.Context, host string, h *hostclaim.Holder, placed recoveryPlacement, work hostwrite.Work, id hostwrite.Identity) error { |
| +// recovery placed and the session this recovery's own (vouchFiles); a |
| +// replaced tree or an unproven session fails the publication and relabels |
| +// nothing. It relabels only a .files-gen naming the generation's cold |
| +// predecessor for this incarnation (the files the start check accepted for |
| +// this session); absent, or anything else, is left for the start check to |
| +// decide. |
| +func (r movesRecoveryOwners) refreshFilesGen(ctx context.Context, host string, v recoveryProbe, placed recoveryPlacement, work hostwrite.Work, id hostwrite.Identity) error { |
| m := r.m |
| + h := v.holder |
| dir, _ := m.FilesDir(host) |
| return work.With(ctx, id, func() error { |
| - relabel, err := r.vouchFiles(host, h, placed) |
| + relabel, err := r.vouchFiles(host, v, placed) |
| if err != nil || !relabel { |
| return err |
| } |
| @@ -675,6 +687,10 @@ func (r movesRecoveryOwners) PublicationEvidence(ctx context.Context, o hostclai |
| // revert). |
| const rcStepFilesPlaced = "rc_files_placed" |
| |
| +// rcStepLineage prefixes the recovery journal's record of each generation |
| +// the recovery admitted (rcStepLineage+<generation>, JSON recoveryLineage). |
| +const rcStepLineage = "rc_lineage." |
| + |
| // recoveryPlacement is what rcStepFilesPlaced records. |
| type recoveryPlacement struct { |
| Files string `json:"files"` |
| @@ -708,17 +724,66 @@ func (r movesRecoveryOwners) placement(host string, v recoveryProbe) (recoveryPl |
| return p, true, nil |
| } |
| |
| +// startGate runs under the start's work before every start. Before the |
| +// recovery's first authorized start (no placement journalled) the tree must |
| +// be the one the recovery restored (recordPlacement); after it, the tree is |
| +// the recovery's own: its identity is the placed one, and its content is |
| +// never compared again (fix round 3). Every start's session must be this |
| +// recovery's through its admitted lineage. |
| +func (r movesRecoveryOwners) startGate(ctx context.Context, o hostclaim.Observed, v recoveryProbe, work hostwrite.Work, id hostwrite.Identity) error { |
| + placed, ok, err := r.placement(o.Host, v) |
| + if err != nil { |
| + return err |
| + } |
| + if !ok { |
| + expected, err := restoredDigest(o.Claim, v) |
| + if err != nil { |
| + return err |
| + } |
| + if err := r.recordPlacement(ctx, o.Host, v, expected, work, id); err != nil { |
| + return err |
| + } |
| + } else if err := work.With(ctx, id, func() error { return r.verifyPlaced(o.Host, placed) }); err != nil { |
| + return err |
| + } |
| + return r.lineage(o.Host, v) |
| +} |
| + |
| +// restoredDigest is the plain digest of the tree the recovery restores, as |
| +// its baselines record it: a dirty return's is the return baseline the |
| +// target journalled in the claim (the tree its install extracted); a clean |
| +// return's or a revert's is the source's stopped baseline (Move.Source, |
| +// which the clean verification and the revert's verify compared). "" when |
| +// the record keeps none (a revert from stopping, J8): the first start then |
| +// takes the tree as it finds it. |
| +func restoredDigest(c *hostclaim.Claim, v recoveryProbe) (string, error) { |
| + if v.dirty { |
| + raw, ok := c.Op.Journal[moves.JournalReturnBaseline] |
| + if !ok { |
| + return "", fmt.Errorf("%w: the claim carries no return baseline", hostbox.ErrChanged) |
| + } |
| + b, err := moves.DecodeReturnBaseline(raw) |
| + if err != nil { |
| + return "", err |
| + } |
| + return b.Files.Plain, nil |
| + } |
| + if src := v.rec.Move.Source; src != nil && src.Files != nil { |
| + return src.Files.Plain, nil |
| + } |
| + return "", nil |
| +} |
| + |
| // recordPlacement journals the tree the recovery restored, as placed, under |
| // the start's work, before the recovery's first start: the files directory's |
| -// and <host>.env's identities and the plain digest. A dirty return's tree is |
| -// the one its install placed (landStepFilesPlaced), and must still be; a |
| -// clean return's and a revert's is the source's own tree the recovery |
| -// verified. Once recorded it is never rewritten. |
| -func (r movesRecoveryOwners) recordPlacement(ctx context.Context, host string, v recoveryProbe, work hostwrite.Work, id hostwrite.Identity) error { |
| +// and <host>.env's identities and the plain digest, which must be expected |
| +// (the restored tree's, restoredDigest) when that is known. A dirty |
| +// return's tree is the one its install placed (landStepFilesPlaced), and |
| +// must still be. A tree that is not the restored one is refused and nothing |
| +// is journalled, so the next attempt checks again. Once recorded it is never |
| +// rewritten. |
| +func (r movesRecoveryOwners) recordPlacement(ctx context.Context, host string, v recoveryProbe, expected string, work hostwrite.Work, id hostwrite.Identity) error { |
| m := r.m |
| - if _, ok, err := r.placement(host, v); err != nil || ok { |
| - return err |
| - } |
| dir, env := m.FilesDir(host) |
| var p recoveryPlacement |
| if err := work.With(ctx, id, func() error { |
| @@ -745,8 +810,13 @@ func (r movesRecoveryOwners) recordPlacement(ctx context.Context, host string, v |
| return fmt.Errorf("%w: the files are not the tree the return installed", hostbox.ErrChanged) |
| } |
| } |
| - p.Plain, err = m.filesDigest(host) |
| - return err |
| + if p.Plain, err = m.filesDigest(host); err != nil { |
| + return err |
| + } |
| + if expected != "" && p.Plain != expected { |
| + return fmt.Errorf("%w: the files are not the tree the recovery restored", hostbox.ErrChanged) |
| + } |
| + return nil |
| }); err != nil { |
| return err |
| } |
| @@ -784,23 +854,82 @@ func (r movesRecoveryOwners) verifyPlaced(host string, p recoveryPlacement) erro |
| return nil |
| } |
| |
| -// verifyContent requires both the placement (identity) and the recorded |
| -// content (the plain digest): never identity alone. |
| -func (r movesRecoveryOwners) verifyContent(host string, p recoveryPlacement, want string) error { |
| - if want == "" { |
| - return fmt.Errorf("%w: no recorded files digest", hostbox.ErrChanged) |
| - } |
| - if err := r.verifyPlaced(host, p); err != nil { |
| +// recoveryLineage is one rcStepLineage record: the generation an admission |
| +// of this recovery observed (its predecessor), and whether it was the |
| +// recovery's first admission (Root: the holder still at the source's stamp |
| +// S). |
| +type recoveryLineage struct { |
| + Predecessor string `json:"predecessor"` |
| + Root bool `json:"root,omitempty"` |
| +} |
| + |
| +// recordLineage journals the admission's link before its CAS, in its |
| +// evidence: the generation it admits, the generation it observed. The |
| +// claim's CAS admits only over the observed generation, so a holder at a |
| +// journalled generation proves that this recovery admitted it over its |
| +// predecessor. A link for a generation never admitted (a refused CAS) is |
| +// inert: generation names are fresh. |
| +func (r movesRecoveryOwners) recordLineage(host string, v recoveryProbe, g string) error { |
| + m := r.m |
| + body, err := json.Marshal(recoveryLineage{Predecessor: v.holder.Generation, Root: v.first}) |
| + if err != nil { |
| return err |
| } |
| - got, err := r.m.filesDigest(host) |
| + m.mu.Lock() |
| + defer m.mu.Unlock() |
| + err = moves.UpdateJournal(m.DataDir, host, v.journalOp(), func(j *moves.Journal) error { |
| + if have, ok := j.Steps[rcStepLineage+g]; ok { |
| + if have != string(body) { |
| + return fmt.Errorf("%w: generation %s is already another admission's", hostbox.ErrChanged, g) |
| + } |
| + return errJournalUnchanged |
| + } |
| + j.Mark(rcStepLineage+g, string(body)) |
| + return nil |
| + }) |
| + if errors.Is(err, errJournalUnchanged) { |
| + return nil |
| + } |
| + return err |
| +} |
| + |
| +// lineage proves the holder's session is this recovery's own: this box |
| +// writes the holder's generation, admitted in this region, and from it the |
| +// journalled links reach the recovery's first admission, each generation |
| +// admitted by this recovery over the one before. Any other writer admitted |
| +// in between breaks the chain (its generation was never journalled here): |
| +// the ownership interval is unproven and refused. |
| +func (r movesRecoveryOwners) lineage(host string, v recoveryProbe) error { |
| + m := r.m |
| + h := v.holder |
| + n := len(h.Admissions) |
| + if n == 0 || h.WriterBox != m.Box { |
| + return fmt.Errorf("%w: this box does not write the holder's session", hostbox.ErrChanged) |
| + } |
| + if a := h.Admissions[n-1]; a.Box != m.Box || a.Generation != h.Generation || !park.SameRegion(m.HomeRegion, a.Region, m.Region) { |
| + return fmt.Errorf("%w: the holder's admission is not this box's", hostbox.ErrChanged) |
| + } |
| + j, err := moves.ReadJournal(m.DataDir, host, v.journalOp()) |
| if err != nil { |
| return err |
| } |
| - if got != want { |
| - return fmt.Errorf("%w: the files on disk are not the recorded ones", hostbox.ErrChanged) |
| + seen := map[string]bool{} |
| + for g := h.Generation; !seen[g]; { |
| + seen[g] = true |
| + raw, ok := j.Steps[rcStepLineage+g] |
| + if !ok { |
| + return fmt.Errorf("%w: generation %s is not this recovery's admission", hostbox.ErrChanged, g) |
| + } |
| + var link recoveryLineage |
| + if err := json.Unmarshal([]byte(raw), &link); err != nil { |
| + return fmt.Errorf("%w: the lineage of %s does not read: %v", hostbox.ErrChanged, g, err) |
| + } |
| + if link.Root { |
| + return nil |
| + } |
| + g = link.Predecessor |
| } |
| - return nil |
| + return fmt.Errorf("%w: the recovery's lineage loops", hostbox.ErrChanged) |
| } |
| |
| // rcStepSourceBaseline is the rollback journal's record of a clean return's |