amadan
layout: recovery at the row phase checks every destination before the CAS (plan 4 final review)
A journal at row claims every file it records present is at its destination. Recovery now Lstats each one before it flips the row, and refuses naming the missing file, keeping the journal, rather than point the row at a database that is not there.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2 files changed,
+50
−1
internal/layout/pass.go+17 −1internal/layout/pass_linux_test.go+33 −0
diff --git a/internal/layout/pass.go b/internal/layout/pass.go| index 21d519eee..e03ba763c 100644 |
| --- a/internal/layout/pass.go |
| +++ b/internal/layout/pass.go |
| @@ -808,7 +808,8 @@ func (p *Pass) recoverHost(ctx context.Context, host string, byHost map[string]r |
| // forged journal can use exactly that to strand a database file). And a |
| // done journal whose row is still at from: done is written only after the |
| // row's swap committed, and releasing it would leave a row that names the |
| -// files' old place. |
| +// files' old place. A journal at row is also checked for every artifact |
| +// it records present being at its destination before the row is swapped. |
| func (p *Pass) complete(plan *Plan, j *Journal, atFrom, remove bool) Result { |
| host, phase := plan.Host, j.Phase |
| for i, a := range plan.Artifacts { |
| @@ -835,6 +836,21 @@ func (p *Pass) complete(plan *Plan, j *Journal, atFrom, remove bool) Result { |
| } |
| fallthrough |
| case PhaseRow: |
| + // The row is about to name the destinations, so every file the |
| + // journal records present must be there: a row pointed at a |
| + // missing database has the next wake restore a second one. |
| + for i, a := range plan.Artifacts { |
| + if a.Name == "socket" || !j.Artifacts[i].Present { |
| + continue |
| + } |
| + present, err := lstatPresent(a.Dst) |
| + if err != nil { |
| + return Result{host, "refused", notActing(err)} |
| + } |
| + if !present { |
| + return Result{host, "refused", fmt.Sprintf("journal records %s present, but it is missing at %s; not acting", a.Name, a.Dst)} |
| + } |
| + } |
| if atFrom { |
| if err := p.Registry.SetInstancePaths(host, j.From.DB, j.From.Socket, j.To.DB, j.To.Socket); err != nil { |
| return Result{host, "refused", notActing(fmt.Errorf("row: %w", err))} |
diff --git a/internal/layout/pass_linux_test.go b/internal/layout/pass_linux_test.go| index a19528225..ac09505f8 100644 |
| --- a/internal/layout/pass_linux_test.go |
| +++ b/internal/layout/pass_linux_test.go |
| @@ -710,6 +710,39 @@ func TestRecoverRefusesAnArtifactAtBothEnds(t *testing.T) { |
| } |
| } |
| +// A journal at row says every file it records present is at its |
| +// destination. Recovery checks that before it flips the row, because a row |
| +// pointed at a missing database would have the next wake restore a second |
| +// one there. |
| +func TestRecoverAtRowRefusesAMissingDestinationBeforeTheCAS(t *testing.T) { |
| + p, f := box(t) |
| + p.abortAt = "after-files" |
| + p.Converge(context.Background(), ModeNested) |
| + p.abortAt = "" |
| + j, found, err := ReadJournal(p.DataDir, host) |
| + if err != nil || !found { |
| + t.Fatalf("setup: %v %v", found, err) |
| + } |
| + if err := UpdatePhase(p.DataDir, j, PhaseRow); err != nil { |
| + t.Fatal(err) |
| + } |
| + wal := filepath.Join(rawInstanceDir(p.DataDir, host), host+".db-wal") |
| + if err := os.Remove(wal); err != nil { |
| + t.Fatal(err) |
| + } |
| + f.events = nil |
| + r := byHost(t, p.Recover(context.Background(), ModeNested))[host] |
| + if r.Outcome != "refused" || !strings.Contains(r.Reason, wal) { |
| + t.Fatalf("result = %+v, want a refusal naming %s", r, wal) |
| + } |
| + if got, found, _ := ReadJournal(p.DataDir, host); !found || got.Phase != PhaseRow { |
| + t.Errorf("journal not kept at row: %+v", got) |
| + } |
| + if rowAt(t, p, host) != endpoints(t, p, host, ModeFlat) { |
| + t.Errorf("the row flipped over a missing file: %+v", rowAt(t, p, host)) |
| + } |
| +} |
| + |
| // Design §3.3: a journal is completed to its own endpoint, and the host is |
| // then converged to the fleet's intent before it is released. The fence |
| // is held from the recovery through the convergence: nothing lifts it in |