amadan
host: the cut's crash matrix: after the evidence, after the final round, after the seal and between the two snapshot PUTs, each replayed by a fresh worker (Task 5 fix round 1, ruling 6)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 file changed,
+95
−0
internal/host/movescut_test.go+95 −0
diff --git a/internal/host/movescut_test.go b/internal/host/movescut_test.go| index ecd90aeb2..d4f79e049 100644 |
| --- a/internal/host/movescut_test.go |
| +++ b/internal/host/movescut_test.go |
| @@ -1375,3 +1375,98 @@ func TestArchiveSealStopsOnALedgerFailure(t *testing.T) { |
| t.Fatalf("ledger bound %v, seal bound %v (was %v)", after.LeaseUntil, seal.LeaseUntil, before.LeaseUntil) |
| } |
| } |
| + |
| +// Ruling 6: the crash matrix for the side effects the review found |
| +// unexercised. Each point crashes once; the durable state a fresh worker |
| +// would find is asserted; the fresh worker replays to the flip, with one |
| +// report, a seal naming the archive at the record's N, and the ledger |
| +// carrying the seal's bound. |
| +func TestCutCrashMatrix(t *testing.T) { |
| + ctx := context.Background() |
| + lane := moves.ReplicaPrefix(p1Host, p1ID) |
| + for _, point := range []string{"after-evidence", "after-final-round", "after-seal", "archive-sealed-snapshot"} { |
| + t.Run(point, func(t *testing.T) { |
| + h := newCutHarness(t) |
| + h.ready(t) |
| + h.writeOnStop(t) // so the archive is the cut's own build |
| + h.crashAt = point |
| + h.cut(t) |
| + h.crashPass(t) |
| + m := h.move(t) |
| + c := h.claim(t) |
| + j, err := moves.ReadJournal(h.cfg.DataDir, p1Host, p1ID) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + _, _, reportErr := h.s.Get(ctx, h.stoppedKey()) |
| + _, sealed := c.Op.Journal[hostclaim.JournalSeal] |
| + switch point { |
| + case "after-evidence": |
| + // stopping: the evidence journalled, execution still the |
| + // source's, no report, no hold given up. |
| + if m.CutPhase != provision.CutStopping || !j.Done(cutStepEvidence) || c.Op.Exec != "ie-box" || !errors.Is(reportErr, os.ErrNotExist) { |
| + t.Fatalf("durable state: %+v, evidence %v, exec %q, report %v", m, j.Done(cutStepEvidence), c.Op.Exec, reportErr) |
| + } |
| + case "after-final-round": |
| + // stopped: execution given up, the report published, the |
| + // lane holding the final round's copy, and no seal yet. |
| + if m.CutPhase != provision.CutStopped || c.Op.Exec != "" || c.Op.Phase != hostclaim.PhaseMoving || reportErr != nil || sealed { |
| + t.Fatalf("durable state: %+v, claim %+v, report %v", m, c.Op, reportErr) |
| + } |
| + if _, ok := h.f.buckets["data-uk"][lane+lastTXKey]; !ok { |
| + t.Fatal("the final round's copy is not in the lane") |
| + } |
| + case "after-seal": |
| + // stopped: the claim sealed with a seal naming the replica and |
| + // the timers but no files; the ledger without a bound yet. |
| + if m.CutPhase != provision.CutStopped || c.Op.Phase != hostclaim.PhaseSealed || !sealed { |
| + t.Fatalf("durable state: %+v, claim %+v", m, c.Op) |
| + } |
| + if seal := h.seal(t); seal.HasFiles() || seal.N != 3 || len(seal.Timers) != 1 { |
| + t.Fatalf("seal: %+v", seal) |
| + } |
| + if l := h.ledger(t); !l.LeaseUntil.IsZero() { |
| + t.Fatalf("the ledger carries a bound (%v) before the seal's pass finished", l.LeaseUntil) |
| + } |
| + case "archive-sealed-snapshot": |
| + // staged: the identity journalled, the sealed object in |
| + // move-snapshots/ and the manifest not, the seal without files. |
| + id := h.journalArchive(t) |
| + snap := moves.SnapshotsPrefix(p1Host, p1ID) |
| + _, sealedThere := h.f.buckets["data-ie"][snap+id.Name+".sealed"] |
| + _, manifestThere := h.f.buckets["data-ie"][snap+id.Name+".json"] |
| + if m.CutPhase != provision.CutStaged || !sealedThere || manifestThere || h.seal(t).HasFiles() { |
| + t.Fatalf("durable state: %+v, sealed %v, manifest %v", m, sealedThere, manifestThere) |
| + } |
| + } |
| + h.freshWorker() |
| + h.pass(t) |
| + m = h.move(t) |
| + if m.State != provision.MoveLanding || m.CutPhase != provision.CutFlipped { |
| + t.Fatalf("the replay: %+v", m) |
| + } |
| + keys, _ := h.s.List(ctx, moves.ReportPrefix("bab", "app", p1Host, p1ID)) |
| + n := 0 |
| + for _, k := range keys { |
| + if strings.Contains(k, "stopped-") { |
| + n++ |
| + } |
| + } |
| + seal := h.seal(t) |
| + if n != 1 || !sealNamesArchive(seal, m.Source.Files.Sealed) || seal.N != m.Source.FinalTXID || !seal.ReachesN() { |
| + t.Fatalf("%d reports; seal %+v; source %+v", n, seal, m.Source) |
| + } |
| + for _, o := range append(append(append([]moves.SealedObject{}, seal.Replica...), seal.Timers...), seal.Files...) { |
| + if h.f.versionOf("data-uk", o.Key) != o.VersionID { |
| + t.Fatalf("the seal names %s@%s, current %s", o.Key, o.VersionID, h.f.versionOf("data-uk", o.Key)) |
| + } |
| + } |
| + if l := h.ledger(t); !l.LeaseUntil.Equal(seal.LeaseUntil) { |
| + t.Fatalf("ledger bound %v, seal bound %v", l.LeaseUntil, seal.LeaseUntil) |
| + } |
| + if c := h.claim(t); c.Op.Kind != hostclaim.KindLanding || c.Op.Exec != "uk-box" { |
| + t.Fatalf("claim: %+v", c.Op) |
| + } |
| + }) |
| + } |
| +} |