| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "os" |
| 7 | "path/filepath" |
| 8 | "reflect" |
| 9 | "strconv" |
| 10 | "strings" |
| 11 | "testing" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/hostclaim" |
| 14 | "github.com/carlosframework/platform/internal/hostguard" |
| 15 | "github.com/carlosframework/platform/internal/provision" |
| 16 | ) |
| 17 | |
| 18 | // Two model witnesses ported onto Task 4's target-side rollback harness and |
| 19 | // Task 5's source-side recovery harness (moves spec §5.1 step 9 and §6.4; |
| 20 | // Plan 4 Task 8). The claim half of the first is in hostclaim's |
| 21 | // move_witness_test.go. |
| 22 | |
| 23 | // sourceGenerationObjects are the objects of the source's generation "cur" |
| 24 | // in the source region's bucket, by key. |
| 25 | func sourceGenerationObjects(h *recoverHarness) map[string]fakeMoveObject { |
| 26 | out := map[string]fakeMoveObject{} |
| 27 | for k, o := range h.f.buckets["data-ie"] { |
| 28 | if strings.Contains(k, "/g-cur/") { |
| 29 | out[k] = o |
| 30 | } |
| 31 | } |
| 32 | return out |
| 33 | } |
| 34 | |
| 35 | // TestRollbackBaselineSurvivesTheCleanReturn is wRollbackBaselinePreserved |
| 36 | // (claim rev 6.14 item 5, R12-3; spec §5.1 step 9 and §6.4): a landing's |
| 37 | // probe publishes the target's generation in uk, and the source's |
| 38 | // generation, the target's predecessor in ie, is not queued for a park by |
| 39 | // PublishGeneration. The target's admission record stays uncovered, no |
| 40 | // obligation reaches ie, and the real sequence RollBack, TargetReleased and |
| 41 | // ReturnToSource(clean) then finds the source's generation exactly as the |
| 42 | // cut left it in the ie bucket fake, so the source restores from it. Only |
| 43 | // Commit covers the record (hostclaim's half of the witness). |
| 44 | func TestRollbackBaselineSurvivesTheCleanReturn(t *testing.T) { |
| 45 | t.Parallel() |
| 46 | ctx := context.Background() |
| 47 | h := newRecoverHarness(t, false) |
| 48 | baseline := sourceGenerationObjects(h) |
| 49 | if len(baseline) == 0 { |
| 50 | t.Fatal("fixture: the source's generation holds no objects in ie") |
| 51 | } |
| 52 | |
| 53 | // The landing's probe admission, with the claim library's own |
| 54 | // transition: cold, in uk, its predecessor the source's generation in ie. |
| 55 | srcGen := h.claim(t).Op.Journal[hostclaim.JournalSrcGen] |
| 56 | if srcGen != "cur" { |
| 57 | t.Fatalf("fixture: src_gen %q", srcGen) |
| 58 | } |
| 59 | obs, err := hostclaim.Read(ctx, h.s, p1Host) |
| 60 | if err != nil { |
| 61 | t.Fatal(err) |
| 62 | } |
| 63 | op, holder := obs.Claim.Op, obs.Claim.Holder |
| 64 | if _, err := hostclaim.Update(ctx, h.s, obs, hostclaim.AdmitWriter(hostclaim.Admission{ |
| 65 | ObservedOpID: op.ID, ObservedPhase: op.Phase, ObservedSession: holder.Session, ObservedGeneration: holder.Generation, |
| 66 | Box: "uk-box", Mode: "cold", NewGeneration: p1ID, Coverage: "sealed", Region: "uk", PredecessorRegion: "ie", KeepStamp: true, |
| 67 | })); err != nil { |
| 68 | t.Fatal(err) |
| 69 | } |
| 70 | |
| 71 | // kept is the witness at one point of the flow. |
| 72 | kept := func(where string) { |
| 73 | t.Helper() |
| 74 | c := h.claim(t) |
| 75 | var target *hostclaim.AdmissionRecord |
| 76 | for i := range c.Holder.Admissions { |
| 77 | if a := &c.Holder.Admissions[i]; a.Box == "uk-box" && a.Mode == "cold" { |
| 78 | target = a |
| 79 | } |
| 80 | } |
| 81 | if target == nil || target.Region != "uk" || target.PredecessorRegion != "ie" || target.Predecessor != srcGen { |
| 82 | t.Fatalf("%s: the target's record does not name the source's generation in ie: %+v", where, c.Holder.Admissions) |
| 83 | } |
| 84 | if target.Covered { |
| 85 | t.Fatalf("%s: the target's record is covered: %+v", where, target) |
| 86 | } |
| 87 | for _, o := range c.Obligations { |
| 88 | if o.Region == "ie" || o.Box == "ie-box" { |
| 89 | t.Fatalf("%s: an obligation reaches the source's region: %+v", where, o) |
| 90 | } |
| 91 | } |
| 92 | if got := sourceGenerationObjects(h); !reflect.DeepEqual(got, baseline) { |
| 93 | t.Fatalf("%s: the source's generation changed in ie: %d objects, was %d", where, len(got), len(baseline)) |
| 94 | } |
| 95 | } |
| 96 | |
| 97 | // The probe's publication, as the claim library has it on main. |
| 98 | obs, err = hostclaim.Read(ctx, h.s, p1Host) |
| 99 | if err != nil { |
| 100 | t.Fatal(err) |
| 101 | } |
| 102 | op, holder = obs.Claim.Op, obs.Claim.Holder |
| 103 | if holder.Generation != p1ID { |
| 104 | t.Fatalf("fixture: holder.generation %q", holder.Generation) |
| 105 | } |
| 106 | publish := hostclaim.PublishGeneration(op.ID, op.Phase, holder.Session, holder.Generation, "uk", func() string { |
| 107 | t.Error("the target's publication allocated an obligation id") |
| 108 | return "id" |
| 109 | }) |
| 110 | if _, err := hostclaim.Update(ctx, h.s, obs, publish); err != nil { |
| 111 | t.Fatal(err) |
| 112 | } |
| 113 | kept(t.Name() + " after the target's publication") |
| 114 | |
| 115 | // The real sequence: the rollback, the target's release, the source's |
| 116 | // clean return, through to opened. |
| 117 | rb := h.rollBack(t) |
| 118 | kept("after the rollback decision") |
| 119 | h.rollbackPass(t) |
| 120 | if got := h.rb(t); got.TargetPhase != provision.RbTargetReleased || got.Dirty { |
| 121 | t.Fatalf("the target did not release clean: %+v", got) |
| 122 | } |
| 123 | kept("after the target's release") |
| 124 | h.srcPass(t) |
| 125 | h.opened(t) |
| 126 | kept("after the clean return") |
| 127 | |
| 128 | c := h.claim(t) |
| 129 | if c.Holder.Generation != srcGen || c.Op.Journal[hostclaim.JournalSrcGen] != srcGen { |
| 130 | t.Fatalf("the return names generation %q, want the source's %q", c.Holder.Generation, srcGen) |
| 131 | } |
| 132 | if len(h.probes) != 1 || !h.probes[0].Clean || h.probes[0].SourceGeneration != srcGen || h.probes[0].RollbackID != rb.ID { |
| 133 | t.Fatalf("the recovery probe was not asked to restore from the source's generation: %+v", h.probes) |
| 134 | } |
| 135 | } |
| 136 | |
| 137 | // loseGuardRecords is the lost-record fault: the source's installation |
| 138 | // record is gone, either the host's file or the whole hostguard directory |
| 139 | // (re-provisioned empty, as the box's boot does). |
| 140 | func loseGuardRecords(t *testing.T, dataDir string, wholeDir bool) { |
| 141 | t.Helper() |
| 142 | dir := filepath.Join(dataDir, "hostguard") |
| 143 | if !wholeDir { |
| 144 | ents, err := os.ReadDir(dir) |
| 145 | if err != nil || len(ents) == 0 { |
| 146 | t.Fatalf("the guard directory holds no record to lose: %v %v", ents, err) |
| 147 | } |
| 148 | for _, e := range ents { |
| 149 | if err := os.RemoveAll(filepath.Join(dir, e.Name())); err != nil { |
| 150 | t.Fatal(err) |
| 151 | } |
| 152 | } |
| 153 | return |
| 154 | } |
| 155 | if err := os.RemoveAll(dir); err != nil { |
| 156 | t.Fatal(err) |
| 157 | } |
| 158 | if err := os.MkdirAll(dir, 0o770); err != nil { |
| 159 | t.Fatal(err) |
| 160 | } |
| 161 | } |
| 162 | |
| 163 | // TestDirtyReturnAfterALostGuardRecord is wReturnLostRecord (claim rev 6.5, |
| 164 | // R6-4; spec §5.1 step 9 and §6.4): a dirty rollback whose source loses its |
| 165 | // installation record after the return's claim. The claimed phase had not |
| 166 | // yet been left, so the source's recovery replays it: ReturnToSource wrote |
| 167 | // S, the pre-move stamp, back into holder.stamp, so Guard.Activate |
| 168 | // re-initialises the record from the claim at S, never at F (the target's |
| 169 | // stamp), and Task 3's installer, entered through recoverSource, tags every |
| 170 | // artifact it registers with S. The source ends opened at S with no path |
| 171 | // tagged F. |
| 172 | func TestDirtyReturnAfterALostGuardRecord(t *testing.T) { |
| 173 | t.Parallel() |
| 174 | for _, wholeDir := range []bool{false, true} { |
| 175 | for _, registered := range []bool{false, true} { |
| 176 | name := map[bool]string{false: "the record file", true: "the whole directory"}[wholeDir] + ", " + map[bool]string{false: "unregistered", true: "registered"}[registered] |
| 177 | t.Run(name, func(t *testing.T) { |
| 178 | h := newRecoverHarness(t, true) |
| 179 | if registered { |
| 180 | rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) { |
| 181 | c.Inventory = map[string]hostclaim.InventoryEntry{"ie-box": {Layout: "flat", Paths: map[string]uint64{p1Host + ".db": 1, p1Host + ".db.gen": 1, p1Host: 1, p1Host + ".env": 1}}} |
| 182 | }) |
| 183 | } |
| 184 | rb := h.released(t) |
| 185 | c := h.claim(t) |
| 186 | s, err := strconv.ParseUint(c.Op.Journal[hostclaim.JournalSourceStamp], 10, 64) |
| 187 | if err != nil || s == 0 { |
| 188 | t.Fatalf("source_stamp %q: %v", c.Op.Journal[hostclaim.JournalSourceStamp], err) |
| 189 | } |
| 190 | f, err := strconv.ParseUint(c.Op.Journal[hostclaim.JournalTargetStamp], 10, 64) |
| 191 | if err != nil || f == 0 || f == s { |
| 192 | t.Fatalf("target_stamp %q, source_stamp %d: %v", c.Op.Journal[hostclaim.JournalTargetStamp], s, err) |
| 193 | } |
| 194 | |
| 195 | // The record is lost right after the return's claim and the |
| 196 | // activation, before the record leaves claimed. |
| 197 | h.rcAt = "after-activate" |
| 198 | h.rcHook = func(point string) { |
| 199 | if point == "after-activate" { |
| 200 | loseGuardRecords(t, h.sdir, wholeDir) |
| 201 | } |
| 202 | } |
| 203 | h.srcPass(t) |
| 204 | h.sourceAt(t, provision.RbSourceClaimed) |
| 205 | if _, err := hostguard.Capture(h.sdir, p1Host); !errors.Is(err, hostguard.ErrNoRecord) && !errors.Is(err, hostguard.ErrNotProvisioned) { |
| 206 | t.Fatalf("the installation record survived the loss: %v", err) |
| 207 | } |
| 208 | if c := h.claim(t); c.Op.Phase != hostclaim.PhaseClaimed || c.Op.Exec != "ie-box" || c.Holder.Stamp != s { |
| 209 | t.Fatalf("claim after the loss: %+v holder stamp %d, want claimed by ie-box at %d", c.Op, c.Holder.Stamp, s) |
| 210 | } |
| 211 | |
| 212 | // The replay re-initialises from the claim and recovers. |
| 213 | h.rcHook = nil |
| 214 | h.srcPass(t) |
| 215 | got := h.opened(t) |
| 216 | if got.ID != rb.ID { |
| 217 | t.Fatalf("opened %s, want %s", got.ID, rb.ID) |
| 218 | } |
| 219 | captured, err := hostguard.Capture(h.sdir, p1Host) |
| 220 | if err != nil || captured.Stamp != s || captured.Incarnation != h.claim(t).Holder.Incarnation { |
| 221 | t.Fatalf("the re-initialised record: %+v %v, want stamp %d (never %d)", captured, err, s, f) |
| 222 | } |
| 223 | c = h.claim(t) |
| 224 | if c.Holder.Stamp != s { |
| 225 | t.Fatalf("holder.stamp %d, want S = %d", c.Holder.Stamp, s) |
| 226 | } |
| 227 | // Everything the install registered on the source is tagged S, |
| 228 | // and none of it F (the target's own entries are its to park). |
| 229 | entry, ok := c.Inventory["ie-box"] |
| 230 | if !ok || len(entry.Paths) == 0 { |
| 231 | t.Fatalf("the install registered nothing: %+v", c.Inventory) |
| 232 | } |
| 233 | for p, stamp := range entry.Paths { |
| 234 | if stamp != s { |
| 235 | t.Fatalf("%s is tagged %d, want S = %d (F = %d)", p, stamp, s, f) |
| 236 | } |
| 237 | } |
| 238 | // The database is the target's rows, installed under S. |
| 239 | if n := countRowsForTest(t, provision.DefaultDBPath(h.sdir, p1Host)); n != h.rows+1 { |
| 240 | t.Fatalf("the returned database holds %d rows, the target wrote %d", n, h.rows+1) |
| 241 | } |
| 242 | }) |
| 243 | } |
| 244 | } |
| 245 | } |
| 246 | |
amadan