| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "os" |
| 8 | "path/filepath" |
| 9 | "strings" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/activator" |
| 14 | "github.com/carlosframework/platform/internal/genselect" |
| 15 | "github.com/carlosframework/platform/internal/hostbox" |
| 16 | "github.com/carlosframework/platform/internal/hostclaim" |
| 17 | "github.com/carlosframework/platform/internal/hostguard" |
| 18 | "github.com/carlosframework/platform/internal/hostwrite" |
| 19 | "github.com/carlosframework/platform/internal/moves" |
| 20 | "github.com/carlosframework/platform/internal/moves/archive" |
| 21 | "github.com/carlosframework/platform/internal/provision" |
| 22 | "github.com/superfly/ltx" |
| 23 | ) |
| 24 | |
| 25 | var poWake = activator.StartPolicy{MaxStarts: 5, Window: 30 * time.Second} |
| 26 | |
| 27 | // committedAfterProbe is the owners' fixture after a passing probe and the |
| 28 | // commit: the move id admitted and published, the claim at |
| 29 | // landing/committed, the record committed, the row open. |
| 30 | func committedAfterProbe(t *testing.T) (*poFixture, hostclaim.Observed) { |
| 31 | t.Helper() |
| 32 | f := newPOFixture(t) |
| 33 | admitted, in := f.selectAndAdmit() |
| 34 | if err := f.owners.Materialize(t.Context(), admitted, in, newFenceWork()); err != nil { |
| 35 | t.Fatal(err) |
| 36 | } |
| 37 | c := *f.read().Claim |
| 38 | c.Op.Phase = hostclaim.PhaseLandingCommitted |
| 39 | f.putClaim(&c) |
| 40 | f.rec.Move.State, f.rec.Move.LandingPhase = provision.MoveCommitted, "" |
| 41 | f.route.Status, f.route.HoldOp = "ready", "" |
| 42 | return f, f.read() |
| 43 | } |
| 44 | |
| 45 | // Task 7 (Task 5's concern): the landing wake after the commit is a cold |
| 46 | // admission into a fresh generation from the certified remote, continuing |
| 47 | // the admitted session's generation; it carries the ordinary launch, no |
| 48 | // probe variable, and is evidenced, materialized, start-checked and |
| 49 | // published as serving's are. |
| 50 | func TestMovesProbeOwnersCommittedWake(t *testing.T) { |
| 51 | f, o := committedAfterProbe(t) |
| 52 | ctx := t.Context() |
| 53 | h := o.Claim.Holder |
| 54 | in, err := f.owners.Select(ctx, f.route, o, poWake) |
| 55 | if err != nil { |
| 56 | t.Fatal(err) |
| 57 | } |
| 58 | a := in.Admission |
| 59 | if a.ObservedOpID != poOp || a.ObservedPhase != hostclaim.PhaseLandingCommitted || a.ObservedSession != h.Session || a.ObservedGeneration != poOp || |
| 60 | a.Mode != "cold" || a.NewGeneration == "" || a.NewGeneration == poOp || a.KeepStamp || a.Coverage != "" || a.Region != "uk" || a.PredecessorRegion != "uk" { |
| 61 | t.Fatalf("the committed admission: %+v", a) |
| 62 | } |
| 63 | if in.Prepared || in.Baseline == nil || in.Baseline.Generation != poOp || in.Baseline.MaxTXID != 42 || a.RemoteMaxTXID != 42 { |
| 64 | t.Fatalf("the committed baseline: %+v", in.Baseline) |
| 65 | } |
| 66 | // Computed from the actual launch (fix round 2): its environment's |
| 67 | // report under the probe's normalisation, and its release; here they |
| 68 | // are the pins'. |
| 69 | if want := committedPinsIdentity(f.rec.Move.Inputs); in.Identity != want { |
| 70 | t.Fatalf("the committed wake's identity %+v, want %+v", in.Identity, want) |
| 71 | } |
| 72 | for _, kv := range in.Env { |
| 73 | if strings.HasPrefix(kv, "CARLOS_MOVE_PROBE=") { |
| 74 | t.Fatalf("the committed wake carries %s", kv) |
| 75 | } |
| 76 | } |
| 77 | if err := f.owners.AdmissionEvidence(ctx, o, a, in.Baseline); err != nil { |
| 78 | t.Fatal(err) |
| 79 | } |
| 80 | stale := *in.Baseline |
| 81 | stale.MaxTXID = 41 |
| 82 | if err := f.owners.AdmissionEvidence(ctx, o, a, &stale); !errors.Is(err, hostbox.ErrChanged) { |
| 83 | t.Fatalf("a stale baseline: %v", err) |
| 84 | } |
| 85 | admitted := f.admit(o, in) |
| 86 | f.writeLocal(poOp, admitted.Claim.Holder.Generation, f.inc) |
| 87 | if err := f.owners.Materialize(ctx, admitted, in, newFenceWork()); err != nil { |
| 88 | t.Fatalf("materialize: %v", err) |
| 89 | } |
| 90 | root := "instances/" + poHost + "/g-" + a.NewGeneration + "/0009/" + ltx.FormatFilename(1, 43) |
| 91 | f.bucket.Put("data", root, f.ltxHeader(1, 43)) |
| 92 | pub := ClaimPublication{Sync: func(context.Context) error { return nil }, Check: func(context.Context) error { return nil }} |
| 93 | if _, err := f.owners.PublishFenced(ctx, admitted, in, pub); err != nil { |
| 94 | t.Fatal(err) |
| 95 | } |
| 96 | if err := f.owners.PublicationEvidence(ctx, admitted); err != nil { |
| 97 | t.Fatal(err) |
| 98 | } |
| 99 | // A probe is never admitted at committed, and a committed wake never at |
| 100 | // probing; a record that is not committed refuses. |
| 101 | if _, err := f.owners.Select(ctx, f.route, o, poProbe); err == nil { |
| 102 | t.Fatal("a probe at committed") |
| 103 | } |
| 104 | f.rec.Move.State = provision.MoveLanding |
| 105 | if _, err := f.owners.Select(ctx, f.route, o, poWake); err == nil { |
| 106 | t.Fatal("a committed wake over a landing record") |
| 107 | } |
| 108 | } |
| 109 | |
| 110 | // filesGen reads the local .files-gen. |
| 111 | func (f *poFixture) filesGen() moves.FilesGen { |
| 112 | f.t.Helper() |
| 113 | raw, err := os.ReadFile(filepath.Join(f.files, archive.MarkerName)) |
| 114 | if err != nil { |
| 115 | f.t.Fatal(err) |
| 116 | } |
| 117 | var fg moves.FilesGen |
| 118 | if err := json.Unmarshal(raw, &fg); err != nil { |
| 119 | f.t.Fatal(err) |
| 120 | } |
| 121 | return fg |
| 122 | } |
| 123 | |
| 124 | // restore is ColdRestore's local effect on a fresh admission: <dbPath>.gen |
| 125 | // names the admitted generation. |
| 126 | func (f *poFixture) restore(gen string) { |
| 127 | f.t.Helper() |
| 128 | if err := hostbox.WriteGenerationMarker(f.route.DBPath+".gen", gen); err != nil { |
| 129 | f.t.Fatal(err) |
| 130 | } |
| 131 | } |
| 132 | |
| 133 | // publishFresh puts the fresh generation's first root and publishes it |
| 134 | // fenced, as the claim's sequence does once the session's root is durable. |
| 135 | func (f *poFixture) publishFresh(admitted hostclaim.Observed, in ClaimLaunchInputs) error { |
| 136 | f.t.Helper() |
| 137 | gen := admitted.Claim.Holder.Generation |
| 138 | f.bucket.Put("data", "instances/"+poHost+"/g-"+gen+"/0009/"+ltx.FormatFilename(1, 44), f.ltxHeader(1, 44)) |
| 139 | pub := ClaimPublication{Sync: func(context.Context) error { return nil }, Check: func(context.Context) error { return nil }} |
| 140 | _, err := f.owners.PublishFenced(f.t.Context(), admitted, in, pub) |
| 141 | return err |
| 142 | } |
| 143 | |
| 144 | // restartedProbePublished runs a dead first attempt and a restarted attempt |
| 145 | // that passes and publishes its fresh generation. |
| 146 | func restartedProbePublished(t *testing.T) (*poFixture, hostclaim.Observed, ClaimLaunchInputs) { |
| 147 | t.Helper() |
| 148 | f := newPOFixture(t) |
| 149 | ctx := t.Context() |
| 150 | first, in := f.selectAndAdmit() |
| 151 | if err := f.owners.Materialize(ctx, first, in, newFenceWork()); err != nil { |
| 152 | t.Fatal(err) |
| 153 | } |
| 154 | again, err := f.owners.Select(ctx, f.route, first, poProbe) |
| 155 | if err != nil { |
| 156 | t.Fatal(err) |
| 157 | } |
| 158 | restarted := f.admit(first, again) |
| 159 | f.restore(restarted.Claim.Holder.Generation) |
| 160 | if err := f.owners.Materialize(ctx, restarted, again, newFenceWork()); err != nil { |
| 161 | t.Fatalf("the restarted probe's start: %v", err) |
| 162 | } |
| 163 | return f, restarted, again |
| 164 | } |
| 165 | |
| 166 | // commit moves the fixture to the commit: the claim at landing/committed, |
| 167 | // the record committed, the row open. |
| 168 | func (f *poFixture) commit() hostclaim.Observed { |
| 169 | f.t.Helper() |
| 170 | c := *f.read().Claim |
| 171 | c.Op.Phase = hostclaim.PhaseLandingCommitted |
| 172 | f.putClaim(&c) |
| 173 | f.rec.Move.State, f.rec.Move.LandingPhase = provision.MoveCommitted, "" |
| 174 | f.route.Status, f.route.HoldOp = "ready", "" |
| 175 | return f.read() |
| 176 | } |
| 177 | |
| 178 | // Task 7 fix round 1, finding 1: a restarted probe's successful publication |
| 179 | // refreshes .files-gen to the published generation, under the writer's |
| 180 | // fence, after the files on disk are proved the landing's. After the commit, |
| 181 | // a process death and a wake, and then several more, each start passes |
| 182 | // Task 4's unweakened start check. |
| 183 | func TestMovesProbeOwnersRestartedProbeThenCommittedWakes(t *testing.T) { |
| 184 | f, restarted, again := restartedProbePublished(t) |
| 185 | ctx := t.Context() |
| 186 | if err := f.publishFresh(restarted, again); err != nil { |
| 187 | t.Fatal(err) |
| 188 | } |
| 189 | fresh := restarted.Claim.Holder.Generation |
| 190 | if fg := f.filesGen(); fg.Op != fresh || fg.Incarnation != f.inc { |
| 191 | t.Fatalf(".files-gen after the restarted probe's publication: %+v, want %s", fg, fresh) |
| 192 | } |
| 193 | o := f.commit() |
| 194 | for i := 0; i < 3; i++ { |
| 195 | in, err := f.owners.Select(ctx, f.route, o, poWake) |
| 196 | if err != nil { |
| 197 | t.Fatalf("wake %d: %v", i, err) |
| 198 | } |
| 199 | admitted := f.admit(o, in) |
| 200 | f.restore(admitted.Claim.Holder.Generation) |
| 201 | if err := f.owners.Materialize(ctx, admitted, in, newFenceWork()); err != nil { |
| 202 | t.Fatalf("wake %d's start after the probe's %s: %v", i, fresh, err) |
| 203 | } |
| 204 | if err := f.publishFresh(admitted, in); err != nil { |
| 205 | t.Fatalf("wake %d's publication: %v", i, err) |
| 206 | } |
| 207 | if fg := f.filesGen(); fg.Op != admitted.Claim.Holder.Generation { |
| 208 | t.Fatalf("wake %d's .files-gen: %+v", i, fg) |
| 209 | } |
| 210 | o = f.read() |
| 211 | } |
| 212 | } |
| 213 | |
| 214 | // A files directory that is not the one the landing placed fails the |
| 215 | // publication: no marker is written and .files-gen is not refreshed. |
| 216 | func TestMovesProbeOwnersPublicationRefusesMismatchedFiles(t *testing.T) { |
| 217 | f, restarted, again := restartedProbePublished(t) |
| 218 | before := f.bucket.LatestID("data", f.markerKey()) |
| 219 | // Another directory in its place: made while the placed one still |
| 220 | // exists, so it cannot reuse its inode. |
| 221 | old := f.files + ".old" |
| 222 | if err := os.Rename(f.files, old); err != nil { |
| 223 | t.Fatal(err) |
| 224 | } |
| 225 | if err := os.Mkdir(f.files, 0o700); err != nil { |
| 226 | t.Fatal(err) |
| 227 | } |
| 228 | t.Cleanup(func() { os.RemoveAll(old) }) |
| 229 | f.writeLocal(poOp, restarted.Claim.Holder.Generation, f.inc) |
| 230 | if err := f.publishFresh(restarted, again); err == nil { |
| 231 | t.Fatal("published over files the landing did not place") |
| 232 | } |
| 233 | if f.bucket.LatestID("data", f.markerKey()) != before { |
| 234 | t.Fatal("a marker was written") |
| 235 | } |
| 236 | if fg := f.filesGen(); fg.Op != poOp { |
| 237 | t.Fatalf(".files-gen was refreshed: %+v", fg) |
| 238 | } |
| 239 | } |
| 240 | |
| 241 | // platform-1e (claim Plan 10 rev 5): the moves owner's input policy binds |
| 242 | // the committed wake's spawn to Move.Inputs, as MovesClaimPolicy binds the |
| 243 | // landing's rows: a committed cold admission with the pins' launch identity |
| 244 | // is accepted, and refused once the pins differ. |
| 245 | func TestMovesProbeOwnersInputPolicyBindsTheCommittedWake(t *testing.T) { |
| 246 | f, o := committedAfterProbe(t) |
| 247 | ctx := t.Context() |
| 248 | in, err := f.owners.Select(ctx, f.route, o, poWake) |
| 249 | if err != nil { |
| 250 | t.Fatal(err) |
| 251 | } |
| 252 | admitted := f.admit(o, in) |
| 253 | c := admitted.Claim |
| 254 | sel := hostbox.Selection{Host: poHost, Action: "spawn", Box: "uk-box", Region: "uk", Captured: hostguard.Captured{Incarnation: c.Holder.Incarnation, Stamp: c.Holder.Stamp}, |
| 255 | Kind: c.Op.Kind, Op: c.Op.ID, Phase: c.Op.Phase, Generation: c.Holder.Generation, Session: c.Holder.Session} |
| 256 | policy := MovesClaimPolicyWithInputs(f.owners.Records, f.owners.InputPolicy) |
| 257 | if err := f.owners.InputPolicy(ctx, c, sel, in.Identity); err != nil { |
| 258 | t.Fatalf("matching pins: %v", err) |
| 259 | } |
| 260 | if err := policy(ctx, c, sel, in.Identity); err != nil { |
| 261 | t.Fatalf("matching pins through the policy: %v", err) |
| 262 | } |
| 263 | for name, mutate := range map[string]func(){ |
| 264 | "another release": func() { f.rec.Move.Inputs.Release = "v9" }, |
| 265 | "another env report": func() { f.rec.Move.Inputs.EnvReport = "sha256:other" }, |
| 266 | } { |
| 267 | pins := *f.rec.Move.Inputs |
| 268 | mutate() |
| 269 | if err := f.owners.InputPolicy(ctx, c, sel, in.Identity); err == nil { |
| 270 | t.Fatalf("%s: accepted", name) |
| 271 | } |
| 272 | if err := policy(ctx, c, sel, in.Identity); err == nil { |
| 273 | t.Fatalf("%s: accepted through the policy", name) |
| 274 | } |
| 275 | f.rec.Move.Inputs = &pins |
| 276 | } |
| 277 | if err := f.owners.InputPolicy(ctx, c, sel, f.launch.Identity); err == nil { |
| 278 | t.Fatal("the ordinary identity was accepted") |
| 279 | } |
| 280 | _ = genselect.Marker{} |
| 281 | _ = hostwrite.Identity{} |
| 282 | } |
| 283 | |
| 284 | // writeFilesGenOnly writes .files-gen alone. |
| 285 | func (f *poFixture) writeFilesGenOnly(op string) { |
| 286 | f.t.Helper() |
| 287 | body, _ := json.Marshal(moves.FilesGen{Op: op, Incarnation: f.inc}) |
| 288 | if err := os.WriteFile(filepath.Join(f.files, archive.MarkerName), body, 0o600); err != nil { |
| 289 | f.t.Fatal(err) |
| 290 | } |
| 291 | } |
| 292 | |
| 293 | // committedWake admits, restores and materializes one committed wake. |
| 294 | func (f *poFixture) committedWake(o hostclaim.Observed) (hostclaim.Observed, ClaimLaunchInputs, error) { |
| 295 | f.t.Helper() |
| 296 | in, err := f.owners.Select(f.t.Context(), f.route, o, poWake) |
| 297 | if err != nil { |
| 298 | f.t.Fatal(err) |
| 299 | } |
| 300 | admitted := f.admit(o, in) |
| 301 | f.restore(admitted.Claim.Holder.Generation) |
| 302 | return admitted, in, f.owners.Materialize(f.t.Context(), admitted, in, newFenceWork()) |
| 303 | } |
| 304 | |
| 305 | // Task 7 fix round 2, item 1: a publication interrupted between its marker |
| 306 | // write and the .files-gen refresh (local g1, published g2) is repaired |
| 307 | // before the next admission's (g3) start check: the published marker names |
| 308 | // g3's cold predecessor, and the files on disk are proved the landing's, so |
| 309 | // .files-gen moves to g2 and the start check, unweakened, passes. |
| 310 | func TestMovesProbeOwnersRepairAnInterruptedFilesGenRefresh(t *testing.T) { |
| 311 | f, restarted, again := restartedProbePublished(t) |
| 312 | if err := f.publishFresh(restarted, again); err != nil { |
| 313 | t.Fatal(err) |
| 314 | } |
| 315 | g1 := restarted.Claim.Holder.Generation |
| 316 | o := f.commit() |
| 317 | w1, in1, err := f.committedWake(o) |
| 318 | if err != nil { |
| 319 | t.Fatal(err) |
| 320 | } |
| 321 | if err := f.publishFresh(w1, in1); err != nil { |
| 322 | t.Fatal(err) |
| 323 | } |
| 324 | g2 := w1.Claim.Holder.Generation |
| 325 | // The crash: the marker names g2, .files-gen still g1. |
| 326 | f.writeFilesGenOnly(g1) |
| 327 | w2, in2, err := f.committedWake(f.read()) |
| 328 | if err != nil { |
| 329 | t.Fatalf("the next admission after an interrupted refresh: %v", err) |
| 330 | } |
| 331 | if fg := f.filesGen(); fg.Op != g2 { |
| 332 | t.Fatalf(".files-gen was not repaired to %s: %+v", g2, fg) |
| 333 | } |
| 334 | if err := f.publishFresh(w2, in2); err != nil { |
| 335 | t.Fatal(err) |
| 336 | } |
| 337 | if fg := f.filesGen(); fg.Op != w2.Claim.Holder.Generation { |
| 338 | t.Fatalf("after the next publication: %+v", fg) |
| 339 | } |
| 340 | } |
| 341 | |
| 342 | // Item 1: files that are not the landing's are never repaired over; the |
| 343 | // start check refuses as before. |
| 344 | func TestMovesProbeOwnersRepairRefusesMismatchedFiles(t *testing.T) { |
| 345 | f, restarted, again := restartedProbePublished(t) |
| 346 | if err := f.publishFresh(restarted, again); err != nil { |
| 347 | t.Fatal(err) |
| 348 | } |
| 349 | g1 := restarted.Claim.Holder.Generation |
| 350 | o := f.commit() |
| 351 | w1, in1, err := f.committedWake(o) |
| 352 | if err != nil { |
| 353 | t.Fatal(err) |
| 354 | } |
| 355 | if err := f.publishFresh(w1, in1); err != nil { |
| 356 | t.Fatal(err) |
| 357 | } |
| 358 | old := f.files + ".old" |
| 359 | if err := os.Rename(f.files, old); err != nil { |
| 360 | t.Fatal(err) |
| 361 | } |
| 362 | if err := os.Mkdir(f.files, 0o700); err != nil { |
| 363 | t.Fatal(err) |
| 364 | } |
| 365 | t.Cleanup(func() { os.RemoveAll(old) }) |
| 366 | f.writeFilesGenOnly(g1) |
| 367 | if _, _, err := f.committedWake(f.read()); !errors.Is(err, moves.ErrStartFiles) { |
| 368 | t.Fatalf("a repair over files the landing did not place: %v", err) |
| 369 | } |
| 370 | if fg := f.filesGen(); fg.Op != g1 { |
| 371 | t.Fatalf(".files-gen was rewritten: %+v", fg) |
| 372 | } |
| 373 | } |
| 374 | |
| 375 | // Item 2: the committed wake's input check judges the real launch. A changed |
| 376 | // launch environment with the record's pins unchanged is refused, by Select |
| 377 | // and by the input policy on the identity that launch would carry; matching |
| 378 | // inputs are accepted. |
| 379 | func TestMovesProbeOwnersCommittedWakeJudgesTheRealLaunch(t *testing.T) { |
| 380 | f, o := committedAfterProbe(t) |
| 381 | ctx := t.Context() |
| 382 | in, err := f.owners.Select(ctx, f.route, o, poWake) |
| 383 | if err != nil { |
| 384 | t.Fatalf("matching inputs: %v", err) |
| 385 | } |
| 386 | c := o.Claim |
| 387 | sel := hostbox.Selection{Host: poHost, Action: "spawn", Box: "uk-box", Region: "uk", Captured: hostguard.Captured{Incarnation: c.Holder.Incarnation, Stamp: c.Holder.Stamp}, |
| 388 | Kind: c.Op.Kind, Op: c.Op.ID, Phase: c.Op.Phase, Generation: c.Holder.Generation, Session: c.Holder.Session} |
| 389 | if err := f.owners.InputPolicy(ctx, c, sel, in.Identity); err != nil { |
| 390 | t.Fatalf("matching inputs through the input policy: %v", err) |
| 391 | } |
| 392 | f.launch.Env = []string{"A=2"} |
| 393 | if _, err := f.owners.Select(ctx, f.route, o, poWake); err == nil { |
| 394 | t.Fatal("a changed launch environment was selected") |
| 395 | } |
| 396 | changed := committedLaunchIdentity(envDigest(poEnvKey, f.launch.Env), f.launch.Version) |
| 397 | if err := f.owners.InputPolicy(ctx, c, sel, changed); err == nil { |
| 398 | t.Fatal("the changed launch's identity was accepted") |
| 399 | } |
| 400 | } |
| 401 | |
| 402 | // committedPinsIdentity is the identity a committed launch matching the pins |
| 403 | // carries. |
| 404 | func committedPinsIdentity(p *provision.MoveInputs) hostwrite.Identity { |
| 405 | return hostwrite.Identity{Config: committedEnvPrefix + p.EnvReport, Release: p.Release} |
| 406 | } |
| 407 | |
| 408 | // Task 7 fix round 3: the repair verifies content, never the directory's |
| 409 | // identity alone. With g2's publication interrupted before its refresh, a |
| 410 | // file modified, deleted or added inside the original directory (the same |
| 411 | // inode) refuses the repair: .files-gen stays g1 and the start check |
| 412 | // refuses. Matching contents still recover (TestMovesProbeOwnersRepairAnInterruptedFilesGenRefresh). |
| 413 | func TestMovesProbeOwnersRepairVerifiesTheFilesContent(t *testing.T) { |
| 414 | for name, change := range map[string]func(dir string) error{ |
| 415 | "a modified file": func(dir string) error { return os.WriteFile(filepath.Join(dir, "a.txt"), []byte("stale"), 0o600) }, |
| 416 | "a deleted file": func(dir string) error { return os.Remove(filepath.Join(dir, "a.txt")) }, |
| 417 | "an added file": func(dir string) error { return os.WriteFile(filepath.Join(dir, "b.txt"), []byte("new"), 0o600) }, |
| 418 | } { |
| 419 | t.Run(name, func(t *testing.T) { |
| 420 | f, restarted, again := restartedProbePublished(t) |
| 421 | if err := f.publishFresh(restarted, again); err != nil { |
| 422 | t.Fatal(err) |
| 423 | } |
| 424 | g1 := restarted.Claim.Holder.Generation |
| 425 | w1, in1, err := f.committedWake(f.commit()) |
| 426 | if err != nil { |
| 427 | t.Fatal(err) |
| 428 | } |
| 429 | if err := f.publishFresh(w1, in1); err != nil { |
| 430 | t.Fatal(err) |
| 431 | } |
| 432 | f.writeFilesGenOnly(g1) |
| 433 | if err := change(f.files); err != nil { |
| 434 | t.Fatal(err) |
| 435 | } |
| 436 | if _, _, err := f.committedWake(f.read()); !errors.Is(err, moves.ErrStartFiles) { |
| 437 | t.Fatalf("a repair over changed contents: %v", err) |
| 438 | } |
| 439 | if fg := f.filesGen(); fg.Op != g1 { |
| 440 | t.Fatalf(".files-gen was repaired over changed contents: %+v", fg) |
| 441 | } |
| 442 | }) |
| 443 | } |
| 444 | } |
| 445 | |
| 446 | // publishedP is the fixture at a published committed generation P: a |
| 447 | // restarted probe published (P), the commit, .files-gen naming P and P's |
| 448 | // files digest journalled. |
| 449 | func publishedP(t *testing.T) (*poFixture, string) { |
| 450 | t.Helper() |
| 451 | f, restarted, again := restartedProbePublished(t) |
| 452 | if err := f.publishFresh(restarted, again); err != nil { |
| 453 | t.Fatal(err) |
| 454 | } |
| 455 | f.commit() |
| 456 | return f, restarted.Claim.Holder.Generation |
| 457 | } |
| 458 | |
| 459 | // Final review, ruling 1: a committed wake G1 that failed before its marker |
| 460 | // leaves .files-gen at the published P while the next admission G2's cold |
| 461 | // predecessor is G1. The repair walks G1 back to P through unpublished cold |
| 462 | // admissions on this box and region and, the files matching P's recorded |
| 463 | // digest, sets .files-gen to G1: G2 starts (StartCheck unchanged) and |
| 464 | // publishes. |
| 465 | func TestMovesProbeOwnersRecoverACommittedWakeThatFailedBeforePublishing(t *testing.T) { |
| 466 | f, p := publishedP(t) |
| 467 | if _, _, err := f.committedWake(f.read()); err != nil { // G1, then it dies before PutMarker |
| 468 | t.Fatal(err) |
| 469 | } |
| 470 | if fg := f.filesGen(); fg.Op != p { |
| 471 | t.Fatalf("the fixture: %+v", fg) |
| 472 | } |
| 473 | g2, in2, err := f.committedWake(f.read()) |
| 474 | if err != nil { |
| 475 | t.Fatalf("G2 after an unpublished G1: %v", err) |
| 476 | } |
| 477 | if err := f.publishFresh(g2, in2); err != nil { |
| 478 | t.Fatal(err) |
| 479 | } |
| 480 | if fg := f.filesGen(); fg.Op != g2.Claim.Holder.Generation { |
| 481 | t.Fatalf(".files-gen after G2's publication: %+v", fg) |
| 482 | } |
| 483 | } |
| 484 | |
| 485 | // Ruling 1: two failed wakes, G1 and G2, then G3 starts. |
| 486 | func TestMovesProbeOwnersRecoverTwoFailedCommittedWakes(t *testing.T) { |
| 487 | f, _ := publishedP(t) |
| 488 | f.committedWake(f.read()) // G1, dies before PutMarker |
| 489 | f.committedWake(f.read()) // G2, dies before PutMarker |
| 490 | if _, _, err := f.committedWake(f.read()); err != nil { |
| 491 | t.Fatalf("G3 after two unpublished wakes: %v", err) |
| 492 | } |
| 493 | } |
| 494 | |
| 495 | // Ruling 1: a file changed after P's publication refuses the walk's repair; |
| 496 | // the start check refuses G2 and .files-gen stays P. |
| 497 | func TestMovesProbeOwnersCommittedWalkVerifiesContent(t *testing.T) { |
| 498 | f, p := publishedP(t) |
| 499 | if _, _, err := f.committedWake(f.read()); err != nil { // G1 |
| 500 | t.Fatal(err) |
| 501 | } |
| 502 | if err := os.WriteFile(filepath.Join(f.files, "a.txt"), []byte("changed after P"), 0o600); err != nil { |
| 503 | t.Fatal(err) |
| 504 | } |
| 505 | if _, _, err := f.committedWake(f.read()); !errors.Is(err, moves.ErrStartFiles) { |
| 506 | t.Fatalf("G2 over changed files: %v", err) |
| 507 | } |
| 508 | if fg := f.filesGen(); fg.Op != p { |
| 509 | t.Fatalf(".files-gen was repaired over changed files: %+v", fg) |
| 510 | } |
| 511 | } |
| 512 | |
amadan