| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/sha256" |
| 6 | "encoding/hex" |
| 7 | "encoding/json" |
| 8 | "errors" |
| 9 | "fmt" |
| 10 | "io/fs" |
| 11 | "os" |
| 12 | "path/filepath" |
| 13 | "sort" |
| 14 | "strings" |
| 15 | "testing" |
| 16 | "time" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/appconfig" |
| 19 | "github.com/carlosframework/platform/internal/hostbox" |
| 20 | "github.com/carlosframework/platform/internal/hostclaim" |
| 21 | "github.com/carlosframework/platform/internal/hostguard" |
| 22 | "github.com/carlosframework/platform/internal/hostwrite" |
| 23 | "github.com/carlosframework/platform/internal/lease" |
| 24 | "github.com/carlosframework/platform/internal/moves" |
| 25 | "github.com/carlosframework/platform/internal/provision" |
| 26 | "github.com/carlosframework/platform/internal/registry" |
| 27 | "github.com/carlosframework/platform/internal/release" |
| 28 | ) |
| 29 | |
| 30 | // The final whole-branch review of Plan 4 (final-p4a-review.md, rulings in |
| 31 | // final-fix-rulings.md): the target's cleaning moves only what this op |
| 32 | // created, an abandoned build's location lease is released when the |
| 33 | // rollback starts, and the install's restore and install rights are granted |
| 34 | // at the phase that performs them. |
| 35 | |
| 36 | // treeSnapshot records every entry under the named paths (relative to dir) |
| 37 | // by identity, mode and bytes, so a test can tell a path left in place from |
| 38 | // one moved away and back or rewritten. |
| 39 | func treeSnapshot(t *testing.T, dir string, names ...string) map[string]string { |
| 40 | t.Helper() |
| 41 | out := map[string]string{} |
| 42 | for _, name := range names { |
| 43 | err := filepath.WalkDir(filepath.Join(dir, name), func(p string, d fs.DirEntry, err error) error { |
| 44 | if err != nil { |
| 45 | return err |
| 46 | } |
| 47 | rel, _ := filepath.Rel(dir, p) |
| 48 | fi, err := os.Lstat(p) |
| 49 | if err != nil { |
| 50 | return err |
| 51 | } |
| 52 | id, err := pathIdentity(p) |
| 53 | if err != nil { |
| 54 | return err |
| 55 | } |
| 56 | body := "" |
| 57 | if fi.Mode().IsRegular() { |
| 58 | b, err := os.ReadFile(p) |
| 59 | if err != nil { |
| 60 | return err |
| 61 | } |
| 62 | body = string(b) |
| 63 | } |
| 64 | out[rel] = fmt.Sprintf("%s %s %q", id, fi.Mode(), body) |
| 65 | return nil |
| 66 | }) |
| 67 | if err != nil { |
| 68 | t.Fatalf("snapshot %s: %v", name, err) |
| 69 | } |
| 70 | } |
| 71 | return out |
| 72 | } |
| 73 | |
| 74 | func sameSnapshot(a, b map[string]string) bool { |
| 75 | if len(a) != len(b) { |
| 76 | return false |
| 77 | } |
| 78 | for k, v := range a { |
| 79 | if b[k] != v { |
| 80 | return false |
| 81 | } |
| 82 | } |
| 83 | return true |
| 84 | } |
| 85 | |
| 86 | // cleanTarget drives a rolling_back record from the decision to cleaned on |
| 87 | // the target: the release, the source's recovery, the cleaning. |
| 88 | func (h *rollbackHarness) cleanTarget(t *testing.T) *provision.MoveRollback { |
| 89 | t.Helper() |
| 90 | h.rollbackPass(t) |
| 91 | if rb := h.rb(t); rb.TargetPhase != provision.RbTargetReleased && rb.TargetPhase != provision.RbTargetReturned { |
| 92 | t.Fatalf("the release: %+v", rb) |
| 93 | } |
| 94 | h.recovered(t) |
| 95 | h.rollbackPass(t) |
| 96 | rb := h.rb(t) |
| 97 | if rb.TargetPhase != provision.RbTargetCleaned || rb.TargetParkedAt == nil { |
| 98 | t.Fatalf("the cleaning did not complete: %+v (lease table %v)", rb, h.leases.holding()) |
| 99 | } |
| 100 | return rb |
| 101 | } |
| 102 | |
| 103 | // Finding 1: files the landing refused as foreign (an unregistered files |
| 104 | // directory and env file, RollbackFilesOccupied) stay where they are, byte |
| 105 | // for byte, through the rollback's cleaning, which still completes. |
| 106 | func TestRollbackCleaningKeepsWhatTheLandingRefused(t *testing.T) { |
| 107 | t.Parallel() |
| 108 | h := &rollbackHarness{landingHarness: newLandingHarness(t)} |
| 109 | h.flip(t) |
| 110 | h.sentinels(t) |
| 111 | foreign := []string{p1Host, p1Host + ".env"} |
| 112 | before := treeSnapshot(t, h.tdir, foreign...) |
| 113 | h.land(t) |
| 114 | if m := h.read(t, p1Host).Move; m.Reason != provision.RollbackFilesOccupied || m.LandingPhase != provision.LandLocal { |
| 115 | t.Fatalf("the landing did not refuse the foreign files: %+v", m) |
| 116 | } |
| 117 | h.rollbackPass(t) // the decision converts the mark |
| 118 | rolledBack(t, h.landingHarness, provision.RollbackFilesOccupied, provision.LandLocal) |
| 119 | h.cleanTarget(t) |
| 120 | h.sentinelsIntact(t, "after the cleaning") |
| 121 | if after := treeSnapshot(t, h.tdir, foreign...); !sameSnapshot(before, after) { |
| 122 | t.Fatalf("the foreign files changed:\nbefore %v\nafter %v", before, after) |
| 123 | } |
| 124 | // Nothing of theirs is in the parked tree, and the op's journal is. |
| 125 | var parkedForeign []string |
| 126 | filepath.WalkDir(filepath.Join(h.tdir, ".parked"), func(p string, d fs.DirEntry, err error) error { |
| 127 | if err == nil && (filepath.Base(p) == "sentinel.txt" || filepath.Base(p) == p1Host+".env") { |
| 128 | parkedForeign = append(parkedForeign, p) |
| 129 | } |
| 130 | return nil |
| 131 | }) |
| 132 | if len(parkedForeign) != 0 { |
| 133 | t.Fatalf("foreign files parked: %v", parkedForeign) |
| 134 | } |
| 135 | if _, err := os.Lstat(moves.JournalDir(h.tdir, p1Host, p1ID)); !errors.Is(err, os.ErrNotExist) { |
| 136 | t.Fatalf("the op journal is still live: %v", err) |
| 137 | } |
| 138 | } |
| 139 | |
| 140 | // Finding 1, the other half: what the op created is still parked by the |
| 141 | // cleaning, on its ownership evidence, and what it did not create is left. |
| 142 | // The landing placed its files and env (the placement receipt) and stopped |
| 143 | // before it registered them under F; a WAL and a rendered config were |
| 144 | // there that it never wrote. The cleaning parks the op's two and leaves the |
| 145 | // other two. Once the landing has installed its database and rendered its |
| 146 | // config (the restored CAS and the render's ack), the config and the |
| 147 | // database's WAL are the op's and are parked. |
| 148 | func TestRollbackCleaningParksWhatTheOpCreated(t *testing.T) { |
| 149 | t.Parallel() |
| 150 | t.Run("placed, not registered", func(t *testing.T) { |
| 151 | h := &rollbackHarness{landingHarness: newLandingHarness(t)} |
| 152 | h.flip(t) |
| 153 | db := provision.DefaultDBPath(h.tdir, p1Host) |
| 154 | for name, body := range map[string]string{db + "-wal": "a wal the landing never wrote", filepath.Join(h.tdir, p1Host+".config.env"): "OTHER=1\n"} { |
| 155 | if err := os.WriteFile(name, []byte(body), 0o600); err != nil { |
| 156 | t.Fatal(err) |
| 157 | } |
| 158 | } |
| 159 | foreign := []string{p1Host + ".db-wal", p1Host + ".config.env"} |
| 160 | before := treeSnapshot(t, h.tdir, foreign...) |
| 161 | h.landAt = "after-extracted" |
| 162 | h.crashLand(t) |
| 163 | j := h.landJournal(t) |
| 164 | if !j.Done(landStepFilesPlaced) || j.Done(landStepInventory) { |
| 165 | t.Fatalf("the fixture: %v", j.Steps) |
| 166 | } |
| 167 | if c := h.claim(t); hostclaim.InventoryHasStamp(c, "uk-box", c.Holder.Stamp) { |
| 168 | t.Fatal("the fixture registered the landing's paths") |
| 169 | } |
| 170 | h.rollBack(t) |
| 171 | h.cleanTarget(t) |
| 172 | for _, p := range []string{p1Host, p1Host + ".env"} { |
| 173 | if _, err := os.Lstat(filepath.Join(h.tdir, p)); !errors.Is(err, os.ErrNotExist) { |
| 174 | t.Fatalf("the landing's own %s is still live: %v", p, err) |
| 175 | } |
| 176 | } |
| 177 | parked := parkedUnder(t, h.tdir) |
| 178 | if !parkedHas(parked, "/artifacts/"+p1Host+"/upload.txt", "hello") { |
| 179 | t.Fatalf("the landing's files were not parked: %v", parked) |
| 180 | } |
| 181 | if after := treeSnapshot(t, h.tdir, foreign...); !sameSnapshot(before, after) { |
| 182 | t.Fatalf("what the op did not create changed:\nbefore %v\nafter %v", before, after) |
| 183 | } |
| 184 | }) |
| 185 | t.Run("installed and rendered", func(t *testing.T) { |
| 186 | h := newRollbackHarness(t) |
| 187 | db := provision.DefaultDBPath(h.tdir, p1Host) |
| 188 | config := filepath.Join(h.tdir, p1Host+".config.env") |
| 189 | if err := os.WriteFile(db+"-wal", []byte("the landing's wal"), 0o600); err != nil { |
| 190 | t.Fatal(err) |
| 191 | } |
| 192 | rendered := []byte("RENDERED=1\n") |
| 193 | if err := os.WriteFile(config, rendered, 0o600); err != nil { |
| 194 | t.Fatal(err) |
| 195 | } |
| 196 | sum := sha256.Sum256(rendered) |
| 197 | if err := appconfig.WriteConfigAck(context.Background(), h.s, appconfig.ConfigAck{Schema: appconfig.ConfigAckSchema, Box: "uk-box", Host: p1Host, |
| 198 | At: h.clock.Now(), MoveOp: p1ID, FileSHA256: hex.EncodeToString(sum[:])}); err != nil { |
| 199 | t.Fatal(err) |
| 200 | } |
| 201 | h.rollBack(t) |
| 202 | h.cleanTarget(t) |
| 203 | for _, p := range sourceArtifactPaths(p1Host) { |
| 204 | if _, err := os.Lstat(filepath.Join(h.tdir, p)); !errors.Is(err, os.ErrNotExist) { |
| 205 | t.Fatalf("%s is still live: %v", p, err) |
| 206 | } |
| 207 | } |
| 208 | parked := parkedUnder(t, h.tdir) |
| 209 | if !parkedHas(parked, "/artifacts/"+p1Host+".db-wal", "the landing's wal") || !parkedHas(parked, "/artifacts/"+p1Host+".config.env", "RENDERED=1\n") { |
| 210 | t.Fatalf("the op's leftovers were not parked: %v", parked) |
| 211 | } |
| 212 | }) |
| 213 | t.Run("a config the render did not write", func(t *testing.T) { |
| 214 | h := newRollbackHarness(t) |
| 215 | config := filepath.Join(h.tdir, p1Host+".config.env") |
| 216 | if err := os.WriteFile(config, []byte("ANOTHER=1\n"), 0o600); err != nil { |
| 217 | t.Fatal(err) |
| 218 | } |
| 219 | // The ack names this op, but another file's bytes. |
| 220 | sum := sha256.Sum256([]byte("RENDERED=1\n")) |
| 221 | if err := appconfig.WriteConfigAck(context.Background(), h.s, appconfig.ConfigAck{Schema: appconfig.ConfigAckSchema, Box: "uk-box", Host: p1Host, |
| 222 | At: h.clock.Now(), MoveOp: p1ID, FileSHA256: hex.EncodeToString(sum[:])}); err != nil { |
| 223 | t.Fatal(err) |
| 224 | } |
| 225 | before := treeSnapshot(t, h.tdir, p1Host+".config.env") |
| 226 | h.rollBack(t) |
| 227 | h.cleanTarget(t) |
| 228 | if after := treeSnapshot(t, h.tdir, p1Host+".config.env"); !sameSnapshot(before, after) { |
| 229 | t.Fatalf("a config the render did not write changed:\nbefore %v\nafter %v", before, after) |
| 230 | } |
| 231 | }) |
| 232 | } |
| 233 | |
| 234 | // parkedUnder maps every regular file under dir's .parked/ to its body. |
| 235 | func parkedUnder(t *testing.T, dir string) map[string]string { |
| 236 | t.Helper() |
| 237 | out := map[string]string{} |
| 238 | err := filepath.WalkDir(filepath.Join(dir, ".parked"), func(p string, d fs.DirEntry, err error) error { |
| 239 | if err != nil || d.IsDir() { |
| 240 | return err |
| 241 | } |
| 242 | body, err := os.ReadFile(p) |
| 243 | if err != nil { |
| 244 | return err |
| 245 | } |
| 246 | out[p] = string(body) |
| 247 | return nil |
| 248 | }) |
| 249 | if err != nil { |
| 250 | t.Fatal(err) |
| 251 | } |
| 252 | return out |
| 253 | } |
| 254 | |
| 255 | // Finding 2: a build that failed leaves the location's lease held across |
| 256 | // passes, as the landing designs; once the landing is rolled back that |
| 257 | // lease is abandoned and is released, whether the rollback is decided by |
| 258 | // this worker's landing pass or outside it. The same worker then cleans, |
| 259 | // and its park engine acquires the location's lease. |
| 260 | func TestRollbackReleasesTheAbandonedBuildLease(t *testing.T) { |
| 261 | t.Parallel() |
| 262 | failedBuild := func(t *testing.T) *rollbackHarness { |
| 263 | t.Helper() |
| 264 | h := &rollbackHarness{landingHarness: newLandingHarness(t)} |
| 265 | h.flip(t) |
| 266 | h.landAt = "after-build" |
| 267 | h.crashLand(t) |
| 268 | if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandBuilding { |
| 269 | t.Fatalf("the fixture: %+v", m) |
| 270 | } |
| 271 | if held := h.leases.holding(); len(held) != 1 { |
| 272 | t.Fatalf("the failed build holds %v, want the location's lease", held) |
| 273 | } |
| 274 | // The same worker carries on: its lease map is the build's. |
| 275 | h.rw = h.lw |
| 276 | return h |
| 277 | } |
| 278 | t.Run("decided in the landing pass", func(t *testing.T) { |
| 279 | h := failedBuild(t) |
| 280 | h.rollBack(t) |
| 281 | if held := h.leases.holding(); len(held) != 0 { |
| 282 | t.Fatalf("the rollback kept the build's lease: %v", held) |
| 283 | } |
| 284 | h.cleanTarget(t) |
| 285 | }) |
| 286 | t.Run("decided outside the landing pass", func(t *testing.T) { |
| 287 | h := failedBuild(t) |
| 288 | // Another worker decides the rollback (the decision is not this |
| 289 | // worker's pass); the target's next pass enters the rollback. |
| 290 | h.clock.Advance(cutLandingWindow + time.Minute) |
| 291 | other := &movesWorker{mirrors: map[string]lease.MirrorStore{}, cfg: h.rollbackCfg(t)} |
| 292 | if done, err := other.rollBackLanding(context.Background(), h.s, h.read(t, p1Host)); !done || err != nil { |
| 293 | t.Fatalf("the outside decision: %v %v", done, err) |
| 294 | } |
| 295 | if held := h.leases.holding(); len(held) != 1 { |
| 296 | t.Fatalf("the fixture: %v", held) |
| 297 | } |
| 298 | h.rollbackPass(t) |
| 299 | if held := h.leases.holding(); len(held) != 0 { |
| 300 | t.Fatalf("the rollback's entry kept the build's lease: %v", held) |
| 301 | } |
| 302 | h.rollbackPass(t) // idempotent: nothing held, nothing to release |
| 303 | h.cleanTarget(t) |
| 304 | }) |
| 305 | } |
| 306 | |
| 307 | // policyProbe checks, from inside an installer's own write hooks, that each |
| 308 | // write it makes is one hostbox.Policy.Check grants with MovesClaimPolicy as |
| 309 | // Policy.Moves: the selection is made as hostbox.WorkSource.Select makes it |
| 310 | // (the live claim's op, kind and phase, the installation record's capture), |
| 311 | // the row's hold is the registry's, and the record is read from the store. |
| 312 | type policyProbe struct { |
| 313 | t *testing.T |
| 314 | s release.Store |
| 315 | dataDir, box, region string |
| 316 | hold func() string |
| 317 | phase func(*provision.Record) string |
| 318 | granted, refused []string |
| 319 | } |
| 320 | |
| 321 | func (p *policyProbe) check(action string, id hostwrite.Identity) { |
| 322 | ctx := context.Background() |
| 323 | obs, err := hostclaim.Read(ctx, p.s, p1Host) |
| 324 | if err != nil || obs.Absent { |
| 325 | p.t.Errorf("read the claim: %v", err) |
| 326 | return |
| 327 | } |
| 328 | captured, err := hostguard.Capture(p.dataDir, p1Host) |
| 329 | if err != nil { |
| 330 | p.t.Errorf("capture: %v", err) |
| 331 | return |
| 332 | } |
| 333 | rec, err := provision.GetRecord(ctx, p.s, "bab", "app", p1Host) |
| 334 | if err != nil { |
| 335 | p.t.Errorf("read the record: %v", err) |
| 336 | return |
| 337 | } |
| 338 | c := obs.Claim |
| 339 | sel := hostbox.Selection{Host: p1Host, Action: action, Box: p.box, Region: p.region, Captured: captured, Kind: c.Op.Kind, Op: c.Op.ID, |
| 340 | Phase: c.Op.Phase, Generation: c.Holder.Generation, Session: c.Holder.Session} |
| 341 | pol := hostbox.Policy{Moves: MovesClaimPolicy(MovesRecordsFrom(p.s)), Held: func(context.Context, string) (string, error) { return p.hold(), nil }} |
| 342 | cell := p.phase(rec) + "/" + action |
| 343 | if err := pol.Check(ctx, c, sel, id); err != nil { |
| 344 | p.refused = append(p.refused, cell+": "+err.Error()) |
| 345 | return |
| 346 | } |
| 347 | p.granted = append(p.granted, cell) |
| 348 | } |
| 349 | |
| 350 | // wrap puts the probe in front of the installer's write hooks: the timers |
| 351 | // import (held-timers), the held render (held-config), the release gate |
| 352 | // (held-release) and the generation marker the database install writes |
| 353 | // inside its guard (restore and install). |
| 354 | func (p *policyProbe) wrap(c *moveCfg) { |
| 355 | timers, render, ready, gen := c.ReplaceTimers, c.RenderHeld, c.ReleaseReady, c.WriteGen |
| 356 | c.ReplaceTimers = func(ctx context.Context, host string, rows []registry.Timer) error { |
| 357 | p.check(movesActionHeldTimers, hostwrite.Identity{}) |
| 358 | return timers(ctx, host, rows) |
| 359 | } |
| 360 | c.RenderHeld = func(ctx context.Context, s release.Store, rt registry.Route, op string, pin *provision.MoveInputs, before func(string) error) (appconfig.HeldRender, error) { |
| 361 | hr, err := render(ctx, s, rt, op, pin, before) |
| 362 | if err == nil { |
| 363 | p.check(movesActionHeldConfig, HeldConfigIdentity(hr.BundleETags, hr.OverlayETag)) |
| 364 | } |
| 365 | return hr, err |
| 366 | } |
| 367 | c.ReleaseReady = func(ctx context.Context, s release.Store, rec provision.Record, v string) moves.Check { |
| 368 | p.check(movesActionHeldRelease, hostwrite.Identity{Release: v}) |
| 369 | return ready(ctx, s, rec, v) |
| 370 | } |
| 371 | if live := c.ReleaseLive; live != nil { |
| 372 | c.ReleaseLive = func(ctx context.Context, s release.Store, rec provision.Record, v string) moves.Check { |
| 373 | p.check(movesActionHeldRelease, hostwrite.Identity{Release: v}) |
| 374 | return live(ctx, s, rec, v) |
| 375 | } |
| 376 | } |
| 377 | c.WriteGen = func(path, generation string) error { |
| 378 | p.check(movesActionRestore, hostwrite.Identity{}) |
| 379 | p.check(movesActionInstall, hostwrite.Identity{}) |
| 380 | return gen(path, generation) |
| 381 | } |
| 382 | } |
| 383 | |
| 384 | func (p *policyProbe) want(t *testing.T, cells ...string) { |
| 385 | t.Helper() |
| 386 | if len(p.refused) != 0 { |
| 387 | t.Fatalf("Policy.Check refused the installer's own writes:\n%s", strings.Join(p.refused, "\n")) |
| 388 | } |
| 389 | got := map[string]bool{} |
| 390 | for _, c := range p.granted { |
| 391 | got[c] = true |
| 392 | } |
| 393 | var missing []string |
| 394 | for _, c := range cells { |
| 395 | if !got[c] { |
| 396 | missing = append(missing, c) |
| 397 | } |
| 398 | } |
| 399 | sort.Strings(missing) |
| 400 | if len(missing) != 0 { |
| 401 | t.Fatalf("never checked %v (granted %v)", missing, p.granted) |
| 402 | } |
| 403 | } |
| 404 | |
| 405 | // Finding 3 (J24, G8): the dirty return's installer, end to end, writes |
| 406 | // only what hostbox.Policy.Check grants under MovesClaimPolicy at the phase |
| 407 | // it writes in, with its row held by the rollback id: held-timers at local; |
| 408 | // held-config, held-release, restore and install at configured, whose |
| 409 | // closing CAS writes restored. |
| 410 | func TestDirtyReturnWritesPassThePolicy(t *testing.T) { |
| 411 | t.Parallel() |
| 412 | h := newReturnHarness(t, true) |
| 413 | p := &policyProbe{t: t, s: h.s, dataDir: h.sdir, box: "ie-box", region: "ie", hold: func() string { return h.srcRoute.HoldOp }, |
| 414 | phase: func(rec *provision.Record) string { return rec.Move.Rollback.SourcePhase }} |
| 415 | p.wrap(&h.rcfg) |
| 416 | pinned := h.read(t, p1Host).Move.Inputs |
| 417 | if refused, err := h.install(t); err != nil || refused != "" { |
| 418 | t.Fatalf("install: %q %v", refused, err) |
| 419 | } |
| 420 | h.installedReturn(t, pinned) |
| 421 | p.want(t, "local/held-timers", "configured/held-config", "configured/held-release", "configured/restore", "configured/install") |
| 422 | } |
| 423 | |
| 424 | // The landing's rows had the same off-by-one: its installer restores and |
| 425 | // installs inside configured too. Its writes, end to end, pass |
| 426 | // hostbox.Policy.Check under MovesClaimPolicy with the row held by the move. |
| 427 | func TestLandingWritesPassThePolicy(t *testing.T) { |
| 428 | t.Parallel() |
| 429 | h := newLandingHarness(t) |
| 430 | h.flip(t) |
| 431 | p := &policyProbe{t: t, s: h.s, dataDir: h.tdir, box: "uk-box", region: "uk", hold: func() string { return h.route.HoldOp }, |
| 432 | phase: func(rec *provision.Record) string { return rec.Move.LandingPhase }} |
| 433 | if h.lw == nil { |
| 434 | h.lw = &movesWorker{mirrors: map[string]lease.MirrorStore{}} |
| 435 | } |
| 436 | for i := 0; i < 4 && h.read(t, p1Host).Move.LandingPhase != provision.LandRestored; i++ { |
| 437 | h.lw.cfg = h.landingCfg(t) |
| 438 | p.wrap(&h.lw.cfg) |
| 439 | if err := movesPass(context.Background(), h.s, h.lw); err != nil { |
| 440 | t.Fatal(err) |
| 441 | } |
| 442 | } |
| 443 | if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandRestored { |
| 444 | t.Fatalf("the landing did not install: %+v", m) |
| 445 | } |
| 446 | p.want(t, "local/held-timers", "configured/held-config", "configured/held-release", "configured/restore", "configured/install") |
| 447 | } |
| 448 | |
| 449 | // plantSiblings writes a quarantine file and a litestream shadow directory |
| 450 | // beside the database at the flat paths. |
| 451 | func plantSiblings(t *testing.T, dir, body string) []string { |
| 452 | t.Helper() |
| 453 | corrupt := p1Host + ".db.corrupt" |
| 454 | shadow := ".activator-" + p1Host + ".db-litestream" |
| 455 | if err := os.MkdirAll(filepath.Join(dir, shadow), 0o750); err != nil { |
| 456 | t.Fatal(err) |
| 457 | } |
| 458 | for name, b := range map[string]string{corrupt: body, filepath.Join(shadow, "generation"): body} { |
| 459 | if err := os.WriteFile(filepath.Join(dir, name), []byte(b), 0o600); err != nil { |
| 460 | t.Fatal(err) |
| 461 | } |
| 462 | } |
| 463 | return []string{corrupt, shadow} |
| 464 | } |
| 465 | |
| 466 | // Finding 1, the re-review's residual (task-final-rereview-1.md): the |
| 467 | // quarantine file and the litestream shadow are the op's only when the |
| 468 | // install journalled them absent before it installed the database. One |
| 469 | // already there when the landing installed survives the cleaning byte for |
| 470 | // byte; ones made after the install are parked; with the journal entry |
| 471 | // missing (an install that recorded none), they are left. |
| 472 | func TestRollbackCleaningOwnsDatabaseSiblingsOnlyWhenRecordedAbsent(t *testing.T) { |
| 473 | t.Parallel() |
| 474 | t.Run("present before the install", func(t *testing.T) { |
| 475 | h := &rollbackHarness{landingHarness: newLandingHarness(t)} |
| 476 | h.flip(t) |
| 477 | foreign := plantSiblings(t, h.tdir, "not the landing's") |
| 478 | before := treeSnapshot(t, h.tdir, foreign...) |
| 479 | h.land(t) |
| 480 | h.installed(t) |
| 481 | h.step(t, provision.LandRestored, provision.LandProbing) |
| 482 | h.tpos = h.n |
| 483 | h.rollBack(t) |
| 484 | h.cleanTarget(t) |
| 485 | if after := treeSnapshot(t, h.tdir, foreign...); !sameSnapshot(before, after) { |
| 486 | t.Fatalf("the siblings the landing found changed:\nbefore %v\nafter %v", before, after) |
| 487 | } |
| 488 | }) |
| 489 | t.Run("made after the install", func(t *testing.T) { |
| 490 | h := newRollbackHarness(t) |
| 491 | made := plantSiblings(t, h.tdir, "the landing's instance") |
| 492 | h.rollBack(t) |
| 493 | h.cleanTarget(t) |
| 494 | for _, p := range made { |
| 495 | if _, err := os.Lstat(filepath.Join(h.tdir, p)); !errors.Is(err, os.ErrNotExist) { |
| 496 | t.Fatalf("%s is still live: %v", p, err) |
| 497 | } |
| 498 | } |
| 499 | parked := parkedUnder(t, h.tdir) |
| 500 | if !parkedHas(parked, "/artifacts/"+made[0], "the landing's instance") || !parkedHas(parked, "/artifacts/"+made[1]+"/generation", "the landing's instance") { |
| 501 | t.Fatalf("the op's siblings were not parked: %v", parked) |
| 502 | } |
| 503 | }) |
| 504 | t.Run("no record of their absence", func(t *testing.T) { |
| 505 | h := newRollbackHarness(t) |
| 506 | j := h.landJournal(t) |
| 507 | if _, ok := j.Steps[landStepPreInstall]; !ok { |
| 508 | t.Fatalf("the install journalled no %s step: %v", landStepPreInstall, j.Steps) |
| 509 | } |
| 510 | j.Unmark(landStepPreInstall) |
| 511 | if err := j.Commit(h.tdir); err != nil { |
| 512 | t.Fatal(err) |
| 513 | } |
| 514 | made := plantSiblings(t, h.tdir, "unrecorded") |
| 515 | before := treeSnapshot(t, h.tdir, made...) |
| 516 | h.rollBack(t) |
| 517 | h.cleanTarget(t) |
| 518 | if after := treeSnapshot(t, h.tdir, made...); !sameSnapshot(before, after) { |
| 519 | t.Fatalf("siblings with no recorded absence changed:\nbefore %v\nafter %v", before, after) |
| 520 | } |
| 521 | }) |
| 522 | } |
| 523 | |
| 524 | // A crash after pre_install is journalled and before the database's rename: |
| 525 | // the replay looks again right before its rename, so a sibling that |
| 526 | // appeared in between is recorded present and survives the cleaning. |
| 527 | func TestPreInstallReplayLooksAgain(t *testing.T) { |
| 528 | t.Parallel() |
| 529 | h := &rollbackHarness{landingHarness: newLandingHarness(t)} |
| 530 | h.flip(t) |
| 531 | h.landAt = "after-" + landStepPreInstall |
| 532 | h.crashLand(t) |
| 533 | foreign := plantSiblings(t, h.tdir, "appeared after the first look") |
| 534 | before := treeSnapshot(t, h.tdir, foreign...) |
| 535 | h.freshTarget() |
| 536 | h.land(t) |
| 537 | h.installed(t) |
| 538 | var pre preInstall |
| 539 | if err := json.Unmarshal([]byte(h.landJournal(t).Steps[landStepPreInstall]), &pre); err != nil || len(pre.Absent) != 0 { |
| 540 | t.Fatalf("pre_install after the replay: %+v %v", pre, err) |
| 541 | } |
| 542 | h.step(t, provision.LandRestored, provision.LandProbing) |
| 543 | h.tpos = h.n |
| 544 | h.rollBack(t) |
| 545 | h.cleanTarget(t) |
| 546 | if after := treeSnapshot(t, h.tdir, foreign...); !sameSnapshot(before, after) { |
| 547 | t.Fatalf("siblings that appeared before the replay changed:\nbefore %v\nafter %v", before, after) |
| 548 | } |
| 549 | } |
| 550 | |
amadan