| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "os" |
| 7 | "path/filepath" |
| 8 | "testing" |
| 9 | "time" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/activator" |
| 12 | "github.com/carlosframework/platform/internal/hostbox" |
| 13 | "github.com/carlosframework/platform/internal/hostclaim" |
| 14 | "github.com/carlosframework/platform/internal/hostguard" |
| 15 | "github.com/carlosframework/platform/internal/moves" |
| 16 | "github.com/carlosframework/platform/internal/provision" |
| 17 | ) |
| 18 | |
| 19 | // The final whole-branch review's three blockers (final-review.md, |
| 20 | // rulings in final-fix.md): the landing never deletes files it did not |
| 21 | // create, every local installation runs inside the host guard with the |
| 22 | // destination checked there, and the absolute cut deadline bounds the |
| 23 | // landing as well as landing_deadline. |
| 24 | |
| 25 | // sentinels plants files this landing never wrote at the live files paths: |
| 26 | // a file in the files directory and <host>.env. No installation record and |
| 27 | // no inventory names them, so local retires nothing (ruling G5). |
| 28 | func (h *landingHarness) sentinels(t *testing.T) { |
| 29 | t.Helper() |
| 30 | if err := os.MkdirAll(filepath.Join(h.tdir, p1Host), 0o755); err != nil { |
| 31 | t.Fatal(err) |
| 32 | } |
| 33 | for name, body := range map[string]string{filepath.Join(p1Host, "sentinel.txt"): "not the landing's", p1Host + ".env": "SENTINEL=1\n"} { |
| 34 | if err := os.WriteFile(filepath.Join(h.tdir, name), []byte(body), 0o644); err != nil { |
| 35 | t.Fatal(err) |
| 36 | } |
| 37 | } |
| 38 | } |
| 39 | |
| 40 | // sentinelsIntact fails unless both sentinels are exactly as planted. |
| 41 | func (h *landingHarness) sentinelsIntact(t *testing.T, when string) { |
| 42 | t.Helper() |
| 43 | for name, want := range map[string]string{filepath.Join(p1Host, "sentinel.txt"): "not the landing's", p1Host + ".env": "SENTINEL=1\n"} { |
| 44 | body, err := os.ReadFile(filepath.Join(h.tdir, name)) |
| 45 | if err != nil || string(body) != want { |
| 46 | t.Fatalf("%s: %s is %q (%v), want %q", when, name, body, err, want) |
| 47 | } |
| 48 | } |
| 49 | if _, err := os.Stat(filepath.Join(h.tdir, p1Host, "upload.txt")); !errors.Is(err, os.ErrNotExist) { |
| 50 | t.Fatalf("%s: the archive was installed over the sentinels: %v", when, err) |
| 51 | } |
| 52 | } |
| 53 | |
| 54 | // retireInstallation retires the installation record's active stamp, as a |
| 55 | // successor taking this box over would before installing its own. |
| 56 | func (h *landingHarness) retireInstallation(t *testing.T) { |
| 57 | t.Helper() |
| 58 | g := hostguard.Guard{DataDir: h.tdir, LockDir: t.TempDir(), Claims: hostguard.StoreReader{Store: h.s}} |
| 59 | if err := g.RetireInstallation(context.Background(), p1Host, func(*hostclaim.Claim) error { return nil }); err != nil { |
| 60 | t.Fatal(err) |
| 61 | } |
| 62 | } |
| 63 | |
| 64 | // Finding 1: files at the live paths that no earlier stay registered are |
| 65 | // not the landing's. Across two passes (and a fresh worker between them, |
| 66 | // so any attempt marker the first pass journalled is what the second |
| 67 | // reads), they are preserved, and the landing records why it stops. |
| 68 | func TestLandingNeverDeletesFilesItDidNotCreate(t *testing.T) { |
| 69 | t.Parallel() |
| 70 | t.Run("present before the landing", func(t *testing.T) { |
| 71 | h := newLandingHarness(t) |
| 72 | h.flip(t) |
| 73 | h.sentinels(t) |
| 74 | h.land(t) |
| 75 | h.sentinelsIntact(t, "after the first pass") |
| 76 | h.freshTarget() |
| 77 | h.land(t) |
| 78 | h.sentinelsIntact(t, "after the second pass") |
| 79 | m := h.read(t, p1Host).Move |
| 80 | if m.Reason != provision.RollbackFilesOccupied || m.LandingPhase != provision.LandLocal { |
| 81 | t.Fatalf("record: %+v", m) |
| 82 | } |
| 83 | if _, ok := h.filesGen(t); ok { |
| 84 | t.Fatal(".files-gen written over files the landing did not write") |
| 85 | } |
| 86 | }) |
| 87 | t.Run("only an env file", func(t *testing.T) { |
| 88 | h := newLandingHarness(t) |
| 89 | h.flip(t) |
| 90 | if err := os.WriteFile(filepath.Join(h.tdir, p1Host+".env"), []byte("SENTINEL=1\n"), 0o644); err != nil { |
| 91 | t.Fatal(err) |
| 92 | } |
| 93 | h.land(t) |
| 94 | h.freshTarget() |
| 95 | h.land(t) |
| 96 | if body, err := os.ReadFile(filepath.Join(h.tdir, p1Host+".env")); err != nil || string(body) != "SENTINEL=1\n" { |
| 97 | t.Fatalf("the env file the landing did not write: %q %v", body, err) |
| 98 | } |
| 99 | if m := h.read(t, p1Host).Move; m.Reason != provision.RollbackFilesOccupied { |
| 100 | t.Fatalf("record: %+v", m) |
| 101 | } |
| 102 | }) |
| 103 | // The review's failure: the attempt is journalled, then the files |
| 104 | // appear (the landing's own crash in between), and the retry reads the |
| 105 | // journal. The point is the one after the journal step that begins the |
| 106 | // extraction, whatever it is called. |
| 107 | t.Run("appearing after the attempt began", func(t *testing.T) { |
| 108 | h := newLandingHarness(t) |
| 109 | h.flip(t) |
| 110 | planted := false |
| 111 | h.landFault = func(p string) error { |
| 112 | if !planted && (p == "after-extracting" || p == "after-staged") { |
| 113 | planted = true |
| 114 | h.sentinels(t) |
| 115 | return errCrash |
| 116 | } |
| 117 | return nil |
| 118 | } |
| 119 | h.land(t) |
| 120 | if !planted { |
| 121 | t.Fatalf("the extraction never began: %v", h.landHits) |
| 122 | } |
| 123 | h.sentinelsIntact(t, "after the crash") |
| 124 | h.freshTarget() |
| 125 | h.land(t) |
| 126 | h.sentinelsIntact(t, "after the retry") |
| 127 | h.freshTarget() |
| 128 | h.land(t) |
| 129 | h.sentinelsIntact(t, "after a third pass") |
| 130 | if m := h.read(t, p1Host).Move; m.Reason != provision.RollbackFilesOccupied { |
| 131 | t.Fatalf("record: %+v", m) |
| 132 | } |
| 133 | }) |
| 134 | // The marker vouches for the tree this landing placed: a files |
| 135 | // directory swapped after the placement is not written into. |
| 136 | t.Run("replaced after the placement", func(t *testing.T) { |
| 137 | h := newLandingHarness(t) |
| 138 | h.flip(t) |
| 139 | dir := filepath.Join(h.tdir, p1Host) |
| 140 | h.landFault = func(p string) error { |
| 141 | if p == "before-files-gen" { |
| 142 | if err := os.Rename(dir, dir+".moved"); err != nil { |
| 143 | t.Fatal(err) |
| 144 | } |
| 145 | if err := os.Mkdir(dir, 0o755); err != nil { |
| 146 | t.Fatal(err) |
| 147 | } |
| 148 | if err := os.WriteFile(filepath.Join(dir, "sentinel.txt"), []byte("not the landing's"), 0o644); err != nil { |
| 149 | t.Fatal(err) |
| 150 | } |
| 151 | } |
| 152 | return nil |
| 153 | } |
| 154 | h.land(t) |
| 155 | if _, ok := h.filesGen(t); ok { |
| 156 | t.Fatal(".files-gen written into a files directory the landing did not place") |
| 157 | } |
| 158 | if body, err := os.ReadFile(filepath.Join(dir, "sentinel.txt")); err != nil || string(body) != "not the landing's" { |
| 159 | t.Fatalf("the sentinel: %q %v", body, err) |
| 160 | } |
| 161 | if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandLocal { |
| 162 | t.Fatalf("record: %+v", m) |
| 163 | } |
| 164 | }) |
| 165 | // An empty files directory is the landing's to fill (as Extract has |
| 166 | // always allowed), and a retry after a crash inside the install, with |
| 167 | // the files moved and the env not, finishes from the staged tree. |
| 168 | t.Run("an empty directory and a crash mid-install", func(t *testing.T) { |
| 169 | h := newLandingHarness(t) |
| 170 | h.flip(t) |
| 171 | if err := os.MkdirAll(filepath.Join(h.tdir, p1Host), 0o750); err != nil { |
| 172 | t.Fatal(err) |
| 173 | } |
| 174 | h.landAt = "after-files-rename" |
| 175 | h.crashLand(t) |
| 176 | if _, err := os.Stat(filepath.Join(h.tdir, p1Host, "upload.txt")); err != nil { |
| 177 | t.Fatalf("the files were not moved before the crash: %v", err) |
| 178 | } |
| 179 | h.freshTarget() |
| 180 | h.land(t) |
| 181 | h.installed(t) |
| 182 | if fi, err := os.Stat(filepath.Join(h.tdir, p1Host)); err != nil || fi.Mode().Perm() != 0o750 { |
| 183 | t.Fatalf("the files directory's mode: %v %v", fi, err) |
| 184 | } |
| 185 | if _, err := os.Stat(filepath.Join(h.tdir, ".moves", p1Host, p1ID, "extract")); !errors.Is(err, os.ErrNotExist) { |
| 186 | t.Fatalf("the staging was left behind: %v", err) |
| 187 | } |
| 188 | }) |
| 189 | } |
| 190 | |
| 191 | // Finding 2: the installation is checked and made inside the host guard. |
| 192 | // A successor that appears between the landing's observation and its |
| 193 | // install is never replaced. |
| 194 | func TestLandingInstallRefusesASuccessor(t *testing.T) { |
| 195 | t.Parallel() |
| 196 | dbPath := func(h *landingHarness) string { return provision.DefaultDBPath(h.tdir, p1Host) } |
| 197 | notRestored := func(t *testing.T, h *landingHarness) { |
| 198 | t.Helper() |
| 199 | m := h.read(t, p1Host).Move |
| 200 | if m.LandingPhase == provision.LandRestored || (m.Landing != nil && m.Landing.InstalledDBSHA256 != "") { |
| 201 | t.Fatalf("the landing completed over a successor: %+v", m) |
| 202 | } |
| 203 | } |
| 204 | t.Run("a successor database", func(t *testing.T) { |
| 205 | h := newLandingHarness(t) |
| 206 | h.flip(t) |
| 207 | h.landFault = func(p string) error { |
| 208 | if p == "before-db-verify" { |
| 209 | if err := os.WriteFile(dbPath(h), []byte("successor database"), 0o644); err != nil { |
| 210 | t.Fatal(err) |
| 211 | } |
| 212 | if err := hostbox.WriteGenerationMarker(dbPath(h)+".gen", "successor"); err != nil { |
| 213 | t.Fatal(err) |
| 214 | } |
| 215 | } |
| 216 | return nil |
| 217 | } |
| 218 | h.land(t) |
| 219 | if !containsString(h.landHits, "before-db-verify") { |
| 220 | t.Fatalf("the install never observed: %v", h.landHits) |
| 221 | } |
| 222 | if body, err := os.ReadFile(dbPath(h)); err != nil || string(body) != "successor database" { |
| 223 | t.Fatalf("the successor's database was replaced: %q %v", body, err) |
| 224 | } |
| 225 | if gen, err := hostbox.ReadGenerationMarker(dbPath(h) + ".gen"); err != nil || gen != "successor" { |
| 226 | t.Fatalf("the successor's marker: %q %v", gen, err) |
| 227 | } |
| 228 | notRestored(t, h) |
| 229 | }) |
| 230 | t.Run("a successor database without a marker", func(t *testing.T) { |
| 231 | h := newLandingHarness(t) |
| 232 | h.flip(t) |
| 233 | h.landFault = func(p string) error { |
| 234 | if p == "before-db-verify" { |
| 235 | if err := os.WriteFile(dbPath(h), []byte("successor database"), 0o644); err != nil { |
| 236 | t.Fatal(err) |
| 237 | } |
| 238 | } |
| 239 | return nil |
| 240 | } |
| 241 | h.land(t) |
| 242 | if body, err := os.ReadFile(dbPath(h)); err != nil || string(body) != "successor database" { |
| 243 | t.Fatalf("the successor's database was replaced: %q %v", body, err) |
| 244 | } |
| 245 | if _, err := os.Stat(dbPath(h) + ".gen"); !errors.Is(err, os.ErrNotExist) { |
| 246 | t.Fatalf("the successor's database was labelled as this landing's: %v", err) |
| 247 | } |
| 248 | notRestored(t, h) |
| 249 | }) |
| 250 | // A symlink at a destination is never this landing's, dangling or not: |
| 251 | // it is not read through (a dangling one would look absent) and never |
| 252 | // replaced (final re-review residual). |
| 253 | for _, tc := range []struct{ name, suffix string }{{"a dangling marker symlink", ".gen"}, {"a dangling database symlink", ""}, {"a dangling WAL symlink", "-wal"}} { |
| 254 | t.Run(tc.name, func(t *testing.T) { |
| 255 | h := newLandingHarness(t) |
| 256 | h.flip(t) |
| 257 | link, target := dbPath(h)+tc.suffix, filepath.Join(h.tdir, "elsewhere", "nothing") |
| 258 | h.landFault = func(p string) error { |
| 259 | if p == "before-db-verify" { |
| 260 | if err := os.Symlink(target, link); err != nil { |
| 261 | t.Fatal(err) |
| 262 | } |
| 263 | } |
| 264 | return nil |
| 265 | } |
| 266 | h.land(t) |
| 267 | if !containsString(h.landHits, "before-db-verify") { |
| 268 | t.Fatalf("the install never observed: %v", h.landHits) |
| 269 | } |
| 270 | fi, err := os.Lstat(link) |
| 271 | if err != nil || fi.Mode()&os.ModeSymlink == 0 { |
| 272 | t.Fatalf("%s is no longer the symlink the landing did not write: %v %v", link, fi, err) |
| 273 | } |
| 274 | if got, err := os.Readlink(link); err != nil || got != target { |
| 275 | t.Fatalf("%s points at %q (%v), want %q", link, got, err, target) |
| 276 | } |
| 277 | if _, err := os.Lstat(target); !errors.Is(err, os.ErrNotExist) { |
| 278 | t.Fatalf("something was written through the symlink: %v", err) |
| 279 | } |
| 280 | if tc.suffix != "" { |
| 281 | if _, err := os.Lstat(dbPath(h)); !errors.Is(err, os.ErrNotExist) { |
| 282 | t.Fatalf("installed beside a symlink the landing did not write: %v", err) |
| 283 | } |
| 284 | } |
| 285 | notRestored(t, h) |
| 286 | }) |
| 287 | } |
| 288 | t.Run("a successor marker", func(t *testing.T) { |
| 289 | h := newLandingHarness(t) |
| 290 | h.flip(t) |
| 291 | h.landFault = func(p string) error { |
| 292 | if p == "before-db-verify" { |
| 293 | if err := hostbox.WriteGenerationMarker(dbPath(h)+".gen", "successor"); err != nil { |
| 294 | t.Fatal(err) |
| 295 | } |
| 296 | } |
| 297 | return nil |
| 298 | } |
| 299 | h.land(t) |
| 300 | if _, err := os.Stat(dbPath(h)); !errors.Is(err, os.ErrNotExist) { |
| 301 | t.Fatalf("installed beside a successor's marker: %v", err) |
| 302 | } |
| 303 | if gen, err := hostbox.ReadGenerationMarker(dbPath(h) + ".gen"); err != nil || gen != "successor" { |
| 304 | t.Fatalf("the successor's marker: %q %v", gen, err) |
| 305 | } |
| 306 | notRestored(t, h) |
| 307 | }) |
| 308 | // The installation retired between the observation and the install: |
| 309 | // only the guard can see it, since nothing is at the destination yet. |
| 310 | t.Run("the installation retired before the database install", func(t *testing.T) { |
| 311 | h := newLandingHarness(t) |
| 312 | h.flip(t) |
| 313 | h.landFault = func(p string) error { |
| 314 | if p == "before-db-verify" { |
| 315 | h.retireInstallation(t) |
| 316 | } |
| 317 | return nil |
| 318 | } |
| 319 | h.land(t) |
| 320 | for _, p := range []string{dbPath(h), dbPath(h) + ".gen"} { |
| 321 | if _, err := os.Stat(p); !errors.Is(err, os.ErrNotExist) { |
| 322 | t.Fatalf("%s installed under a retired installation: %v", p, err) |
| 323 | } |
| 324 | } |
| 325 | notRestored(t, h) |
| 326 | }) |
| 327 | t.Run("the installation retired before the files install", func(t *testing.T) { |
| 328 | h := newLandingHarness(t) |
| 329 | h.flip(t) |
| 330 | retired := false |
| 331 | h.landFault = func(p string) error { |
| 332 | if !retired && (p == "after-extracting" || p == "after-staged") { |
| 333 | retired = true |
| 334 | h.retireInstallation(t) |
| 335 | } |
| 336 | return nil |
| 337 | } |
| 338 | h.land(t) |
| 339 | if !retired { |
| 340 | t.Fatalf("the extraction never began: %v", h.landHits) |
| 341 | } |
| 342 | for _, p := range []string{filepath.Join(h.tdir, p1Host), filepath.Join(h.tdir, p1Host+".env")} { |
| 343 | if _, err := os.Stat(p); !errors.Is(err, os.ErrNotExist) { |
| 344 | t.Fatalf("%s installed under a retired installation: %v", p, err) |
| 345 | } |
| 346 | } |
| 347 | if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandLocal { |
| 348 | t.Fatalf("record: %+v", m) |
| 349 | } |
| 350 | }) |
| 351 | } |
| 352 | |
| 353 | // Finding 3: the landing is bounded by the earlier of the cut's absolute |
| 354 | // deadline and landing_deadline (spec §7.4). The cut deadline here falls a |
| 355 | // minute after the flip; the landing deadline is fifteen minutes away. |
| 356 | func TestLandingCutDeadlineMarksRollback(t *testing.T) { |
| 357 | t.Parallel() |
| 358 | ctx := context.Background() |
| 359 | setCutDeadline := func(t *testing.T, h *landingHarness) time.Time { |
| 360 | t.Helper() |
| 361 | rec := h.read(t, p1Host) |
| 362 | d := rec.Move.FlippedAt.Add(time.Minute) |
| 363 | if _, err := provision.SetRecordMove(ctx, h.s, rec.Account, rec.App, rec.Host, func(r *provision.Record) error { |
| 364 | r.Move.CutDeadline = &d |
| 365 | return nil |
| 366 | }); err != nil { |
| 367 | t.Fatal(err) |
| 368 | } |
| 369 | if ld := h.read(t, p1Host).Move.LandingDeadline; ld == nil || !ld.After(d.Add(2*time.Minute)) { |
| 370 | t.Fatalf("fixture: landing deadline %v is not well after the cut deadline %v", ld, d) |
| 371 | } |
| 372 | return d |
| 373 | } |
| 374 | pastCut := func(h *landingHarness, d time.Time) { |
| 375 | h.clock.Advance(d.Sub(h.clock.Now()) + time.Second) |
| 376 | } |
| 377 | t.Run("at checking", func(t *testing.T) { |
| 378 | h := newLandingHarness(t) |
| 379 | h.flip(t) |
| 380 | d := setCutDeadline(t, h) |
| 381 | pastCut(h, d) |
| 382 | h.land(t) |
| 383 | // The rollback decision reads the earlier deadline too (J11). |
| 384 | rolledBack(t, h, provision.RollbackLandingDeadline, provision.LandChecking) |
| 385 | }) |
| 386 | t.Run("at configured while the release waits", func(t *testing.T) { |
| 387 | h := newLandingHarness(t) |
| 388 | h.flip(t) |
| 389 | d := setCutDeadline(t, h) |
| 390 | h.release = moves.Check{Reason: t7NotProven} |
| 391 | h.land(t) |
| 392 | if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandConfigured || m.Reason != "" { |
| 393 | t.Fatalf("waiting on the proof: %+v", m) |
| 394 | } |
| 395 | pastCut(h, d) |
| 396 | h.release = moves.Check{OK: true} |
| 397 | h.land(t) |
| 398 | rolledBack(t, h, provision.RollbackLandingDeadline, provision.LandConfigured) |
| 399 | if _, err := os.Stat(provision.DefaultDBPath(h.tdir, p1Host)); !errors.Is(err, os.ErrNotExist) { |
| 400 | t.Fatalf("installed past the cut deadline: %v", err) |
| 401 | } |
| 402 | }) |
| 403 | t.Run("before the install", func(t *testing.T) { |
| 404 | h := newLandingHarness(t) |
| 405 | h.flip(t) |
| 406 | d := setCutDeadline(t, h) |
| 407 | h.landFault = func(p string) error { |
| 408 | if p == "after-release-ready" { |
| 409 | pastCut(h, d) |
| 410 | } |
| 411 | return nil |
| 412 | } |
| 413 | h.land(t) |
| 414 | m := h.read(t, p1Host).Move |
| 415 | if m.Reason != provision.RollbackLandingDeadline || m.LandingPhase != provision.LandConfigured || (m.Landing != nil && m.Landing.InstalledDBSHA256 != "") { |
| 416 | t.Fatalf("past the cut deadline before the install: %+v", m) |
| 417 | } |
| 418 | if _, err := os.Stat(provision.DefaultDBPath(h.tdir, p1Host)); !errors.Is(err, os.ErrNotExist) { |
| 419 | t.Fatalf("installed past the cut deadline: %v", err) |
| 420 | } |
| 421 | }) |
| 422 | } |
| 423 | |
| 424 | // Re-review finding 8: a seed marker left beside the destination by some |
| 425 | // earlier seed never survives the landing's database install, so the |
| 426 | // installed database is never read as that seed's leftover. |
| 427 | func TestLandingInstallRemovesAStaleSeedMarker(t *testing.T) { |
| 428 | t.Parallel() |
| 429 | h := newLandingHarness(t) |
| 430 | h.flip(t) |
| 431 | h.route.Replication = "" |
| 432 | db := provision.DefaultDBPath(h.tdir, p1Host) |
| 433 | if err := os.MkdirAll(filepath.Dir(db), 0o700); err != nil { |
| 434 | t.Fatal(err) |
| 435 | } |
| 436 | if err := os.WriteFile(db+activator.SeedMarkerSuffix, []byte("sd_aaaaaaaaaaaaaaaaaaaaaaaaaa\n"), 0o600); err != nil { |
| 437 | t.Fatal(err) |
| 438 | } |
| 439 | h.land(t) |
| 440 | h.installed(t) |
| 441 | if _, err := os.Stat(db); err != nil { |
| 442 | t.Fatalf("the database was not installed: %v", err) |
| 443 | } |
| 444 | if _, err := os.Lstat(db + activator.SeedMarkerSuffix); !errors.Is(err, os.ErrNotExist) { |
| 445 | t.Fatalf("the stale seed marker survived the install: %v", err) |
| 446 | } |
| 447 | } |
| 448 | |
amadan