| 1 | //go:build linux |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "errors" |
| 7 | "os" |
| 8 | "path/filepath" |
| 9 | "sync/atomic" |
| 10 | "syscall" |
| 11 | "testing" |
| 12 | "time" |
| 13 | |
| 14 | "github.com/carlosframework/platform/internal/hostbox" |
| 15 | "github.com/carlosframework/platform/internal/hostclaim" |
| 16 | "github.com/carlosframework/platform/internal/registry" |
| 17 | "github.com/carlosframework/platform/internal/release" |
| 18 | ) |
| 19 | |
| 20 | // The held-stage lock's file is removed when the stage finishes (region HA |
| 21 | // M-K25, answered by moves on #236): main's empty-installation census |
| 22 | // refuses a file that names the host being installed, so a lock left |
| 23 | // behind would refuse every later from-empty install of that host on the |
| 24 | // box that held a stage for it. |
| 25 | |
| 26 | // heldLockCensus runs main's empty-installation census for a from-empty |
| 27 | // install of host on a box whose data root is data, with the production |
| 28 | // classification (claimGlobalSharedPaths) and any extra data entries. |
| 29 | func heldLockCensus(t *testing.T, data, host string, extra ...string) error { |
| 30 | t.Helper() |
| 31 | base := t.TempDir() |
| 32 | inventory, err := openLegacyInventory(t, filepath.Join(base, "inventory.db")) |
| 33 | if err != nil { |
| 34 | t.Fatal(err) |
| 35 | } |
| 36 | defer inventory.Close() |
| 37 | reg, err := registry.Open(filepath.Join(base, "registry.db")) |
| 38 | if err != nil { |
| 39 | t.Fatal(err) |
| 40 | } |
| 41 | defer reg.Close() |
| 42 | launches, err := hostbox.OpenLaunchJournal(filepath.Join(base, "launches.db")) |
| 43 | if err != nil { |
| 44 | t.Fatal(err) |
| 45 | } |
| 46 | defer launches.Close() |
| 47 | r := ClaimRuntimeResources{DataDir: data, OptDir: filepath.Join(base, "opt"), UnitsDir: filepath.Join(base, "units")} |
| 48 | for _, dir := range []string{r.OptDir, r.UnitsDir} { |
| 49 | if err := os.Mkdir(dir, 0o755); err != nil { |
| 50 | t.Fatal(err) |
| 51 | } |
| 52 | } |
| 53 | shared := claimGlobalSharedPaths(r) |
| 54 | shared["data"] = append(shared["data"], extra...) |
| 55 | c := hostbox.EmptyInstallationCensus{Inventory: inventory, Registry: reg, Launches: launches, Box: "box", Enrollment: "enrollment", DirectoryRevision: "1", |
| 56 | Roots: map[string]string{"data": r.DataDir, "opt": r.OptDir, "units": r.UnitsDir}, SharedPaths: shared} |
| 57 | obs := hostclaim.Observed{Host: host, Claim: &hostclaim.Claim{Holder: &hostclaim.Holder{Incarnation: "inc", Stamp: 1}}} |
| 58 | return c.CheckLocal(t.Context(), obs, hostbox.RecordSelection{Host: host, Incarnation: "inc"}) |
| 59 | } |
| 60 | |
| 61 | func heldLockInode(t *testing.T, path string) uint64 { |
| 62 | t.Helper() |
| 63 | info, err := os.Lstat(path) |
| 64 | if err != nil { |
| 65 | t.Fatal(err) |
| 66 | } |
| 67 | return info.Sys().(*syscall.Stat_t).Ino |
| 68 | } |
| 69 | |
| 70 | func TestHeldStageLockIsRemovedWhenTheStageFinishes(t *testing.T) { |
| 71 | t.Parallel() |
| 72 | data := t.TempDir() |
| 73 | path := filepath.Join(data, ".moves", heldStageLockFile(hsHost)) |
| 74 | if err := heldLockCensus(t, data, hsHost); err != nil { |
| 75 | t.Fatalf("census before any stage: %v", err) |
| 76 | } |
| 77 | for _, result := range []error{nil, errors.New("the stage failed")} { |
| 78 | err := withHeldStageLock(data, hsHost, func() error { |
| 79 | if _, err := os.Lstat(path); err != nil { |
| 80 | t.Fatalf("no lock file while the lock is held: %v", err) |
| 81 | } |
| 82 | return result |
| 83 | }) |
| 84 | if err != result { |
| 85 | t.Fatalf("stage: %v, want %v", err, result) |
| 86 | } |
| 87 | if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) { |
| 88 | t.Fatalf("after a stage that returned %v, %s remains: %v", result, path, err) |
| 89 | } |
| 90 | } |
| 91 | // The publication path: heldStageAuthority's then, inside the freeze. |
| 92 | s := release.NewDirStore(t.TempDir()) |
| 93 | hsClaim(t, s, movingOp()) |
| 94 | hsReport(t, s, hsOp, "v7") |
| 95 | auth := heldStageAuthority(testRegistry(t), s, hsBox, data) |
| 96 | published := false |
| 97 | if err := auth(hsRequest(), func() error { published = true; return nil }); err != nil || !published { |
| 98 | t.Fatalf("publication: %v (ran %v)", err, published) |
| 99 | } |
| 100 | if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) { |
| 101 | t.Fatalf("after a publication, %s remains: %v", path, err) |
| 102 | } |
| 103 | // A from-empty install of that host on this box passes main's census. |
| 104 | if err := heldLockCensus(t, data, hsHost); err != nil { |
| 105 | t.Fatalf("census after held stages of the host: %v", err) |
| 106 | } |
| 107 | } |
| 108 | |
| 109 | // A waiter that opened the file a holder then removes must not lock that |
| 110 | // file and run beside whoever holds the new one at the same name: it |
| 111 | // re-checks the name after its flock and retries. |
| 112 | func TestHeldStageLockWaiterOnARemovedFileRetries(t *testing.T) { |
| 113 | prevWait, prevHook := heldLockWait, lockNamedOpened |
| 114 | t.Cleanup(func() { heldLockWait, lockNamedOpened = prevWait, prevHook }) |
| 115 | heldLockWait = 10 * time.Second |
| 116 | data := t.TempDir() |
| 117 | path := filepath.Join(data, ".moves", heldStageLockFile(hsHost)) |
| 118 | var holders, overlaps atomic.Int32 |
| 119 | hold := func(entered chan<- struct{}, leave <-chan struct{}) func() error { |
| 120 | return func() error { |
| 121 | if holders.Add(1) != 1 { |
| 122 | overlaps.Add(1) |
| 123 | } |
| 124 | defer holders.Add(-1) |
| 125 | close(entered) |
| 126 | if leave != nil { |
| 127 | <-leave |
| 128 | } |
| 129 | return nil |
| 130 | } |
| 131 | } |
| 132 | |
| 133 | // The first holder holds the lock on the first file. |
| 134 | in1, out1, done1 := make(chan struct{}), make(chan struct{}), make(chan error, 1) |
| 135 | go func() { done1 <- withHeldStageLock(data, hsHost, hold(in1, out1)) }() |
| 136 | <-in1 |
| 137 | |
| 138 | // The waiter opens that file and is parked before its flock. |
| 139 | opened, resume := make(chan struct{}), make(chan struct{}) |
| 140 | var opens atomic.Int32 |
| 141 | lockNamedOpened = func(name string) { |
| 142 | if name == heldStageLockFile(hsHost) && opens.Add(1) == 1 { |
| 143 | close(opened) |
| 144 | <-resume |
| 145 | } |
| 146 | } |
| 147 | inW, doneW := make(chan struct{}), make(chan error, 1) |
| 148 | go func() { doneW <- withHeldStageLock(data, hsHost, hold(inW, nil)) }() |
| 149 | <-opened |
| 150 | |
| 151 | // The first holder finishes: it removes its file and releases. |
| 152 | close(out1) |
| 153 | if err := <-done1; err != nil { |
| 154 | t.Fatal(err) |
| 155 | } |
| 156 | if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) { |
| 157 | close(resume) |
| 158 | t.Fatalf("the first holder left %s: %v", path, err) |
| 159 | } |
| 160 | |
| 161 | // A second holder arrives and locks a new file at the same name. |
| 162 | in2, out2, done2 := make(chan struct{}), make(chan struct{}), make(chan error, 1) |
| 163 | go func() { done2 <- withHeldStageLock(data, hsHost, hold(in2, out2)) }() |
| 164 | <-in2 |
| 165 | |
| 166 | // The waiter resumes. Its fd names the removed file, which nobody |
| 167 | // holds: it must not run until the second holder is done. |
| 168 | close(resume) |
| 169 | select { |
| 170 | case <-inW: |
| 171 | close(out2) |
| 172 | t.Fatalf("the waiter ran while the second holder held the lock (overlaps %d)", overlaps.Load()) |
| 173 | case <-time.After(500 * time.Millisecond): |
| 174 | } |
| 175 | close(out2) |
| 176 | if err := <-done2; err != nil { |
| 177 | t.Fatal(err) |
| 178 | } |
| 179 | select { |
| 180 | case <-inW: |
| 181 | case <-time.After(5 * time.Second): |
| 182 | t.Fatal("the waiter never took the lock after the second holder released it") |
| 183 | } |
| 184 | if err := <-doneW; err != nil { |
| 185 | t.Fatal(err) |
| 186 | } |
| 187 | if n := overlaps.Load(); n != 0 { |
| 188 | t.Fatalf("%d holders ran beside another", n) |
| 189 | } |
| 190 | if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) { |
| 191 | t.Fatalf("after every holder finished, %s remains: %v", path, err) |
| 192 | } |
| 193 | } |
| 194 | |
| 195 | // A holder killed mid-stage never reaches the removal, so its file stays |
| 196 | // (the existing behaviour). Main's census then refuses an install of that |
| 197 | // host; region HA's fallback (CensusTree(".moves")) refuses that host only. |
| 198 | // The next stage of the host locks the file the crash left and removes it. |
| 199 | func TestHeldStageLockLeftByACrashIsReusedThenRemoved(t *testing.T) { |
| 200 | t.Parallel() |
| 201 | data := t.TempDir() |
| 202 | path := filepath.Join(data, ".moves", heldStageLockFile(hsHost)) |
| 203 | if err := os.Mkdir(filepath.Dir(path), 0o755); err != nil { |
| 204 | t.Fatal(err) |
| 205 | } |
| 206 | if err := os.WriteFile(path, nil, 0o644); err != nil { |
| 207 | t.Fatal(err) |
| 208 | } |
| 209 | left := heldLockInode(t, path) |
| 210 | if err := heldLockCensus(t, data, hsHost); !errors.Is(err, hostbox.ErrTypedInventory) { |
| 211 | t.Fatalf("main's census with a crashed stage's lock: %v, want ErrTypedInventory", err) |
| 212 | } |
| 213 | if err := heldLockCensus(t, data, hsHost, hostbox.CensusTree(".moves")); !errors.Is(err, hostbox.ErrTypedInventory) { |
| 214 | t.Fatalf("the fallback census for the lock's host: %v, want ErrTypedInventory", err) |
| 215 | } |
| 216 | if err := heldLockCensus(t, data, "other.example.com", hostbox.CensusTree(".moves")); err != nil { |
| 217 | t.Fatalf("the fallback census for another host: %v", err) |
| 218 | } |
| 219 | ran := false |
| 220 | if err := withHeldStageLock(data, hsHost, func() error { |
| 221 | ran = true |
| 222 | if got := heldLockInode(t, path); got != left { |
| 223 | t.Fatalf("the next stage locked a new file (inode %d), not the one the crash left (%d)", got, left) |
| 224 | } |
| 225 | return nil |
| 226 | }); err != nil || !ran { |
| 227 | t.Fatalf("the next stage: %v (ran %v)", err, ran) |
| 228 | } |
| 229 | if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) { |
| 230 | t.Fatalf("the next stage left %s: %v", path, err) |
| 231 | } |
| 232 | if err := heldLockCensus(t, data, hsHost); err != nil { |
| 233 | t.Fatalf("census after the next stage: %v", err) |
| 234 | } |
| 235 | } |
| 236 | |
| 237 | // A holder's removal can land inside a waiter's open: the name vanishes |
| 238 | // between the create and the reopen, or the file reopened has no link |
| 239 | // left. lockNamed opens again then, but keeps the refusal of anything |
| 240 | // else at the name. |
| 241 | func TestHeldStageLockOpenRetriesOnlyTheRemovalRace(t *testing.T) { |
| 242 | data := t.TempDir() |
| 243 | d, err := openMovesDir(data, true) |
| 244 | if err != nil { |
| 245 | t.Fatal(err) |
| 246 | } |
| 247 | defer d.Close() |
| 248 | at := func(name string) string { return filepath.Join(data, ".moves", name) } |
| 249 | if err := os.WriteFile(at("fresh.lock"), nil, 0o644); err != nil { |
| 250 | t.Fatal(err) |
| 251 | } |
| 252 | if err := os.WriteFile(at("linked.lock"), nil, 0o644); err != nil { |
| 253 | t.Fatal(err) |
| 254 | } |
| 255 | if err := os.Link(at("linked.lock"), at("second.lock")); err != nil { |
| 256 | t.Fatal(err) |
| 257 | } |
| 258 | if err := os.Symlink(t.TempDir(), at("planted.lock")); err != nil { |
| 259 | t.Fatal(err) |
| 260 | } |
| 261 | if err := syscall.Mkfifo(at("fifo.lock"), 0o644); err != nil { |
| 262 | t.Fatal(err) |
| 263 | } |
| 264 | unlinked := &os.PathError{Op: "open", Path: "x", Err: errors.New("not a regular file with one link")} |
| 265 | for _, c := range []struct { |
| 266 | name string |
| 267 | err error |
| 268 | want bool |
| 269 | }{ |
| 270 | {"gone.lock", &os.PathError{Op: "open", Path: "x", Err: syscall.ENOENT}, true}, |
| 271 | {"gone.lock", unlinked, true}, |
| 272 | {"fresh.lock", unlinked, true}, |
| 273 | {"fresh.lock", &os.PathError{Op: "open", Path: "x", Err: syscall.EACCES}, false}, |
| 274 | {"linked.lock", unlinked, false}, |
| 275 | {"planted.lock", unlinked, false}, |
| 276 | {"fifo.lock", unlinked, false}, |
| 277 | } { |
| 278 | if got := d.removedDuringOpen(c.name, c.err); got != c.want { |
| 279 | t.Errorf("%s after %v: retry %v, want %v", c.name, c.err, got, c.want) |
| 280 | } |
| 281 | } |
| 282 | // A .moves removed under an open movesDir: nothing can be created in |
| 283 | // it, so the open's error stands at once. |
| 284 | gone := t.TempDir() |
| 285 | g, err := openMovesDir(gone, true) |
| 286 | if err != nil { |
| 287 | t.Fatal(err) |
| 288 | } |
| 289 | defer g.Close() |
| 290 | if err := os.Remove(filepath.Join(gone, ".moves")); err != nil { |
| 291 | t.Fatal(err) |
| 292 | } |
| 293 | if g.removedDuringOpen("gone.lock", &os.PathError{Op: "open", Path: "x", Err: syscall.ENOENT}) { |
| 294 | t.Error("a removed .moves was retried") |
| 295 | } |
| 296 | } |
| 297 | |
| 298 | // Fix round 1: the re-check's retries are bounded by the one wait. With |
| 299 | // every file the waiter opens replaced at the name before its flock, each |
| 300 | // re-check finds another file there; the call still ends with the busy |
| 301 | // error once the wait has passed, and never runs fn. Publication waits |
| 302 | // inside the registry's freeze, so the wait must hold. |
| 303 | func TestHeldStageLockRetriesEndWithTheWait(t *testing.T) { |
| 304 | prevWait, prevHook := heldLockWait, lockNamedOpened |
| 305 | const wait = 200 * time.Millisecond |
| 306 | heldLockWait = wait |
| 307 | data := t.TempDir() |
| 308 | path := filepath.Join(data, ".moves", heldStageLockFile(hsHost)) |
| 309 | var stop atomic.Bool |
| 310 | var replaced atomic.Int32 |
| 311 | lockNamedOpened = func(name string) { |
| 312 | if name != heldStageLockFile(hsHost) || stop.Load() { |
| 313 | return |
| 314 | } |
| 315 | if err := os.Remove(path); err != nil { |
| 316 | return |
| 317 | } |
| 318 | if err := os.WriteFile(path, nil, 0o644); err == nil { |
| 319 | replaced.Add(1) |
| 320 | } |
| 321 | } |
| 322 | var ran atomic.Bool |
| 323 | done := make(chan error, 1) |
| 324 | start := time.Now() |
| 325 | go func() { done <- withHeldStageLock(data, hsHost, func() error { ran.Store(true); return nil }) }() |
| 326 | t.Cleanup(func() { |
| 327 | // A waiter still retrying is let through before the seam goes. |
| 328 | stop.Store(true) |
| 329 | select { |
| 330 | case <-done: |
| 331 | case <-time.After(10 * time.Second): |
| 332 | } |
| 333 | heldLockWait, lockNamedOpened = prevWait, prevHook |
| 334 | }) |
| 335 | var err error |
| 336 | select { |
| 337 | case err = <-done: |
| 338 | done <- err |
| 339 | case <-time.After(5 * time.Second): |
| 340 | t.Fatalf("still retrying %v after a %v wait (%d files replaced, ran %v)", time.Since(start), wait, replaced.Load(), ran.Load()) |
| 341 | } |
| 342 | elapsed := time.Since(start) |
| 343 | if !errors.Is(err, errHeldLockBusy) { |
| 344 | t.Fatalf("with every opened file replaced: %v, want errHeldLockBusy", err) |
| 345 | } |
| 346 | if ran.Load() { |
| 347 | t.Fatal("fn ran without the lock") |
| 348 | } |
| 349 | if elapsed > wait+2*time.Second { |
| 350 | t.Fatalf("gave up after %v, want about %v", elapsed, wait) |
| 351 | } |
| 352 | if replaced.Load() < 2 { |
| 353 | t.Fatalf("only %d files were replaced; the re-check was not exercised", replaced.Load()) |
| 354 | } |
| 355 | // Fix round 2: replacement retries pause (lockNamedRetryPause, capped |
| 356 | // at what is left of the wait), so they cannot spin. A pause is a floor |
| 357 | // on each retry's time, so this bound holds however loaded the box. |
| 358 | if most := int32(wait/lockNamedRetryPause) + 2; replaced.Load() > most { |
| 359 | t.Fatalf("%d replacement retries in a %v wait, want at most %d", replaced.Load(), wait, most) |
| 360 | } |
| 361 | } |
| 362 | |
| 363 | // Fix round 2: an open that meets a holder's removal after the wait has |
| 364 | // passed ends in the caller's busy error, like every other way the wait |
| 365 | // runs out, never the open's own error. Every open here meets that race: |
| 366 | // it fails as an unlinked file does and the name is gone. |
| 367 | func TestHeldStageLockOpenRaceAfterTheWaitIsBusy(t *testing.T) { |
| 368 | prevWait, prevOpen := heldLockWait, lockNamedOpenAt |
| 369 | t.Cleanup(func() { heldLockWait, lockNamedOpenAt = prevWait, prevOpen }) |
| 370 | const wait = 150 * time.Millisecond |
| 371 | heldLockWait = wait |
| 372 | var opens atomic.Int32 |
| 373 | lockNamedOpenAt = func(dir int, name string, flags int, mode os.FileMode) (*os.File, error) { |
| 374 | opens.Add(1) |
| 375 | return nil, &os.PathError{Op: "open", Path: name, Err: errors.New("not a regular file with one link")} |
| 376 | } |
| 377 | ran := false |
| 378 | start := time.Now() |
| 379 | err := withHeldStageLock(t.TempDir(), hsHost, func() error { ran = true; return nil }) |
| 380 | elapsed := time.Since(start) |
| 381 | if !errors.Is(err, errHeldLockBusy) { |
| 382 | t.Fatalf("every open raced a removal until past the wait: %v, want errHeldLockBusy", err) |
| 383 | } |
| 384 | if ran { |
| 385 | t.Fatal("fn ran without the lock") |
| 386 | } |
| 387 | if elapsed < wait || elapsed > wait+time.Second { |
| 388 | t.Fatalf("gave up after %v, want about %v", elapsed, wait) |
| 389 | } |
| 390 | if opens.Load() < 2 { |
| 391 | t.Fatalf("only %d opens; the race was not retried", opens.Load()) |
| 392 | } |
| 393 | } |
| 394 | |
amadan