| 1 | package activator |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "os" |
| 7 | "path/filepath" |
| 8 | "slices" |
| 9 | "strings" |
| 10 | "sync" |
| 11 | "testing" |
| 12 | "time" |
| 13 | |
| 14 | "github.com/carlosframework/platform/internal/cgroup" |
| 15 | "github.com/carlosframework/platform/internal/provision" |
| 16 | "github.com/carlosframework/platform/internal/registry" |
| 17 | ) |
| 18 | |
| 19 | // sidecarStartDoneOf is the host's supervised-start join channel, nil when |
| 20 | // none is in flight. |
| 21 | func (a *Service) sidecarStartDoneOf(host string) chan struct{} { |
| 22 | a.mu.Lock() |
| 23 | defer a.mu.Unlock() |
| 24 | if e := a.entries[host]; e != nil { |
| 25 | return e.sidecarStartDone |
| 26 | } |
| 27 | return nil |
| 28 | } |
| 29 | |
| 30 | // sidecarProc is the i'th sidecar the fake backend started. |
| 31 | func (g *fakeSwapBackend) sidecarProc(i int) *fakeProc { |
| 32 | g.mu.Lock() |
| 33 | defer g.mu.Unlock() |
| 34 | if i >= len(g.sidecars) { |
| 35 | return nil |
| 36 | } |
| 37 | return g.sidecars[i] |
| 38 | } |
| 39 | |
| 40 | // sidecarStoppingIn reports whether the i'th sidecar's Stop has been |
| 41 | // entered: with a stop gate on it, that Stop is in flight and blocked. |
| 42 | func (g *fakeSwapBackend) sidecarStoppingIn(i int) bool { |
| 43 | p := g.sidecarProc(i) |
| 44 | return p != nil && p.stopEntered() > 0 |
| 45 | } |
| 46 | |
| 47 | // generationSwitched is true once the entry's current generation is not the |
| 48 | // fixture's incumbent. |
| 49 | func (f *swapFixture) generationSwitched() bool { |
| 50 | g, _ := f.a.genStateForTest(f.host) |
| 51 | return g.N != f.incumbent.N |
| 52 | } |
| 53 | |
| 54 | // Flag off, on a backend that cannot pin an executable (the fixture never |
| 55 | // opts into the identity capability, as a unit row's backend has none): a |
| 56 | // generation with no staged file starts its legacy sidecar (the wake tail |
| 57 | // and the supervisor) through StartSidecar and the live link, as before. |
| 58 | // A generation that names its file runs it whatever the flag (ruling |
| 59 | // R-EXEC-SIDECAR; TestLegacySidecarOwnershipCheckRunsWithSwapOff). |
| 60 | func TestLegacySidecarUsesTheLiveLinkWithSwapOff(t *testing.T) { |
| 61 | t.Parallel() |
| 62 | f := newSwapFixture(t) |
| 63 | f.a.gens = false |
| 64 | f.b.sidecarStaged = true |
| 65 | f.a.Bounce(f.host) |
| 66 | waitFor(t, "the stop", func() bool { return f.a.entryCount() == 0 }) |
| 67 | arrangeAwake(t, f.a, testRoute(f.host)) |
| 68 | if got := f.b.sidecarStartCount(); got != 1 { |
| 69 | t.Fatalf("StartSidecar calls = %d after the wake, want 1", got) |
| 70 | } |
| 71 | f.b.lastSidecar().kill() |
| 72 | f.clk.advance(3 * time.Second) |
| 73 | f.a.sweepOnce() |
| 74 | waitFor(t, "the supervised restart", func() bool { return f.a.sidecarSettled(f.host) && f.b.sidecarStartCount() == 2 }) |
| 75 | if paths := f.b.sidecarGenPaths(); len(paths) != 0 { |
| 76 | t.Fatalf("StartSidecarGen ran with swaps off: %v", paths) |
| 77 | } |
| 78 | } |
| 79 | |
| 80 | // Flag off, ownership: startSidecar asks ownedLocked whatever the flag (an |
| 81 | // accepted dormancy exception, recorded in task-7-adapt.md). Exactly what |
| 82 | // changes against today, called directly so no earlier sweep check masks it: |
| 83 | // a host that does not own itself (fenced, shutting down, lease lost, no |
| 84 | // longer the host's entry) gets NO co-spawned sidecar, where today's |
| 85 | // StartSidecar would have spawned one. A host that owns itself gets its |
| 86 | // sidecar on the generation's own staged file, with the flag off too |
| 87 | // (ruling R-EXEC-SIDECAR, settings Task 5c), never the live link. |
| 88 | func TestLegacySidecarOwnershipCheckRunsWithSwapOff(t *testing.T) { |
| 89 | f := newSwapFixture(t) |
| 90 | f.a.gens = false |
| 91 | f.b.sidecarStaged = true |
| 92 | f.a.mu.Lock() |
| 93 | e := f.a.entries[f.host] |
| 94 | rt := e.route |
| 95 | f.a.mu.Unlock() |
| 96 | rep := f.b.lastRep() |
| 97 | |
| 98 | // Owned: the generation's recorded file, by path. |
| 99 | sc, err := f.a.startSidecar(context.Background(), e, rt, f.incumbent, false) |
| 100 | if err != nil || sc == nil { |
| 101 | t.Fatalf("an owned host's sidecar = %v, %v; want the generation's spawn", sc, err) |
| 102 | } |
| 103 | if paths := f.b.sidecarGenPaths(); len(paths) != 1 || paths[0] != f.incumbent.StagedPath { |
| 104 | t.Fatalf("StartSidecarGen paths = %v, want the generation's %q", paths, f.incumbent.StagedPath) |
| 105 | } |
| 106 | if got := f.b.sidecarStartCount(); got != 0 { |
| 107 | t.Fatalf("StartSidecar (live link) calls = %d, want none", got) |
| 108 | } |
| 109 | |
| 110 | refused := func(name string, arm, disarm func(), want error) { |
| 111 | t.Helper() |
| 112 | arm() |
| 113 | sc, err := f.a.startSidecar(context.Background(), e, rt, f.incumbent, false) |
| 114 | disarm() |
| 115 | if !errors.Is(err, want) || sc != nil { |
| 116 | t.Fatalf("%s: startSidecar = %v, %v; want %v and no process", name, sc, err, want) |
| 117 | } |
| 118 | if got := len(f.b.sidecarGenPaths()); got != 1 || f.b.sidecarStartCount() != 0 { |
| 119 | t.Fatalf("%s: sidecar spawns = %d (live link %d), want 1 (none beside a host that does not own itself)", name, got, f.b.sidecarStartCount()) |
| 120 | } |
| 121 | } |
| 122 | refused("fenced", |
| 123 | func() { f.a.mu.Lock(); f.a.fences = map[string]uint64{f.host: 1}; f.a.mu.Unlock() }, |
| 124 | func() { f.a.mu.Lock(); f.a.fences = nil; f.a.mu.Unlock() }, errSwapFenced) |
| 125 | refused("shutting down", |
| 126 | func() { f.a.mu.Lock(); f.a.shuttingDown = true; f.a.mu.Unlock() }, |
| 127 | func() { f.a.mu.Lock(); f.a.shuttingDown = false; f.a.mu.Unlock() }, errSwapShutdown) |
| 128 | refused("lease lost", |
| 129 | func() { rep.setErr(errors.New("lease lost")) }, |
| 130 | func() { rep.setErr(nil) }, errSwapLeaseLost) |
| 131 | refused("not the host's entry", |
| 132 | func() { f.a.mu.Lock(); delete(f.a.entries, f.host); f.a.mu.Unlock() }, |
| 133 | func() { f.a.mu.Lock(); f.a.entries[f.host] = e; f.a.mu.Unlock() }, errSwapNotAwake) |
| 134 | |
| 135 | // And once owned again, the same call spawns as before. |
| 136 | if _, err := f.a.startSidecar(context.Background(), e, rt, f.incumbent, false); err != nil { |
| 137 | t.Fatalf("owned again: %v", err) |
| 138 | } |
| 139 | if got := len(f.b.sidecarGenPaths()); got != 2 { |
| 140 | t.Fatalf("StartSidecarGen calls = %d, want 2", got) |
| 141 | } |
| 142 | } |
| 143 | |
| 144 | // Flag on: late attachment, the supervisor, a swap's replacement and the |
| 145 | // wake tail all exec the CURRENT generation's captured staged path (spec |
| 146 | // §7.5). The live link names v3 throughout and is never used. |
| 147 | func TestLegacySidecarRunsTheServingGeneration(t *testing.T) { |
| 148 | t.Parallel() |
| 149 | t.Run("late attachment and the supervisor", func(t *testing.T) { |
| 150 | f := newSwapFixture(t) |
| 151 | f.b.execVerFn = func() string { return "v3" } |
| 152 | f.b.sidecarStaged = true // staged after the wake: the sweep attaches it |
| 153 | f.a.sweepOnce() |
| 154 | waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 }) |
| 155 | f.b.lastSidecar().kill() |
| 156 | f.clk.advance(3 * time.Second) |
| 157 | f.a.sweepOnce() |
| 158 | waitFor(t, "the supervised restart", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 2 }) |
| 159 | for _, p := range f.b.sidecarGenPaths() { |
| 160 | if p != f.incumbent.StagedPath { |
| 161 | t.Fatalf("a sidecar ran %q, want the serving generation's %q", p, f.incumbent.StagedPath) |
| 162 | } |
| 163 | } |
| 164 | if got := f.b.sidecarStartCount(); got != 0 { |
| 165 | t.Fatalf("StartSidecar (live link) ran %d time(s) with swaps on", got) |
| 166 | } |
| 167 | }) |
| 168 | t.Run("a swap replaces the sidecar stop-then-start", func(t *testing.T) { |
| 169 | f := newSwapFixture(t) |
| 170 | f.b.sidecarStaged = true |
| 171 | f.a.sweepOnce() |
| 172 | waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 }) |
| 173 | old := f.b.lastSidecar() |
| 174 | if res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest); res.Outcome != SwapDone { |
| 175 | t.Fatalf("Swap = %+v", res) |
| 176 | } |
| 177 | paths := f.b.sidecarGenPaths() |
| 178 | if len(paths) != 2 || paths[1] != f.staged { |
| 179 | t.Fatalf("sidecar spawns %v: the replacement must run the new generation's %q", paths, f.staged) |
| 180 | } |
| 181 | if !old.isStopped() { |
| 182 | t.Fatal("the old sidecar outlived the replacement") |
| 183 | } |
| 184 | ev := f.b.events() |
| 185 | if i, j := slices.Index(ev, "stop:sidecar"), slices.Index(ev, "startSidecarGen:"+f.staged); i < 0 || j < 0 || i > j { |
| 186 | t.Fatalf("order %v: stop the old sidecar, then start the new one", ev) |
| 187 | } |
| 188 | }) |
| 189 | t.Run("the wake tail of a hold", func(t *testing.T) { |
| 190 | f := newSwapFixture(t) |
| 191 | f.b.sidecarStaged = true |
| 192 | if res := f.a.Swap(context.Background(), f.attempt(ModeHold), f.newest); res.Outcome != SwapDone { |
| 193 | t.Fatalf("Swap = %+v", res) |
| 194 | } |
| 195 | if paths := f.b.sidecarGenPaths(); len(paths) == 0 || paths[len(paths)-1] != f.staged { |
| 196 | t.Fatalf("the hold's wake tail started sidecars %v, want %q last", paths, f.staged) |
| 197 | } |
| 198 | }) |
| 199 | } |
| 200 | |
| 201 | // The swap's replacement holds the supervisor's interlock, so a supervised |
| 202 | // restart that falls due meanwhile cannot start a second sidecar (review |
| 203 | // focus 4). |
| 204 | func TestSwapSidecarReplacementHoldsTheSupervisorOff(t *testing.T) { |
| 205 | t.Parallel() |
| 206 | f := newSwapFixture(t) |
| 207 | f.b.sidecarStaged = true |
| 208 | f.a.sweepOnce() |
| 209 | waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 }) |
| 210 | gate := make(chan struct{}) |
| 211 | f.b.gmu.Lock() |
| 212 | f.b.sidecarGenGate = gate |
| 213 | f.b.gmu.Unlock() |
| 214 | results := make(chan SwapResult, 1) |
| 215 | go func() { results <- f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) }() |
| 216 | waitFor(t, "the replacement's start", func() bool { return len(f.b.sidecarGenPaths()) == 2 }) |
| 217 | held := f.a.sidecarStartDoneOf(f.host) |
| 218 | if held == nil { |
| 219 | t.Fatal("the replacement holds no interlock") |
| 220 | } |
| 221 | f.clk.advance(time.Minute) |
| 222 | f.a.sweepOnce() // the old sidecar is stopped (exited): the supervisor would restart it |
| 223 | // The supervisor claims sidecarStarting under the sweep's own lock, so a |
| 224 | // claim would have replaced the join channel before sweepOnce returned. |
| 225 | if now := f.a.sidecarStartDoneOf(f.host); now != held { |
| 226 | t.Fatal("a supervised restart claimed the interlock during the replacement") |
| 227 | } |
| 228 | if n := len(f.b.sidecarGenPaths()); n != 2 { |
| 229 | t.Fatalf("sidecar spawns = %d during the replacement, want 2", n) |
| 230 | } |
| 231 | close(gate) |
| 232 | if res := <-results; res.Outcome != SwapDone { |
| 233 | t.Fatalf("Swap = %+v", res) |
| 234 | } |
| 235 | } |
| 236 | |
| 237 | // A supervised restart already in flight when the swap switches captured the |
| 238 | // OLD generation. The swap's replacement joins it and then replaces what it |
| 239 | // installed, so the new server never runs beside the old build's sidecar |
| 240 | // (spec §7.5). Returning early because sidecarStarting is set (the old |
| 241 | // replaceSidecar) leaves the supervisor's old-build sidecar installed with |
| 242 | // no replacement pending: the third spawn never comes. |
| 243 | func TestSwapSidecarReplacementJoinsAnInFlightSupervisor(t *testing.T) { |
| 244 | t.Parallel() |
| 245 | f := newSwapFixture(t) |
| 246 | f.b.sidecarStaged = true |
| 247 | f.a.sweepOnce() |
| 248 | waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 }) |
| 249 | gate := make(chan struct{}) |
| 250 | f.b.gmu.Lock() |
| 251 | f.b.sidecarGenGate = gate |
| 252 | f.b.gmu.Unlock() |
| 253 | f.b.lastSidecar().kill() |
| 254 | f.clk.advance(3 * time.Second) |
| 255 | f.a.sweepOnce() // the supervisor starts a sidecar for the incumbent and blocks |
| 256 | waitFor(t, "the supervisor's start", func() bool { return len(f.b.sidecarGenPaths()) == 2 }) |
| 257 | results := make(chan SwapResult, 1) |
| 258 | go func() { results <- f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) }() |
| 259 | waitFor(t, "the switch", f.generationSwitched) |
| 260 | close(gate) |
| 261 | if res := <-results; res.Outcome != SwapDone { |
| 262 | t.Fatalf("Swap = %+v", res) |
| 263 | } |
| 264 | waitFor(t, "the replacement onto the new generation", func() bool { |
| 265 | p := f.b.sidecarGenPaths() |
| 266 | return len(p) == 3 && f.a.sidecarSettled(f.host) |
| 267 | }) |
| 268 | paths := f.b.sidecarGenPaths() |
| 269 | if paths[1] != f.incumbent.StagedPath || paths[2] != f.staged { |
| 270 | t.Fatalf("sidecar spawns %v: want the supervisor's on %q, then the replacement on %q", paths, f.incumbent.StagedPath, f.staged) |
| 271 | } |
| 272 | // The supervisor's sidecar is stopped by the replacement's |
| 273 | // stop-then-start. |
| 274 | f.b.mu.Lock() |
| 275 | supervised, current := f.b.sidecars[1], f.b.sidecars[len(f.b.sidecars)-1] |
| 276 | f.b.mu.Unlock() |
| 277 | if !supervised.isStopped() { |
| 278 | t.Fatal("the supervisor's old-build sidecar outlived the swap") |
| 279 | } |
| 280 | f.a.mu.Lock() |
| 281 | installed := f.a.entries[f.host].sidecar |
| 282 | f.a.mu.Unlock() |
| 283 | if installed != Proc(current) { |
| 284 | t.Fatal("the installed sidecar is not the replacement") |
| 285 | } |
| 286 | } |
| 287 | |
| 288 | // The supervisor's start for the retired generation is joined until that |
| 289 | // process is proven gone (spec §7.4, §7.5): the replacement never starts |
| 290 | // while it still runs. Discarding it asynchronously let the supervisor |
| 291 | // close its join at once, so the replacement started beside a sidecar |
| 292 | // still draining on the old build. |
| 293 | func TestSwapSidecarNeverRunsBesideTheRetiredGenerationsSidecar(t *testing.T) { |
| 294 | t.Parallel() |
| 295 | f := newSwapFixture(t) |
| 296 | f.b.sidecarStaged = true |
| 297 | f.a.sweepOnce() |
| 298 | waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 }) |
| 299 | startGate, stopGate := make(chan struct{}), make(chan struct{}) |
| 300 | f.b.gmu.Lock() |
| 301 | f.b.sidecarGenGate, f.b.sidecarStopGate = startGate, stopGate |
| 302 | f.b.gmu.Unlock() |
| 303 | f.b.lastSidecar().kill() |
| 304 | f.clk.advance(3 * time.Second) |
| 305 | f.a.sweepOnce() // the supervisor starts the incumbent's sidecar and blocks |
| 306 | waitFor(t, "the supervisor's start", func() bool { return len(f.b.sidecarGenPaths()) == 2 }) |
| 307 | results := make(chan SwapResult, 1) |
| 308 | go func() { results <- f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) }() |
| 309 | waitFor(t, "the switch", f.generationSwitched) |
| 310 | close(startGate) // the supervisor's sidecar starts; its Stop blocks on stopGate |
| 311 | // The retired generation's sidecar is being stopped, and the replacement |
| 312 | // is sequenced behind that Stop: nothing can have started beside it. |
| 313 | waitFor(t, "the retired sidecar's stop to be in flight", func() bool { return f.b.sidecarStoppingIn(1) }) |
| 314 | if n := len(f.b.sidecarGenPaths()); n != 2 { |
| 315 | t.Fatalf("%d sidecar starts: the replacement started while the retired generation's sidecar still ran", n) |
| 316 | } |
| 317 | select { |
| 318 | case res := <-results: |
| 319 | t.Fatalf("the swap returned (%+v) before the retired sidecar was proven gone", res) |
| 320 | default: |
| 321 | } |
| 322 | close(stopGate) |
| 323 | if res := <-results; res.Outcome != SwapDone { |
| 324 | t.Fatalf("Swap = %+v", res) |
| 325 | } |
| 326 | waitFor(t, "the replacement", func() bool { return len(f.b.sidecarGenPaths()) == 3 }) |
| 327 | if p := f.b.sidecarGenPaths(); p[2] != f.staged { |
| 328 | t.Fatalf("sidecar starts %v: want the new generation's last", p) |
| 329 | } |
| 330 | } |
| 331 | |
| 332 | // A terminal stop that begins while a supervised sidecar start is in flight |
| 333 | // joins that start, and the sidecar it produced, before the lease is |
| 334 | // released (spec §7.4). Discarding it asynchronously let the supervisor |
| 335 | // close its join at once, so the stop released the lease while the new |
| 336 | // sidecar was still running. |
| 337 | func TestSwapStopJoinsTheSidecarAStartLeftBehind(t *testing.T) { |
| 338 | t.Parallel() |
| 339 | f := newSwapFixture(t) |
| 340 | f.b.sidecarStaged = true |
| 341 | f.a.sweepOnce() |
| 342 | waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 }) |
| 343 | startGate, stopGate := make(chan struct{}), make(chan struct{}) |
| 344 | f.b.gmu.Lock() |
| 345 | f.b.sidecarGenGate, f.b.sidecarStopGate = startGate, stopGate |
| 346 | f.b.gmu.Unlock() |
| 347 | f.b.lastSidecar().kill() |
| 348 | f.clk.advance(3 * time.Second) |
| 349 | f.a.sweepOnce() // the supervisor's start blocks |
| 350 | waitFor(t, "the supervisor's start", func() bool { return len(f.b.sidecarGenPaths()) == 2 }) |
| 351 | rep := f.b.lastRep() |
| 352 | joining := make(chan struct{}) |
| 353 | var joinOnce sync.Once |
| 354 | f.a.mu.Lock() |
| 355 | f.a.joinWaiting = func(_, what string) { |
| 356 | if what == "sidecar start" { |
| 357 | joinOnce.Do(func() { close(joining) }) |
| 358 | } |
| 359 | } |
| 360 | f.a.mu.Unlock() |
| 361 | f.a.Bounce(f.host) |
| 362 | // The stop is joined on the supervisor's start: it goes no further (no |
| 363 | // instance stop, so no lease release) while the start is in flight. Wait |
| 364 | // for the stop to reach that join (the join seam's positive signal), or, |
| 365 | // if something skipped it, for the instance's stop. |
| 366 | waitFor(t, "the stop at its sidecar-start join", func() bool { |
| 367 | select { |
| 368 | case <-joining: |
| 369 | return true |
| 370 | default: |
| 371 | } |
| 372 | return slices.Contains(f.b.events(), "stop:instance") |
| 373 | }) |
| 374 | if slices.Contains(f.b.events(), "stop:instance") || rep.isClosed() { |
| 375 | t.Fatal("the stop went past its join on the in-flight sidecar start") |
| 376 | } |
| 377 | close(startGate) // the start lands on a stopping entry; its sidecar's Stop blocks |
| 378 | waitFor(t, "the left-behind sidecar's stop to be in flight", func() bool { return f.b.sidecarStoppingIn(1) }) |
| 379 | if rep.isClosed() { |
| 380 | t.Fatal("the lease was released while the sidecar a start left behind still ran") |
| 381 | } |
| 382 | close(stopGate) |
| 383 | waitFor(t, "the lease release", rep.isClosed) |
| 384 | if started := f.b.sidecarProc(1); !started.isStopped() { |
| 385 | t.Fatal("the sidecar the start left behind was never stopped") |
| 386 | } |
| 387 | } |
| 388 | |
| 389 | // Spec §7.4: a discarded sidecar whose stop is unproven is never forgotten. |
| 390 | // It stays tracked, the host takes no new wake beside it, and the sweep |
| 391 | // stops it again until it exits. Logging it and moving on let the next wake |
| 392 | // run a second sidecar beside a live one. |
| 393 | func TestAStraySidecarIsNeverForgotten(t *testing.T) { |
| 394 | t.Parallel() |
| 395 | f := newSwapFixture(t) |
| 396 | f.a.Bounce(f.host) |
| 397 | waitFor(t, "the instance asleep", func() bool { return f.a.entryCount() == 0 }) |
| 398 | stray := &fakeProc{label: "sidecar", rec: f.b.fakeBackend, stuck: true} |
| 399 | f.a.discardSidecar(testRoute(f.host), stray) |
| 400 | if got := f.a.strayCount(f.host); got != 1 { |
| 401 | t.Fatalf("tracked strays = %d, want 1", got) |
| 402 | } |
| 403 | if _, err := f.a.beginRequest(context.Background(), testRoute(f.host)); err == nil || !strings.Contains(err.Error(), errWakeUnproven.Error()) { |
| 404 | t.Fatalf("a wake beside an unproven sidecar = %v, want refused", err) |
| 405 | } |
| 406 | stray.exit() |
| 407 | f.a.sweepOnce() // the reaper re-proves it off the lock |
| 408 | waitFor(t, "the stray proven gone", func() bool { return f.a.strayCount(f.host) == 0 }) |
| 409 | release, err := f.a.beginRequest(context.Background(), testRoute(f.host)) |
| 410 | if err != nil { |
| 411 | t.Fatalf("the wake once the stray exited: %v", err) |
| 412 | } |
| 413 | release() |
| 414 | } |
| 415 | |
| 416 | func (a *Service) strayCount(host string) int { |
| 417 | a.mu.Lock() |
| 418 | defer a.mu.Unlock() |
| 419 | return len(a.straySidecars[host]) |
| 420 | } |
| 421 | |
| 422 | // strayReaping is true while any stray of host has a re-proof in flight. |
| 423 | func (a *Service) strayReaping(host string) bool { |
| 424 | a.mu.Lock() |
| 425 | defer a.mu.Unlock() |
| 426 | for _, s := range a.straySidecars[host] { |
| 427 | if s.reaping { |
| 428 | return true |
| 429 | } |
| 430 | } |
| 431 | return false |
| 432 | } |
| 433 | |
| 434 | func (a *Service) stopPending(host string) bool { |
| 435 | a.mu.Lock() |
| 436 | defer a.mu.Unlock() |
| 437 | e := a.entries[host] |
| 438 | return e != nil && e.pendingStop |
| 439 | } |
| 440 | |
| 441 | // Spec §7.4, §7.5: a legacy sidecar is gone only when its cgroup is empty. |
| 442 | // childProc.Stop signals only the leader, so a process the sidecar forked |
| 443 | // outlives its exit in role-sidecar. Here the leader exits but a descendant |
| 444 | // stays (KillSidecar cannot empty the cgroup): the replacement does not |
| 445 | // start beside it, and a terminal stop keeps the lease until it is proven |
| 446 | // gone. Accepting Exited() started the new sidecar beside the descendant |
| 447 | // and released the lease under it. |
| 448 | func TestASidecarsDescendantBlocksItsReplacementAndTheLease(t *testing.T) { |
| 449 | t.Parallel() |
| 450 | f := newSwapFixture(t) |
| 451 | f.b.sidecarStaged = true |
| 452 | f.a.sweepOnce() |
| 453 | waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 }) |
| 454 | f.b.setSidecarDescendant(true) |
| 455 | if res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest); res.Outcome != SwapDone { |
| 456 | t.Fatalf("Swap = %+v", res) |
| 457 | } |
| 458 | if got := f.a.strayCount(f.host); got != 1 { |
| 459 | t.Fatalf("tracked strays = %d after the replacement, want 1 (the old sidecar's descendant)", got) |
| 460 | } |
| 461 | // A supervisor pass that falls due must not start one either. It runs |
| 462 | // startSidecar, which refuses while the stray is tracked. |
| 463 | f.clk.advance(time.Minute) |
| 464 | f.a.sweepOnce() |
| 465 | waitFor(t, "the supervisor's refused start", func() bool { return f.a.sidecarSettled(f.host) }) |
| 466 | if n := len(f.b.sidecarGenPaths()); n != 1 { |
| 467 | t.Fatalf("%d sidecar starts: a new sidecar started beside the old one's descendant", n) |
| 468 | } |
| 469 | rep := f.b.lastRep() |
| 470 | f.a.Bounce(f.host) |
| 471 | waitFor(t, "the stop to wait on the sidecar's cgroup", func() bool { return f.a.stopPending(f.host) }) |
| 472 | // Each sweep re-proves the stray; the descendant is still there. |
| 473 | for range 2 { |
| 474 | before := f.b.sidecarKillCount() |
| 475 | f.a.sweepOnce() |
| 476 | waitFor(t, "a proof attempt", func() bool { return f.b.sidecarKillCount() > before }) |
| 477 | waitFor(t, "the re-proof to finish", func() bool { return !f.a.strayReaping(f.host) }) |
| 478 | if rep.isClosed() { |
| 479 | t.Fatal("the lease was released while the sidecar's descendant still ran") |
| 480 | } |
| 481 | } |
| 482 | if got := f.a.strayCount(f.host); got != 1 { |
| 483 | t.Fatalf("tracked strays = %d with the descendant alive, want 1", got) |
| 484 | } |
| 485 | f.b.setSidecarDescendant(false) |
| 486 | waitFor(t, "the lease release once the cgroup is empty", func() bool { |
| 487 | f.a.sweepOnce() |
| 488 | return rep.isClosed() |
| 489 | }) |
| 490 | } |
| 491 | |
| 492 | // A terminal stop's co-stop is held to the same proof: the sidecar's leader |
| 493 | // exits but a process it forked is left in the cgroup, so the stop keeps the |
| 494 | // lease (releasable) until the sweep proves the cgroup empty. |
| 495 | func TestATerminalStopKeepsTheLeaseOverASidecarsDescendant(t *testing.T) { |
| 496 | t.Parallel() |
| 497 | f := newSwapFixture(t) |
| 498 | f.b.sidecarStaged = true |
| 499 | f.a.sweepOnce() |
| 500 | waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 }) |
| 501 | f.b.setSidecarDescendant(true) |
| 502 | rep := f.b.lastRep() |
| 503 | f.a.Bounce(f.host) |
| 504 | waitFor(t, "the stop to wait on the sidecar's cgroup", func() bool { return f.a.stopPending(f.host) }) |
| 505 | if got := f.a.strayCount(f.host); got != 1 { |
| 506 | t.Fatalf("tracked strays = %d after the co-stop, want 1", got) |
| 507 | } |
| 508 | before := f.b.sidecarKillCount() |
| 509 | f.a.sweepOnce() |
| 510 | waitFor(t, "a proof attempt", func() bool { return f.b.sidecarKillCount() > before }) |
| 511 | waitFor(t, "the re-proof to finish", func() bool { return !f.a.strayReaping(f.host) }) |
| 512 | if rep.isClosed() { |
| 513 | t.Fatal("the lease was released while the sidecar's descendant still ran") |
| 514 | } |
| 515 | f.b.setSidecarDescendant(false) |
| 516 | waitFor(t, "the lease release once the cgroup is empty", func() bool { |
| 517 | f.a.sweepOnce() |
| 518 | return rep.isClosed() |
| 519 | }) |
| 520 | } |
| 521 | |
| 522 | // A supervised restart of a sidecar that exited on its own is a replacement |
| 523 | // too: if something it forked is left in the cgroup, the old one becomes a |
| 524 | // tracked stray and no new sidecar starts beside it. |
| 525 | func TestASupervisedRestartWaitsForTheCrashedSidecarsDescendant(t *testing.T) { |
| 526 | t.Parallel() |
| 527 | f := newSwapFixture(t) |
| 528 | f.b.sidecarStaged = true |
| 529 | f.a.sweepOnce() |
| 530 | waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 }) |
| 531 | f.b.setSidecarDescendant(true) |
| 532 | f.b.lastSidecar().kill() |
| 533 | f.clk.advance(3 * time.Second) |
| 534 | f.a.sweepOnce() |
| 535 | waitFor(t, "the refused restart", func() bool { return f.a.sidecarSettled(f.host) && f.a.strayCount(f.host) == 1 }) |
| 536 | if n := len(f.b.sidecarGenPaths()); n != 1 { |
| 537 | t.Fatalf("%d sidecar starts: a restart ran beside the crashed sidecar's descendant", n) |
| 538 | } |
| 539 | f.b.setSidecarDescendant(false) |
| 540 | waitFor(t, "the restart once the cgroup is empty", func() bool { |
| 541 | f.clk.advance(time.Minute) |
| 542 | f.a.sweepOnce() |
| 543 | return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 2 |
| 544 | }) |
| 545 | if got := f.a.strayCount(f.host); got != 0 { |
| 546 | t.Fatalf("tracked strays = %d after the proof, want 0", got) |
| 547 | } |
| 548 | } |
| 549 | |
| 550 | // A legacy sidecar left on a retired generation's build (a replacement whose |
| 551 | // stop of it was unproven) is replaced by the sweep's retry. |
| 552 | func TestSweepReplacesASidecarLeftOnARetiredGeneration(t *testing.T) { |
| 553 | t.Parallel() |
| 554 | f := newSwapFixture(t) |
| 555 | f.b.sidecarStaged = true |
| 556 | f.a.sweepOnce() |
| 557 | waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 }) |
| 558 | old := f.b.lastSidecar() |
| 559 | f.a.mu.Lock() |
| 560 | f.a.entries[f.host].sidecarGen = f.incumbent.N - 1 // a build the entry no longer serves |
| 561 | f.a.mu.Unlock() |
| 562 | f.a.sweepOnce() |
| 563 | waitFor(t, "the replacement", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 2 }) |
| 564 | if !old.isStopped() { |
| 565 | t.Fatal("the retired generation's sidecar was not stopped") |
| 566 | } |
| 567 | f.a.mu.Lock() |
| 568 | got := f.a.entries[f.host].sidecarGen |
| 569 | f.a.mu.Unlock() |
| 570 | if got != f.incumbent.N { |
| 571 | t.Fatalf("sidecarGen = %d, want the current generation %d", got, f.incumbent.N) |
| 572 | } |
| 573 | } |
| 574 | |
| 575 | // Under plan 2's containment the sidecar keeps its role-sidecar child: no |
| 576 | // limit of its own, and the generation parent keeps memory.oom.group=0 and |
| 577 | // holds no process (spec §7.3, plan 2 D4). Spawning through spawnInCgroup |
| 578 | // instead writes the tier and memory.oom.group=1 onto the parent and makes |
| 579 | // no role-sidecar child. |
| 580 | func TestStartSidecarGenKeepsTheRoleSidecarChild(t *testing.T) { |
| 581 | const host = "sc.example.com" |
| 582 | b, parent := delegatedBackend(t, host) |
| 583 | cgroup.FakePlacementForTest(t) // a temp dir is no cgroupfs, and a generation is never moved by PID |
| 584 | b.GenContainment = true |
| 585 | v2 := filepath.Join(b.OptDir, "versions", "v2", "app") |
| 586 | if err := os.MkdirAll(filepath.Dir(v2), 0o755); err != nil { |
| 587 | t.Fatal(err) |
| 588 | } |
| 589 | // It stays up until the stop: its exit removes its own child. |
| 590 | if err := os.WriteFile(v2, []byte("#!/bin/sh\nexec sleep 30\n"), 0o755); err != nil { |
| 591 | t.Fatal(err) |
| 592 | } |
| 593 | if err := os.WriteFile(filepath.Join(b.DataDir, host+"-sidecar.env"), []byte("X=1\n"), 0o600); err != nil { |
| 594 | t.Fatal(err) |
| 595 | } |
| 596 | rt := registry.Route{Host: host, Kind: "instance", DBPath: provision.DefaultDBPath(b.DataDir, host), |
| 597 | Socket: filepath.Join(provision.SocketDir, host+".sock")} |
| 598 | p, err := b.StartSidecarGen(context.Background(), rt, Gen{N: 3, Build: "v2", StagedPath: v2}) |
| 599 | if err != nil { |
| 600 | t.Fatal(err) |
| 601 | } |
| 602 | // Checked before the stop: the exit's cleanup removes the empty child |
| 603 | // (with placement before exec nothing is written into it to keep it). |
| 604 | if fi, err := os.Stat(filepath.Join(parent, "role-sidecar")); err != nil || !fi.IsDir() { |
| 605 | _ = p.Stop(context.Background()) |
| 606 | t.Fatalf("no role-sidecar child: %v", err) |
| 607 | } |
| 608 | if _, err := os.Stat(filepath.Join(parent, "role-sidecar", "memory.max")); !os.IsNotExist(err) { |
| 609 | t.Errorf("the sidecar's child got a limit of its own: %v", err) |
| 610 | } |
| 611 | _ = p.Stop(context.Background()) |
| 612 | if got, err := os.ReadFile(filepath.Join(parent, "memory.oom.group")); err != nil || string(got) != "0" { |
| 613 | t.Errorf("parent memory.oom.group = %q, %v; want 0", got, err) |
| 614 | } |
| 615 | } |
| 616 | |
| 617 | // ExecBackend: a flat sidecar execs the generation's staged path. A nested |
| 618 | // sidecar is refused until Task 9 hands root the staged executable: it never |
| 619 | // reads the live link to decide, because a repoint can land between that |
| 620 | // read and the start. |
| 621 | func TestExecBackendStartSidecarGen(t *testing.T) { |
| 622 | t.Parallel() |
| 623 | stage := func(t *testing.T, b *ExecBackend, host string) (v2, marker string) { |
| 624 | t.Helper() |
| 625 | marker = filepath.Join(t.TempDir(), "argv0") |
| 626 | v2 = filepath.Join(b.OptDir, "versions", "v2", "app") |
| 627 | if err := os.MkdirAll(filepath.Dir(v2), 0o755); err != nil { |
| 628 | t.Fatal(err) |
| 629 | } |
| 630 | if err := os.WriteFile(v2, []byte("#!/bin/sh\nprintf '%s' \"$0\" > '"+marker+"'\n"), 0o755); err != nil { |
| 631 | t.Fatal(err) |
| 632 | } |
| 633 | if err := os.WriteFile(filepath.Join(b.DataDir, host+"-sidecar.env"), []byte("X=1\n"), 0o600); err != nil { |
| 634 | t.Fatal(err) |
| 635 | } |
| 636 | return v2, marker |
| 637 | } |
| 638 | t.Run("flat", func(t *testing.T) { |
| 639 | b := testExecBackend(t) |
| 640 | host := "sc.example.com" |
| 641 | stageLive(t, b, host, "v1") |
| 642 | v2, marker := stage(t, b, host) |
| 643 | // The default db and socket: Classify puts this route in the flat layout. |
| 644 | rt := registry.Route{Host: host, Kind: "instance", DBPath: provision.DefaultDBPath(b.DataDir, host), |
| 645 | Socket: filepath.Join(provision.SocketDir, host+".sock")} |
| 646 | if l, why := b.Layout(rt); l != LayoutFlat { |
| 647 | t.Fatalf("route classified %v (%s), want the flat layout", l, why) |
| 648 | } |
| 649 | p, err := b.StartSidecarGen(context.Background(), rt, Gen{N: 3, Build: "v2", StagedPath: v2}) |
| 650 | if err != nil { |
| 651 | t.Fatal(err) |
| 652 | } |
| 653 | defer p.Stop(context.Background()) //nolint:errcheck |
| 654 | if got := waitForFile(t, marker, 2*time.Second); got != v2 { |
| 655 | t.Fatalf("the sidecar execd %q, want the generation's %q", got, v2) |
| 656 | } |
| 657 | }) |
| 658 | t.Run("foreign", func(t *testing.T) { |
| 659 | b := testExecBackend(t) |
| 660 | host := "fsc.example.com" |
| 661 | stageLive(t, b, host, "v1") |
| 662 | v2, marker := stage(t, b, host) |
| 663 | rt := registry.Route{Host: host, Kind: "instance", Socket: filepath.Join(t.TempDir(), "s.sock")} |
| 664 | if l, _ := b.Layout(rt); l != LayoutForeign { |
| 665 | t.Fatalf("route classified %v, want the foreign layout", l) |
| 666 | } |
| 667 | p, err := b.StartSidecarGen(context.Background(), rt, Gen{N: 3, Build: "v2", StagedPath: v2}) |
| 668 | if err != nil { |
| 669 | t.Fatal(err) |
| 670 | } |
| 671 | defer p.Stop(context.Background()) //nolint:errcheck |
| 672 | if got := waitForFile(t, marker, 2*time.Second); got != v2 { |
| 673 | t.Fatalf("the sidecar execd %q, want the generation's %q", got, v2) |
| 674 | } |
| 675 | }) |
| 676 | t.Run("nested", func(t *testing.T) { |
| 677 | b := testExecBackend(t) |
| 678 | rd := t.TempDir() |
| 679 | b.RunDirOverride = rd |
| 680 | host := "nsc.example.com" |
| 681 | stageLive(t, b, host, "v1") |
| 682 | rt := registry.Route{Host: host, Kind: "instance", DBPath: provision.NestedDBPath(b.DataDir, host), |
| 683 | Socket: filepath.Join(rd, "instances", provision.InstanceID(host), "app.sock")} |
| 684 | for _, build := range []string{"v1", "v2"} { // the live build and another |
| 685 | // A generation with no staged executable (woken on the live |
| 686 | // link) has nothing to name to root, so it still refuses. |
| 687 | if _, err := b.StartSidecarGen(context.Background(), rt, Gen{N: 3, Build: build}); !errors.Is(err, errNestedSidecarGen) { |
| 688 | t.Fatalf("serving %s with no staged path: %v, want errNestedSidecarGen", build, err) |
| 689 | } |
| 690 | } |
| 691 | }) |
| 692 | } |
| 693 | |
| 694 | // KillSidecar: the nested arm asks the sidecar unit's stop, and with no |
| 695 | // Control wired it answers an error, never a nil dereference. Flat without |
| 696 | // containment has no cgroup of its own to prove: errNoGenContainment. |
| 697 | func TestExecBackendKillSidecar(t *testing.T) { |
| 698 | t.Run("nested without a control channel", func(t *testing.T) { |
| 699 | b := testExecBackend(t) |
| 700 | rd := t.TempDir() |
| 701 | b.RunDirOverride = rd |
| 702 | host := "nsc.example.com" |
| 703 | rt := registry.Route{Host: host, Kind: "instance", DBPath: provision.NestedDBPath(b.DataDir, host), |
| 704 | Socket: filepath.Join(rd, "instances", provision.InstanceID(host), "app.sock")} |
| 705 | if err := b.KillSidecar(context.Background(), rt); err == nil { |
| 706 | t.Fatal("KillSidecar with no Control proved a nested sidecar gone") |
| 707 | } |
| 708 | }) |
| 709 | t.Run("flat without containment", func(t *testing.T) { |
| 710 | b := testExecBackend(t) |
| 711 | host := "sc.example.com" |
| 712 | rt := registry.Route{Host: host, Kind: "instance", DBPath: provision.DefaultDBPath(b.DataDir, host), |
| 713 | Socket: filepath.Join(provision.SocketDir, host+".sock")} |
| 714 | if err := b.KillSidecar(context.Background(), rt); !errors.Is(err, errNoGenContainment) { |
| 715 | t.Fatalf("KillSidecar = %v, want errNoGenContainment", err) |
| 716 | } |
| 717 | }) |
| 718 | } |
| 719 | |
| 720 | var _ SidecarGenStarter = (*SplitBackend)(nil) |
| 721 | |
amadan