| new file mode 100644 |
| index 000000000..2b4c79cc0 |
| --- /dev/null |
| +++ b/internal/activator/swap_sidecar_test.go |
| @@ -0,0 +1,680 @@ |
| +package activator |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "os" |
| + "path/filepath" |
| + "slices" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/provision" |
| + "github.com/carlosframework/platform/internal/registry" |
| +) |
| + |
| +// sidecarStartDoneOf is the host's supervised-start join channel, nil when |
| +// none is in flight. |
| +func (a *Service) sidecarStartDoneOf(host string) chan struct{} { |
| + a.mu.Lock() |
| + defer a.mu.Unlock() |
| + if e := a.entries[host]; e != nil { |
| + return e.sidecarStartDone |
| + } |
| + return nil |
| +} |
| + |
| +// sidecarProc is the i'th sidecar the fake backend started. |
| +func (g *fakeSwapBackend) sidecarProc(i int) *fakeProc { |
| + g.mu.Lock() |
| + defer g.mu.Unlock() |
| + if i >= len(g.sidecars) { |
| + return nil |
| + } |
| + return g.sidecars[i] |
| +} |
| + |
| +// sidecarStoppingIn reports whether the i'th sidecar's Stop has been |
| +// entered: with a stop gate on it, that Stop is in flight and blocked. |
| +func (g *fakeSwapBackend) sidecarStoppingIn(i int) bool { |
| + p := g.sidecarProc(i) |
| + return p != nil && p.stopEntered() > 0 |
| +} |
| + |
| +// generationSwitched is true once the entry's current generation is not the |
| +// fixture's incumbent. |
| +func (f *swapFixture) generationSwitched() bool { |
| + g, _ := f.a.genStateForTest(f.host) |
| + return g.N != f.incumbent.N |
| +} |
| + |
| +// Flag off: every legacy sidecar spawn (the wake tail and the supervisor) |
| +// goes through StartSidecar and the live link, as today. |
| +func TestLegacySidecarUsesTheLiveLinkWithSwapOff(t *testing.T) { |
| + f := newSwapFixture(t) |
| + f.a.gens = false |
| + f.b.sidecarStaged = true |
| + f.a.Bounce(f.host) |
| + waitFor(t, "the stop", func() bool { return f.a.entryCount() == 0 }) |
| + arrangeAwake(t, f.a, testRoute(f.host)) |
| + if got := f.b.sidecarStartCount(); got != 1 { |
| + t.Fatalf("StartSidecar calls = %d after the wake, want 1", got) |
| + } |
| + f.b.lastSidecar().kill() |
| + f.clk.advance(3 * time.Second) |
| + f.a.sweepOnce() |
| + waitFor(t, "the supervised restart", func() bool { return f.a.sidecarSettled(f.host) && f.b.sidecarStartCount() == 2 }) |
| + if paths := f.b.sidecarGenPaths(); len(paths) != 0 { |
| + t.Fatalf("StartSidecarGen ran with swaps off: %v", paths) |
| + } |
| +} |
| + |
| +// Flag off, ownership: startSidecar asks ownedLocked whatever the flag (an |
| +// accepted dormancy exception, recorded in task-7-adapt.md). Exactly what |
| +// changes against today, called directly so no earlier sweep check masks it: |
| +// a host that does not own itself (fenced, shutting down, lease lost, no |
| +// longer the host's entry) gets NO co-spawned sidecar, where today's |
| +// StartSidecar would have spawned one. A host that owns itself gets the |
| +// live-link sidecar exactly as before, and StartSidecarGen is never used. |
| +func TestLegacySidecarOwnershipCheckRunsWithSwapOff(t *testing.T) { |
| + f := newSwapFixture(t) |
| + f.a.gens = false |
| + f.b.sidecarStaged = true |
| + f.a.mu.Lock() |
| + e := f.a.entries[f.host] |
| + rt := e.route |
| + f.a.mu.Unlock() |
| + rep := f.b.lastRep() |
| + |
| + // Owned: today's spawn, through the live link. |
| + sc, err := f.a.startSidecar(context.Background(), e, rt, f.incumbent, false) |
| + if err != nil || sc == nil { |
| + t.Fatalf("an owned host's sidecar = %v, %v; want the live-link spawn", sc, err) |
| + } |
| + if got := f.b.sidecarStartCount(); got != 1 { |
| + t.Fatalf("StartSidecar calls = %d, want 1", got) |
| + } |
| + if paths := f.b.sidecarGenPaths(); len(paths) != 0 { |
| + t.Fatalf("StartSidecarGen ran with swaps off: %v", paths) |
| + } |
| + |
| + refused := func(name string, arm, disarm func(), want error) { |
| + t.Helper() |
| + arm() |
| + sc, err := f.a.startSidecar(context.Background(), e, rt, f.incumbent, false) |
| + disarm() |
| + if !errors.Is(err, want) || sc != nil { |
| + t.Fatalf("%s: startSidecar = %v, %v; want %v and no process", name, sc, err, want) |
| + } |
| + if got := f.b.sidecarStartCount(); got != 1 { |
| + t.Fatalf("%s: StartSidecar calls = %d, want 1 (none beside a host that does not own itself)", name, got) |
| + } |
| + } |
| + refused("fenced", |
| + func() { f.a.mu.Lock(); f.a.fences = map[string]uint64{f.host: 1}; f.a.mu.Unlock() }, |
| + func() { f.a.mu.Lock(); f.a.fences = nil; f.a.mu.Unlock() }, errSwapFenced) |
| + refused("shutting down", |
| + func() { f.a.mu.Lock(); f.a.shuttingDown = true; f.a.mu.Unlock() }, |
| + func() { f.a.mu.Lock(); f.a.shuttingDown = false; f.a.mu.Unlock() }, errSwapShutdown) |
| + refused("lease lost", |
| + func() { rep.setErr(errors.New("lease lost")) }, |
| + func() { rep.setErr(nil) }, errSwapLeaseLost) |
| + refused("not the host's entry", |
| + func() { f.a.mu.Lock(); delete(f.a.entries, f.host); f.a.mu.Unlock() }, |
| + func() { f.a.mu.Lock(); f.a.entries[f.host] = e; f.a.mu.Unlock() }, errSwapNotAwake) |
| + |
| + // And once owned again, the same call spawns as before. |
| + if _, err := f.a.startSidecar(context.Background(), e, rt, f.incumbent, false); err != nil { |
| + t.Fatalf("owned again: %v", err) |
| + } |
| + if got := f.b.sidecarStartCount(); got != 2 { |
| + t.Fatalf("StartSidecar calls = %d, want 2", got) |
| + } |
| +} |
| + |
| +// Flag on: late attachment, the supervisor, a swap's replacement and the |
| +// wake tail all exec the CURRENT generation's captured staged path (spec |
| +// §7.5). The live link names v3 throughout and is never used. |
| +func TestLegacySidecarRunsTheServingGeneration(t *testing.T) { |
| + t.Run("late attachment and the supervisor", func(t *testing.T) { |
| + f := newSwapFixture(t) |
| + f.b.execVerFn = func() string { return "v3" } |
| + f.b.sidecarStaged = true // staged after the wake: the sweep attaches it |
| + f.a.sweepOnce() |
| + waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 }) |
| + f.b.lastSidecar().kill() |
| + f.clk.advance(3 * time.Second) |
| + f.a.sweepOnce() |
| + waitFor(t, "the supervised restart", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 2 }) |
| + for _, p := range f.b.sidecarGenPaths() { |
| + if p != f.incumbent.StagedPath { |
| + t.Fatalf("a sidecar ran %q, want the serving generation's %q", p, f.incumbent.StagedPath) |
| + } |
| + } |
| + if got := f.b.sidecarStartCount(); got != 0 { |
| + t.Fatalf("StartSidecar (live link) ran %d time(s) with swaps on", got) |
| + } |
| + }) |
| + t.Run("a swap replaces the sidecar stop-then-start", func(t *testing.T) { |
| + f := newSwapFixture(t) |
| + f.b.sidecarStaged = true |
| + f.a.sweepOnce() |
| + waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 }) |
| + old := f.b.lastSidecar() |
| + if res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest); res.Outcome != SwapDone { |
| + t.Fatalf("Swap = %+v", res) |
| + } |
| + paths := f.b.sidecarGenPaths() |
| + if len(paths) != 2 || paths[1] != f.staged { |
| + t.Fatalf("sidecar spawns %v: the replacement must run the new generation's %q", paths, f.staged) |
| + } |
| + if !old.isStopped() { |
| + t.Fatal("the old sidecar outlived the replacement") |
| + } |
| + ev := f.b.events() |
| + if i, j := slices.Index(ev, "stop:sidecar"), slices.Index(ev, "startSidecarGen:"+f.staged); i < 0 || j < 0 || i > j { |
| + t.Fatalf("order %v: stop the old sidecar, then start the new one", ev) |
| + } |
| + }) |
| + t.Run("the wake tail of a hold", func(t *testing.T) { |
| + f := newSwapFixture(t) |
| + f.b.sidecarStaged = true |
| + if res := f.a.Swap(context.Background(), f.attempt(ModeHold), f.newest); res.Outcome != SwapDone { |
| + t.Fatalf("Swap = %+v", res) |
| + } |
| + if paths := f.b.sidecarGenPaths(); len(paths) == 0 || paths[len(paths)-1] != f.staged { |
| + t.Fatalf("the hold's wake tail started sidecars %v, want %q last", paths, f.staged) |
| + } |
| + }) |
| +} |
| + |
| +// The swap's replacement holds the supervisor's interlock, so a supervised |
| +// restart that falls due meanwhile cannot start a second sidecar (review |
| +// focus 4). |
| +func TestSwapSidecarReplacementHoldsTheSupervisorOff(t *testing.T) { |
| + f := newSwapFixture(t) |
| + f.b.sidecarStaged = true |
| + f.a.sweepOnce() |
| + waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 }) |
| + gate := make(chan struct{}) |
| + f.b.gmu.Lock() |
| + f.b.sidecarGenGate = gate |
| + f.b.gmu.Unlock() |
| + results := make(chan SwapResult, 1) |
| + go func() { results <- f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) }() |
| + waitFor(t, "the replacement's start", func() bool { return len(f.b.sidecarGenPaths()) == 2 }) |
| + held := f.a.sidecarStartDoneOf(f.host) |
| + if held == nil { |
| + t.Fatal("the replacement holds no interlock") |
| + } |
| + f.clk.advance(time.Minute) |
| + f.a.sweepOnce() // the old sidecar is stopped (exited): the supervisor would restart it |
| + // The supervisor claims sidecarStarting under the sweep's own lock, so a |
| + // claim would have replaced the join channel before sweepOnce returned. |
| + if now := f.a.sidecarStartDoneOf(f.host); now != held { |
| + t.Fatal("a supervised restart claimed the interlock during the replacement") |
| + } |
| + if n := len(f.b.sidecarGenPaths()); n != 2 { |
| + t.Fatalf("sidecar spawns = %d during the replacement, want 2", n) |
| + } |
| + close(gate) |
| + if res := <-results; res.Outcome != SwapDone { |
| + t.Fatalf("Swap = %+v", res) |
| + } |
| +} |
| + |
| +// A supervised restart already in flight when the swap switches captured the |
| +// OLD generation. The swap's replacement joins it and then replaces what it |
| +// installed, so the new server never runs beside the old build's sidecar |
| +// (spec §7.5). Returning early because sidecarStarting is set (the old |
| +// replaceSidecar) leaves the supervisor's old-build sidecar installed with |
| +// no replacement pending: the third spawn never comes. |
| +func TestSwapSidecarReplacementJoinsAnInFlightSupervisor(t *testing.T) { |
| + f := newSwapFixture(t) |
| + f.b.sidecarStaged = true |
| + f.a.sweepOnce() |
| + waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 }) |
| + gate := make(chan struct{}) |
| + f.b.gmu.Lock() |
| + f.b.sidecarGenGate = gate |
| + f.b.gmu.Unlock() |
| + f.b.lastSidecar().kill() |
| + f.clk.advance(3 * time.Second) |
| + f.a.sweepOnce() // the supervisor starts a sidecar for the incumbent and blocks |
| + waitFor(t, "the supervisor's start", func() bool { return len(f.b.sidecarGenPaths()) == 2 }) |
| + results := make(chan SwapResult, 1) |
| + go func() { results <- f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) }() |
| + waitFor(t, "the switch", f.generationSwitched) |
| + close(gate) |
| + if res := <-results; res.Outcome != SwapDone { |
| + t.Fatalf("Swap = %+v", res) |
| + } |
| + waitFor(t, "the replacement onto the new generation", func() bool { |
| + p := f.b.sidecarGenPaths() |
| + return len(p) == 3 && f.a.sidecarSettled(f.host) |
| + }) |
| + paths := f.b.sidecarGenPaths() |
| + if paths[1] != f.incumbent.StagedPath || paths[2] != f.staged { |
| + t.Fatalf("sidecar spawns %v: want the supervisor's on %q, then the replacement on %q", paths, f.incumbent.StagedPath, f.staged) |
| + } |
| + // The supervisor's sidecar is stopped by the replacement's |
| + // stop-then-start. |
| + f.b.mu.Lock() |
| + supervised, current := f.b.sidecars[1], f.b.sidecars[len(f.b.sidecars)-1] |
| + f.b.mu.Unlock() |
| + if !supervised.isStopped() { |
| + t.Fatal("the supervisor's old-build sidecar outlived the swap") |
| + } |
| + f.a.mu.Lock() |
| + installed := f.a.entries[f.host].sidecar |
| + f.a.mu.Unlock() |
| + if installed != Proc(current) { |
| + t.Fatal("the installed sidecar is not the replacement") |
| + } |
| +} |
| + |
| +// The supervisor's start for the retired generation is joined until that |
| +// process is proven gone (spec §7.4, §7.5): the replacement never starts |
| +// while it still runs. Discarding it asynchronously let the supervisor |
| +// close its join at once, so the replacement started beside a sidecar |
| +// still draining on the old build. |
| +func TestSwapSidecarNeverRunsBesideTheRetiredGenerationsSidecar(t *testing.T) { |
| + f := newSwapFixture(t) |
| + f.b.sidecarStaged = true |
| + f.a.sweepOnce() |
| + waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 }) |
| + startGate, stopGate := make(chan struct{}), make(chan struct{}) |
| + f.b.gmu.Lock() |
| + f.b.sidecarGenGate, f.b.sidecarStopGate = startGate, stopGate |
| + f.b.gmu.Unlock() |
| + f.b.lastSidecar().kill() |
| + f.clk.advance(3 * time.Second) |
| + f.a.sweepOnce() // the supervisor starts the incumbent's sidecar and blocks |
| + waitFor(t, "the supervisor's start", func() bool { return len(f.b.sidecarGenPaths()) == 2 }) |
| + results := make(chan SwapResult, 1) |
| + go func() { results <- f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) }() |
| + waitFor(t, "the switch", f.generationSwitched) |
| + close(startGate) // the supervisor's sidecar starts; its Stop blocks on stopGate |
| + // The retired generation's sidecar is being stopped, and the replacement |
| + // is sequenced behind that Stop: nothing can have started beside it. |
| + waitFor(t, "the retired sidecar's stop to be in flight", func() bool { return f.b.sidecarStoppingIn(1) }) |
| + if n := len(f.b.sidecarGenPaths()); n != 2 { |
| + t.Fatalf("%d sidecar starts: the replacement started while the retired generation's sidecar still ran", n) |
| + } |
| + select { |
| + case res := <-results: |
| + t.Fatalf("the swap returned (%+v) before the retired sidecar was proven gone", res) |
| + default: |
| + } |
| + close(stopGate) |
| + if res := <-results; res.Outcome != SwapDone { |
| + t.Fatalf("Swap = %+v", res) |
| + } |
| + waitFor(t, "the replacement", func() bool { return len(f.b.sidecarGenPaths()) == 3 }) |
| + if p := f.b.sidecarGenPaths(); p[2] != f.staged { |
| + t.Fatalf("sidecar starts %v: want the new generation's last", p) |
| + } |
| +} |
| + |
| +// A terminal stop that begins while a supervised sidecar start is in flight |
| +// joins that start, and the sidecar it produced, before the lease is |
| +// released (spec §7.4). Discarding it asynchronously let the supervisor |
| +// close its join at once, so the stop released the lease while the new |
| +// sidecar was still running. |
| +func TestSwapStopJoinsTheSidecarAStartLeftBehind(t *testing.T) { |
| + f := newSwapFixture(t) |
| + f.b.sidecarStaged = true |
| + f.a.sweepOnce() |
| + waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 }) |
| + startGate, stopGate := make(chan struct{}), make(chan struct{}) |
| + f.b.gmu.Lock() |
| + f.b.sidecarGenGate, f.b.sidecarStopGate = startGate, stopGate |
| + f.b.gmu.Unlock() |
| + f.b.lastSidecar().kill() |
| + f.clk.advance(3 * time.Second) |
| + f.a.sweepOnce() // the supervisor's start blocks |
| + waitFor(t, "the supervisor's start", func() bool { return len(f.b.sidecarGenPaths()) == 2 }) |
| + rep := f.b.lastRep() |
| + f.a.Bounce(f.host) |
| + // The stop is joined on the supervisor's start: it goes no further (no |
| + // instance stop, so no lease release) while the start is in flight. Only |
| + // a bounded look can show an absence, so it fails the moment the instance |
| + // is stopped. |
| + for range 40 { |
| + if slices.Contains(f.b.events(), "stop:instance") { |
| + t.Fatal("the stop went past its join on the in-flight sidecar start") |
| + } |
| + time.Sleep(5 * time.Millisecond) |
| + } |
| + close(startGate) // the start lands on a stopping entry; its sidecar's Stop blocks |
| + waitFor(t, "the left-behind sidecar's stop to be in flight", func() bool { return f.b.sidecarStoppingIn(1) }) |
| + if rep.isClosed() { |
| + t.Fatal("the lease was released while the sidecar a start left behind still ran") |
| + } |
| + close(stopGate) |
| + waitFor(t, "the lease release", rep.isClosed) |
| + if started := f.b.sidecarProc(1); !started.isStopped() { |
| + t.Fatal("the sidecar the start left behind was never stopped") |
| + } |
| +} |
| + |
| +// Spec §7.4: a discarded sidecar whose stop is unproven is never forgotten. |
| +// It stays tracked, the host takes no new wake beside it, and the sweep |
| +// stops it again until it exits. Logging it and moving on let the next wake |
| +// run a second sidecar beside a live one. |
| +func TestAStraySidecarIsNeverForgotten(t *testing.T) { |
| + f := newSwapFixture(t) |
| + f.a.Bounce(f.host) |
| + waitFor(t, "the instance asleep", func() bool { return f.a.entryCount() == 0 }) |
| + stray := &fakeProc{label: "sidecar", rec: f.b.fakeBackend, stuck: true} |
| + f.a.discardSidecar(testRoute(f.host), stray) |
| + if got := f.a.strayCount(f.host); got != 1 { |
| + t.Fatalf("tracked strays = %d, want 1", got) |
| + } |
| + if _, err := f.a.beginRequest(context.Background(), testRoute(f.host)); err == nil || !strings.Contains(err.Error(), errWakeUnproven.Error()) { |
| + t.Fatalf("a wake beside an unproven sidecar = %v, want refused", err) |
| + } |
| + stray.exit() |
| + f.a.sweepOnce() // the reaper re-proves it off the lock |
| + waitFor(t, "the stray proven gone", func() bool { return f.a.strayCount(f.host) == 0 }) |
| + release, err := f.a.beginRequest(context.Background(), testRoute(f.host)) |
| + if err != nil { |
| + t.Fatalf("the wake once the stray exited: %v", err) |
| + } |
| + release() |
| +} |
| + |
| +func (a *Service) strayCount(host string) int { |
| + a.mu.Lock() |
| + defer a.mu.Unlock() |
| + return len(a.straySidecars[host]) |
| +} |
| + |
| +// strayReaping is true while any stray of host has a re-proof in flight. |
| +func (a *Service) strayReaping(host string) bool { |
| + a.mu.Lock() |
| + defer a.mu.Unlock() |
| + for _, s := range a.straySidecars[host] { |
| + if s.reaping { |
| + return true |
| + } |
| + } |
| + return false |
| +} |
| + |
| +func (a *Service) stopPending(host string) bool { |
| + a.mu.Lock() |
| + defer a.mu.Unlock() |
| + e := a.entries[host] |
| + return e != nil && e.pendingStop |
| +} |
| + |
| +// Spec §7.4, §7.5: a legacy sidecar is gone only when its cgroup is empty. |
| +// childProc.Stop signals only the leader, so a process the sidecar forked |
| +// outlives its exit in role-sidecar. Here the leader exits but a descendant |
| +// stays (KillSidecar cannot empty the cgroup): the replacement does not |
| +// start beside it, and a terminal stop keeps the lease until it is proven |
| +// gone. Accepting Exited() started the new sidecar beside the descendant |
| +// and released the lease under it. |
| +func TestASidecarsDescendantBlocksItsReplacementAndTheLease(t *testing.T) { |
| + f := newSwapFixture(t) |
| + f.b.sidecarStaged = true |
| + f.a.sweepOnce() |
| + waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 }) |
| + f.b.setSidecarDescendant(true) |
| + if res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest); res.Outcome != SwapDone { |
| + t.Fatalf("Swap = %+v", res) |
| + } |
| + if got := f.a.strayCount(f.host); got != 1 { |
| + t.Fatalf("tracked strays = %d after the replacement, want 1 (the old sidecar's descendant)", got) |
| + } |
| + // A supervisor pass that falls due must not start one either. It runs |
| + // startSidecar, which refuses while the stray is tracked. |
| + f.clk.advance(time.Minute) |
| + f.a.sweepOnce() |
| + waitFor(t, "the supervisor's refused start", func() bool { return f.a.sidecarSettled(f.host) }) |
| + if n := len(f.b.sidecarGenPaths()); n != 1 { |
| + t.Fatalf("%d sidecar starts: a new sidecar started beside the old one's descendant", n) |
| + } |
| + rep := f.b.lastRep() |
| + f.a.Bounce(f.host) |
| + waitFor(t, "the stop to wait on the sidecar's cgroup", func() bool { return f.a.stopPending(f.host) }) |
| + // Each sweep re-proves the stray; the descendant is still there. |
| + for range 2 { |
| + before := f.b.sidecarKillCount() |
| + f.a.sweepOnce() |
| + waitFor(t, "a proof attempt", func() bool { return f.b.sidecarKillCount() > before }) |
| + waitFor(t, "the re-proof to finish", func() bool { return !f.a.strayReaping(f.host) }) |
| + if rep.isClosed() { |
| + t.Fatal("the lease was released while the sidecar's descendant still ran") |
| + } |
| + } |
| + if got := f.a.strayCount(f.host); got != 1 { |
| + t.Fatalf("tracked strays = %d with the descendant alive, want 1", got) |
| + } |
| + f.b.setSidecarDescendant(false) |
| + waitFor(t, "the lease release once the cgroup is empty", func() bool { |
| + f.a.sweepOnce() |
| + return rep.isClosed() |
| + }) |
| +} |
| + |
| +// A terminal stop's co-stop is held to the same proof: the sidecar's leader |
| +// exits but a process it forked is left in the cgroup, so the stop keeps the |
| +// lease (releasable) until the sweep proves the cgroup empty. |
| +func TestATerminalStopKeepsTheLeaseOverASidecarsDescendant(t *testing.T) { |
| + f := newSwapFixture(t) |
| + f.b.sidecarStaged = true |
| + f.a.sweepOnce() |
| + waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 }) |
| + f.b.setSidecarDescendant(true) |
| + rep := f.b.lastRep() |
| + f.a.Bounce(f.host) |
| + waitFor(t, "the stop to wait on the sidecar's cgroup", func() bool { return f.a.stopPending(f.host) }) |
| + if got := f.a.strayCount(f.host); got != 1 { |
| + t.Fatalf("tracked strays = %d after the co-stop, want 1", got) |
| + } |
| + before := f.b.sidecarKillCount() |
| + f.a.sweepOnce() |
| + waitFor(t, "a proof attempt", func() bool { return f.b.sidecarKillCount() > before }) |
| + waitFor(t, "the re-proof to finish", func() bool { return !f.a.strayReaping(f.host) }) |
| + if rep.isClosed() { |
| + t.Fatal("the lease was released while the sidecar's descendant still ran") |
| + } |
| + f.b.setSidecarDescendant(false) |
| + waitFor(t, "the lease release once the cgroup is empty", func() bool { |
| + f.a.sweepOnce() |
| + return rep.isClosed() |
| + }) |
| +} |
| + |
| +// A supervised restart of a sidecar that exited on its own is a replacement |
| +// too: if something it forked is left in the cgroup, the old one becomes a |
| +// tracked stray and no new sidecar starts beside it. |
| +func TestASupervisedRestartWaitsForTheCrashedSidecarsDescendant(t *testing.T) { |
| + f := newSwapFixture(t) |
| + f.b.sidecarStaged = true |
| + f.a.sweepOnce() |
| + waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 }) |
| + f.b.setSidecarDescendant(true) |
| + f.b.lastSidecar().kill() |
| + f.clk.advance(3 * time.Second) |
| + f.a.sweepOnce() |
| + waitFor(t, "the refused restart", func() bool { return f.a.sidecarSettled(f.host) && f.a.strayCount(f.host) == 1 }) |
| + if n := len(f.b.sidecarGenPaths()); n != 1 { |
| + t.Fatalf("%d sidecar starts: a restart ran beside the crashed sidecar's descendant", n) |
| + } |
| + f.b.setSidecarDescendant(false) |
| + waitFor(t, "the restart once the cgroup is empty", func() bool { |
| + f.clk.advance(time.Minute) |
| + f.a.sweepOnce() |
| + return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 2 |
| + }) |
| + if got := f.a.strayCount(f.host); got != 0 { |
| + t.Fatalf("tracked strays = %d after the proof, want 0", got) |
| + } |
| +} |
| + |
| +// A legacy sidecar left on a retired generation's build (a replacement whose |
| +// stop of it was unproven) is replaced by the sweep's retry. |
| +func TestSweepReplacesASidecarLeftOnARetiredGeneration(t *testing.T) { |
| + f := newSwapFixture(t) |
| + f.b.sidecarStaged = true |
| + f.a.sweepOnce() |
| + waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 }) |
| + old := f.b.lastSidecar() |
| + f.a.mu.Lock() |
| + f.a.entries[f.host].sidecarGen = f.incumbent.N - 1 // a build the entry no longer serves |
| + f.a.mu.Unlock() |
| + f.a.sweepOnce() |
| + waitFor(t, "the replacement", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 2 }) |
| + if !old.isStopped() { |
| + t.Fatal("the retired generation's sidecar was not stopped") |
| + } |
| + f.a.mu.Lock() |
| + got := f.a.entries[f.host].sidecarGen |
| + f.a.mu.Unlock() |
| + if got != f.incumbent.N { |
| + t.Fatalf("sidecarGen = %d, want the current generation %d", got, f.incumbent.N) |
| + } |
| +} |
| + |
| +// Under plan 2's containment the sidecar keeps its role-sidecar child: no |
| +// limit of its own, and the generation parent keeps memory.oom.group=0 and |
| +// holds no process (spec §7.3, plan 2 D4). Spawning through spawnInCgroup |
| +// instead writes the tier and memory.oom.group=1 onto the parent and makes |
| +// no role-sidecar child. |
| +func TestStartSidecarGenKeepsTheRoleSidecarChild(t *testing.T) { |
| + const host = "sc.example.com" |
| + b, parent := delegatedBackend(t, host) |
| + b.GenContainment = true |
| + v2 := filepath.Join(b.OptDir, "versions", "v2", "app") |
| + if err := os.MkdirAll(filepath.Dir(v2), 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.WriteFile(v2, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.WriteFile(filepath.Join(b.DataDir, host+"-sidecar.env"), []byte("X=1\n"), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + rt := registry.Route{Host: host, Kind: "instance", DBPath: provision.DefaultDBPath(b.DataDir, host), |
| + Socket: filepath.Join(provision.SocketDir, host+".sock")} |
| + p, err := b.StartSidecarGen(context.Background(), rt, Gen{N: 3, Build: "v2", StagedPath: v2}) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + _ = p.Stop(context.Background()) |
| + if fi, err := os.Stat(filepath.Join(parent, "role-sidecar")); err != nil || !fi.IsDir() { |
| + t.Fatalf("no role-sidecar child: %v", err) |
| + } |
| + if _, err := os.Stat(filepath.Join(parent, "role-sidecar", "memory.max")); !os.IsNotExist(err) { |
| + t.Errorf("the sidecar's child got a limit of its own: %v", err) |
| + } |
| + if got, err := os.ReadFile(filepath.Join(parent, "memory.oom.group")); err != nil || string(got) != "0" { |
| + t.Errorf("parent memory.oom.group = %q, %v; want 0", got, err) |
| + } |
| +} |
| + |
| +// ExecBackend: a flat sidecar execs the generation's staged path. A nested |
| +// sidecar is refused until Task 9 hands root the staged executable: it never |
| +// reads the live link to decide, because a repoint can land between that |
| +// read and the start. |
| +func TestExecBackendStartSidecarGen(t *testing.T) { |
| + stage := func(t *testing.T, b *ExecBackend, host string) (v2, marker string) { |
| + t.Helper() |
| + marker = filepath.Join(t.TempDir(), "argv0") |
| + v2 = filepath.Join(b.OptDir, "versions", "v2", "app") |
| + if err := os.MkdirAll(filepath.Dir(v2), 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.WriteFile(v2, []byte("#!/bin/sh\nprintf '%s' \"$0\" > '"+marker+"'\n"), 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.WriteFile(filepath.Join(b.DataDir, host+"-sidecar.env"), []byte("X=1\n"), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + return v2, marker |
| + } |
| + t.Run("flat", func(t *testing.T) { |
| + b := testExecBackend(t) |
| + host := "sc.example.com" |
| + stageLive(t, b, host, "v1") |
| + v2, marker := stage(t, b, host) |
| + // The default db and socket: Classify puts this route in the flat layout. |
| + rt := registry.Route{Host: host, Kind: "instance", DBPath: provision.DefaultDBPath(b.DataDir, host), |
| + Socket: filepath.Join(provision.SocketDir, host+".sock")} |
| + if l, why := b.Layout(rt); l != LayoutFlat { |
| + t.Fatalf("route classified %v (%s), want the flat layout", l, why) |
| + } |
| + p, err := b.StartSidecarGen(context.Background(), rt, Gen{N: 3, Build: "v2", StagedPath: v2}) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + defer p.Stop(context.Background()) //nolint:errcheck |
| + if got := waitForFile(t, marker, 2*time.Second); got != v2 { |
| + t.Fatalf("the sidecar execd %q, want the generation's %q", got, v2) |
| + } |
| + }) |
| + t.Run("foreign", func(t *testing.T) { |
| + b := testExecBackend(t) |
| + host := "fsc.example.com" |
| + stageLive(t, b, host, "v1") |
| + v2, marker := stage(t, b, host) |
| + rt := registry.Route{Host: host, Kind: "instance", Socket: filepath.Join(t.TempDir(), "s.sock")} |
| + if l, _ := b.Layout(rt); l != LayoutForeign { |
| + t.Fatalf("route classified %v, want the foreign layout", l) |
| + } |
| + p, err := b.StartSidecarGen(context.Background(), rt, Gen{N: 3, Build: "v2", StagedPath: v2}) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + defer p.Stop(context.Background()) //nolint:errcheck |
| + if got := waitForFile(t, marker, 2*time.Second); got != v2 { |
| + t.Fatalf("the sidecar execd %q, want the generation's %q", got, v2) |
| + } |
| + }) |
| + t.Run("nested", func(t *testing.T) { |
| + b := testExecBackend(t) |
| + rd := t.TempDir() |
| + b.RunDirOverride = rd |
| + host := "nsc.example.com" |
| + stageLive(t, b, host, "v1") |
| + rt := registry.Route{Host: host, Kind: "instance", DBPath: provision.NestedDBPath(b.DataDir, host), |
| + Socket: filepath.Join(rd, "instances", provision.InstanceID(host), "app.sock")} |
| + for _, build := range []string{"v1", "v2"} { // the live build and another |
| + if _, err := b.StartSidecarGen(context.Background(), rt, Gen{N: 3, Build: build}); !errors.Is(err, errNestedSidecarGen) { |
| + t.Fatalf("serving %s: %v, want errNestedSidecarGen until Task 9", build, err) |
| + } |
| + } |
| + }) |
| +} |
| + |
| +// KillSidecar: the nested arm asks the sidecar unit's stop, and with no |
| +// Control wired it answers an error, never a nil dereference. Flat without |
| +// containment has no cgroup of its own to prove: errNoGenContainment. |
| +func TestExecBackendKillSidecar(t *testing.T) { |
| + t.Run("nested without a control channel", func(t *testing.T) { |
| + b := testExecBackend(t) |
| + rd := t.TempDir() |
| + b.RunDirOverride = rd |
| + host := "nsc.example.com" |
| + rt := registry.Route{Host: host, Kind: "instance", DBPath: provision.NestedDBPath(b.DataDir, host), |
| + Socket: filepath.Join(rd, "instances", provision.InstanceID(host), "app.sock")} |
| + if err := b.KillSidecar(context.Background(), rt); err == nil { |
| + t.Fatal("KillSidecar with no Control proved a nested sidecar gone") |
| + } |
| + }) |
| + t.Run("flat without containment", func(t *testing.T) { |
| + b := testExecBackend(t) |
| + host := "sc.example.com" |
| + rt := registry.Route{Host: host, Kind: "instance", DBPath: provision.DefaultDBPath(b.DataDir, host), |
| + Socket: filepath.Join(provision.SocketDir, host+".sock")} |
| + if err := b.KillSidecar(context.Background(), rt); !errors.Is(err, errNoGenContainment) { |
| + t.Fatalf("KillSidecar = %v, want errNoGenContainment", err) |
| + } |
| + }) |
| +} |
| + |
| +var _ SidecarGenStarter = (*SplitBackend)(nil) |
amadan