| 1 | package activator |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "path/filepath" |
| 7 | "sync" |
| 8 | "testing" |
| 9 | "time" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/provision" |
| 12 | "github.com/carlosframework/platform/internal/registry" |
| 13 | ) |
| 14 | |
| 15 | func TestWorkerGenerations(t *testing.T) { |
| 16 | for _, on := range []bool{true, false} { |
| 17 | a, b, _, _ := testActivator(t, time.Hour) |
| 18 | a.gens = on |
| 19 | old := sidecarSettle |
| 20 | sidecarSettle = 20 * time.Millisecond |
| 21 | t.Cleanup(func() { sidecarSettle = old }) |
| 22 | rt := testWorkerRoute("bot.example.com") |
| 23 | if err := a.WakeWorker(rt); err != nil { |
| 24 | t.Fatal(err) |
| 25 | } |
| 26 | first, members := a.genStateForTest(rt.Host) |
| 27 | if on != (first.N != 0) || first.Socket != "" || len(members) != 0 { |
| 28 | t.Fatalf("gens %v: worker generation %+v, members %v", on, first, members) |
| 29 | } |
| 30 | b.mu.Lock() |
| 31 | b.workerStartDelay = 200 * time.Millisecond // hold the respawn open |
| 32 | b.mu.Unlock() |
| 33 | b.lastWorker().crash() |
| 34 | a.sweepOnce() // a crashed hibernating worker respawns in place |
| 35 | waitFor(t, "the respawn to begin", func() bool { return b.workerEnters() == 2 }) |
| 36 | _, members = a.genStateForTest(rt.Host) |
| 37 | if on != (len(members) == 1) { |
| 38 | t.Fatalf("gens %v: members during the respawn = %v", on, members) |
| 39 | } |
| 40 | waitFor(t, "the respawn to finish", func() bool { return b.workerStarts() == 2 }) |
| 41 | waitFor(t, "the member to leave", func() bool { _, m := a.genStateForTest(rt.Host); return len(m) == 0 }) |
| 42 | if on { |
| 43 | if second, _ := a.genStateForTest(rt.Host); second.N <= first.N || !second.Worker { |
| 44 | t.Fatalf("respawned worker generation %+v, want a Worker above %d", second, first.N) |
| 45 | } |
| 46 | } |
| 47 | } |
| 48 | } |
| 49 | |
| 50 | // StartWorker spawns every worker into the flat host cgroup, whatever the |
| 51 | // row's layout, so KillHost proves that cgroup for a nested-looking row too. |
| 52 | // Here the cgroup is a fixture, not cgroup v2, so the proof is refused. |
| 53 | // Answering nil for a nested layout skipped the proof for a worker that |
| 54 | // runs in carlos-exec@<host> all the same. |
| 55 | func TestKillHostProvesANestedLayoutWorkersHostCgroup(t *testing.T) { |
| 56 | host := "nw.example.com" |
| 57 | b, _ := delegatedBackend(t, host) |
| 58 | rd := t.TempDir() |
| 59 | b.RunDirOverride = rd |
| 60 | rt := registry.Route{Host: host, Kind: "sidecar", DBPath: provision.NestedDBPath(b.DataDir, host), |
| 61 | Socket: filepath.Join(rd, "instances", provision.InstanceID(host), "app.sock")} |
| 62 | if err := b.KillHost(context.Background(), rt); err == nil { |
| 63 | t.Fatal("KillHost answered a nested-layout worker's host cgroup as proven without proving it") |
| 64 | } |
| 65 | } |
| 66 | |
| 67 | // hostKillBackend is fakeBackend plus KillHost, failing while bad. |
| 68 | type hostKillBackend struct { |
| 69 | *fakeBackend |
| 70 | hmu sync.Mutex |
| 71 | bad bool |
| 72 | kills int |
| 73 | } |
| 74 | |
| 75 | func (h *hostKillBackend) KillHost(context.Context, registry.Route) error { |
| 76 | h.hmu.Lock() |
| 77 | defer h.hmu.Unlock() |
| 78 | h.kills++ |
| 79 | if h.bad { |
| 80 | return errors.New("the host cgroup still holds a process") |
| 81 | } |
| 82 | return nil |
| 83 | } |
| 84 | |
| 85 | func (h *hostKillBackend) killCount() int { |
| 86 | h.hmu.Lock() |
| 87 | defer h.hmu.Unlock() |
| 88 | return h.kills |
| 89 | } |
| 90 | |
| 91 | func (h *hostKillBackend) heal() { |
| 92 | h.hmu.Lock() |
| 93 | h.bad = false |
| 94 | h.hmu.Unlock() |
| 95 | } |
| 96 | |
| 97 | // Spec §7.4, §7.9: a worker runs in its host cgroup (plan 2 D4), so that is |
| 98 | // what proves it terminated, not a g<n> it never ran in (whose absence |
| 99 | // KillGen counts as empty). While the host cgroup still holds a process |
| 100 | // (a descendant the worker forked), a terminal stop keeps the lease and a |
| 101 | // respawn execs nothing beside it. |
| 102 | func TestWorkerTerminationProvesItsHostCgroup(t *testing.T) { |
| 103 | t.Run("a terminal stop keeps the lease", func(t *testing.T) { |
| 104 | a, b, _, _ := testActivator(t, time.Hour) |
| 105 | a.gens = true |
| 106 | hk := &hostKillBackend{fakeBackend: b, bad: true} |
| 107 | a.backend = hk |
| 108 | rt := testWorkerRoute("bot.example.com") |
| 109 | if err := a.WakeWorker(rt); err != nil { |
| 110 | t.Fatal(err) |
| 111 | } |
| 112 | rep := b.lastRep() |
| 113 | a.Bounce(rt.Host) |
| 114 | // The stop pends only once it has decided the lease stays: its proof |
| 115 | // (KillHost) ran and failed, and the release was not made. |
| 116 | waitFor(t, "the stop to pend", func() bool { |
| 117 | a.mu.Lock() |
| 118 | defer a.mu.Unlock() |
| 119 | e := a.entries[rt.Host] |
| 120 | return e != nil && e.pendingStop |
| 121 | }) |
| 122 | if n := hk.killCount(); n == 0 { |
| 123 | t.Fatal("the stop pended without asking the host cgroup's proof") |
| 124 | } |
| 125 | if rep.isClosed() { |
| 126 | t.Fatal("the lease was released while the worker's host cgroup still held a process") |
| 127 | } |
| 128 | hk.heal() |
| 129 | a.sweepOnce() |
| 130 | waitFor(t, "the lease release", rep.isClosed) |
| 131 | }) |
| 132 | t.Run("a respawn with no old handle still proves the host cgroup", func(t *testing.T) { |
| 133 | a, b, _, _ := testActivator(t, time.Hour) |
| 134 | a.gens = true |
| 135 | hk := &hostKillBackend{fakeBackend: b} |
| 136 | a.backend = hk |
| 137 | rt := testWorkerRoute("bot3.example.com") |
| 138 | if err := a.WakeWorker(rt); err != nil { |
| 139 | t.Fatal(err) |
| 140 | } |
| 141 | hk.hmu.Lock() |
| 142 | hk.bad = true |
| 143 | hk.hmu.Unlock() |
| 144 | a.mu.Lock() |
| 145 | // main's signature; once plan 4 Task 9 has merged, pass nil as its |
| 146 | // fourth (onto) argument. |
| 147 | a.respawnWorkerLocked(a.entries[rt.Host], nil, "test: a restart with no old handle", nil) |
| 148 | a.mu.Unlock() |
| 149 | waitFor(t, "the respawn to finish", func() bool { return !a.entryRespawning(rt.Host) }) |
| 150 | if got := b.workerStarts(); got != 1 { |
| 151 | t.Fatalf("worker starts = %d: a nil-stopOld respawn skipped the host cgroup proof", got) |
| 152 | } |
| 153 | }) |
| 154 | t.Run("a failed worker wake keeps its lease", func(t *testing.T) { |
| 155 | old := sidecarSettle |
| 156 | sidecarSettle = 20 * time.Millisecond |
| 157 | t.Cleanup(func() { sidecarSettle = old }) |
| 158 | a, b, _, _ := testActivator(t, time.Hour) |
| 159 | a.gens = true |
| 160 | hk := &hostKillBackend{fakeBackend: b, bad: true} |
| 161 | a.backend = hk |
| 162 | b.mu.Lock() |
| 163 | b.workerDeadOnStart = true // the wake fails its settle |
| 164 | b.mu.Unlock() |
| 165 | if err := a.WakeWorker(testWorkerRoute("bot4.example.com")); err == nil { |
| 166 | t.Fatal("a worker dead on start woke") |
| 167 | } |
| 168 | // The lease goes back by Release, or by closing the replica when the |
| 169 | // wake had one (cleanupFailedWake). |
| 170 | given := func() bool { |
| 171 | b.mu.Lock() |
| 172 | released, reps := b.releases, append([]*fakeRep(nil), b.reps...) |
| 173 | b.mu.Unlock() |
| 174 | if released > 0 { |
| 175 | return true |
| 176 | } |
| 177 | for _, r := range reps { |
| 178 | if r.isClosed() { |
| 179 | return true |
| 180 | } |
| 181 | } |
| 182 | return false |
| 183 | } |
| 184 | if given() { |
| 185 | t.Fatal("a failed worker wake gave its lease back over an unproven host cgroup") |
| 186 | } |
| 187 | hk.heal() |
| 188 | a.sweepOnce() |
| 189 | waitFor(t, "the lease given back", given) |
| 190 | }) |
| 191 | t.Run("a respawn execs nothing beside the old child", func(t *testing.T) { |
| 192 | old := sidecarSettle |
| 193 | sidecarSettle = 20 * time.Millisecond |
| 194 | t.Cleanup(func() { sidecarSettle = old }) |
| 195 | a, b, clk, _ := testActivator(t, time.Hour) |
| 196 | a.gens = true |
| 197 | hk := &hostKillBackend{fakeBackend: b} |
| 198 | a.backend = hk |
| 199 | rt := testWorkerRoute("bot2.example.com") |
| 200 | if err := a.WakeWorker(rt); err != nil { |
| 201 | t.Fatal(err) |
| 202 | } |
| 203 | hk.hmu.Lock() |
| 204 | hk.bad = true |
| 205 | hk.hmu.Unlock() |
| 206 | b.lastWorker().crash() |
| 207 | kills := hk.killCount() |
| 208 | a.sweepOnce() // the respawn retires the crashed child, and its proof fails |
| 209 | waitFor(t, "the respawn to finish", func() bool { return !a.entryRespawning(rt.Host) }) |
| 210 | if got := hk.killCount(); got <= kills { |
| 211 | t.Fatalf("host cgroup kills = %d (was %d): the respawn never asked the old child's proof", got, kills) |
| 212 | } |
| 213 | if got := b.workerStarts(); got != 1 { |
| 214 | t.Fatalf("worker starts = %d: a respawn exec'd beside an unproven old child", got) |
| 215 | } |
| 216 | // Positive control: once the proof succeeds, the next sweep's |
| 217 | // crash restart respawns the worker. |
| 218 | hk.heal() |
| 219 | clk.advance(sidecarRestartMax + time.Second) // past the crash restart's backoff |
| 220 | a.sweepOnce() |
| 221 | waitFor(t, "the respawn after the proof healed", func() bool { return b.workerStarts() == 2 && !a.entryRespawning(rt.Host) }) |
| 222 | }) |
| 223 | } |
| 224 | |
| 225 | // Index P5/P6: every exec asks the gate's ownership check, whatever the |
| 226 | // switch says. A respawn whose lease has been lost execs nothing, with |
| 227 | // generations on or off. |
| 228 | func TestWorkerRespawnAsksOwnershipAtItsExec(t *testing.T) { |
| 229 | t.Parallel() |
| 230 | for _, on := range []bool{true, false} { |
| 231 | a, b, _, _ := testActivator(t, time.Hour) |
| 232 | a.gens = on |
| 233 | rt := testWorkerRoute("bot5.example.com") |
| 234 | if err := a.WakeWorker(rt); err != nil { |
| 235 | t.Fatal(err) |
| 236 | } |
| 237 | b.lastRep().setErr(errors.New("renewal failed")) |
| 238 | a.mu.Lock() |
| 239 | a.respawnWorkerLocked(a.entries[rt.Host], nil, "test: the lease is lost", nil) |
| 240 | a.mu.Unlock() |
| 241 | waitFor(t, "the respawn to finish", func() bool { return !a.entryRespawning(rt.Host) }) |
| 242 | if got := b.workerStarts(); got != 1 { |
| 243 | t.Fatalf("gens %v: worker starts = %d: a respawn exec'd without the lease", on, got) |
| 244 | } |
| 245 | if _, members := a.genStateForTest(rt.Host); len(members) != 0 { |
| 246 | t.Fatalf("gens %v: members after the refused respawn = %v", on, members) |
| 247 | } |
| 248 | } |
| 249 | } |
| 250 | |
amadan