| 1 | package activator |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "os" |
| 8 | "path/filepath" |
| 9 | "strings" |
| 10 | "sync" |
| 11 | "testing" |
| 12 | "time" |
| 13 | |
| 14 | "github.com/carlosframework/platform/internal/cgroup" |
| 15 | "github.com/carlosframework/platform/internal/instancectl" |
| 16 | "github.com/carlosframework/platform/internal/provision" |
| 17 | "github.com/carlosframework/platform/internal/registry" |
| 18 | ) |
| 19 | |
| 20 | func TestPrepareAtAGenerationKeysItsFilesByGeneration(t *testing.T) { |
| 21 | ctl := &fakeControl{} |
| 22 | b, rt := nestedBackend(t, ctl, &fakeUnit{}) |
| 23 | b.GenContainment = true |
| 24 | p := b.pathsFor(rt) |
| 25 | if err := os.MkdirAll(p.runDir, 0o750); err != nil { |
| 26 | t.Fatal(err) |
| 27 | } |
| 28 | for _, name := range []string{"exit.run.aaa.json", "exit.gen.7.bbb.json", "exit.gen.8.ccc.json"} { |
| 29 | if err := os.WriteFile(filepath.Join(p.runDir, name), []byte("{}"), 0o600); err != nil { |
| 30 | t.Fatal(err) |
| 31 | } |
| 32 | } |
| 33 | p2, ctlL, logs, _, err := b.prepareAt(rt, instancectl.RoleRun, nestedGenSocketIn(p.runDir, 7), 7) |
| 34 | if err != nil { |
| 35 | t.Fatal(err) |
| 36 | } |
| 37 | defer logs.Close() |
| 38 | if ctlL != nil { |
| 39 | defer ctlL.Close() |
| 40 | } |
| 41 | for name, want := range map[string]bool{"exit.run.aaa.json": true, "exit.gen.7.bbb.json": false, "exit.gen.8.ccc.json": true} { |
| 42 | if _, err := os.Stat(filepath.Join(p.runDir, name)); (err == nil) != want { |
| 43 | t.Errorf("%s present=%v, want %v", name, err == nil, want) |
| 44 | } |
| 45 | } |
| 46 | wantSock := filepath.Join(b.RunDirOverride, "logs", provision.InstanceID(rt.Host)+"@7.run.sock") |
| 47 | if got := b.logSocketFor(rt.Host, instancectl.RoleRun, 7); got != wantSock { |
| 48 | t.Errorf("log socket = %q, want %q", got, wantSock) |
| 49 | } |
| 50 | if _, err := os.Stat(wantSock); err != nil { |
| 51 | t.Errorf("generation log listener not bound: %v", err) |
| 52 | } |
| 53 | if err := b.writeLaunchFor(p2, instancectl.RoleRun, 7, rt.Host, []string{"/opt/carlos/live/" + rt.Host}, nil); err != nil { |
| 54 | t.Fatal(err) |
| 55 | } |
| 56 | if _, err := os.Stat(filepath.Join(p2.dataDir, "launch.gen.7.json")); err != nil { |
| 57 | t.Errorf("launch.gen.7.json: %v", err) |
| 58 | } |
| 59 | if _, err := os.Stat(filepath.Join(p2.dataDir, "launch.run.json")); !os.IsNotExist(err) { |
| 60 | t.Errorf("a generation wrote the legacy launch file: %v", err) |
| 61 | } |
| 62 | } |
| 63 | |
| 64 | func TestGenerationUnitProcStopsAndWatchesItsOwnUnit(t *testing.T) { |
| 65 | ctl := &fakeControl{} |
| 66 | unit := &fakeUnit{} |
| 67 | b, rt := nestedBackend(t, ctl, unit) |
| 68 | b.GenContainment = true |
| 69 | b.Control = fakeGenControl{ctl} |
| 70 | var mu sync.Mutex |
| 71 | var seen []string |
| 72 | b.UnitState = func(ctx context.Context, name string) (string, string, error) { |
| 73 | mu.Lock() |
| 74 | seen = append(seen, name) |
| 75 | mu.Unlock() |
| 76 | return unit.state(ctx, name) |
| 77 | } |
| 78 | unit.set("active", "inv7", true, true) |
| 79 | ctl.stopHook = func(string, instancectl.Role) { unit.set("inactive", "inv7", false, false) } |
| 80 | up := b.newUnitProcFor(rt, instancectl.RoleRun, 7, b.pathsFor(rt), "inv7", nil, nil) |
| 81 | ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) |
| 82 | defer cancel() |
| 83 | if err := up.Stop(ctx); err != nil { |
| 84 | t.Fatal(err) |
| 85 | } |
| 86 | if got := ctl.joined(); !strings.Contains(got, "stopgen h.example.com 7 kill=false") || strings.Contains(got, "stop h.example.com run") { |
| 87 | t.Errorf("calls =\n%s", got) |
| 88 | } |
| 89 | want := instancectl.GenUnitName(rt.Host, 7) |
| 90 | if up.unit != want { |
| 91 | t.Errorf("unit = %q, want %q", up.unit, want) |
| 92 | } |
| 93 | mu.Lock() |
| 94 | defer mu.Unlock() |
| 95 | if len(seen) == 0 { |
| 96 | t.Fatal("the observer never asked about the unit") |
| 97 | } |
| 98 | for _, name := range seen { |
| 99 | if name != want { |
| 100 | t.Errorf("the observer asked about %s, want %s", name, want) |
| 101 | } |
| 102 | } |
| 103 | } |
| 104 | |
| 105 | func TestGenerationUnitProcConsumesOnlyItsOwnRecordAndLaunchFile(t *testing.T) { |
| 106 | t.Parallel() |
| 107 | ctl := &fakeControl{} |
| 108 | unit := &fakeUnit{} |
| 109 | b, rt := nestedBackend(t, ctl, unit) |
| 110 | b.GenContainment = true |
| 111 | b.Control = fakeGenControl{ctl} |
| 112 | p := b.pathsFor(rt) |
| 113 | for _, d := range []string{p.runDir, p.dataDir} { |
| 114 | if err := os.MkdirAll(d, 0o750); err != nil { |
| 115 | t.Fatal(err) |
| 116 | } |
| 117 | } |
| 118 | rec, _ := json.Marshal(instancectl.ExitRecord{Invocation: "inv7", Result: "success", Code: "exited", Status: "0"}) |
| 119 | for dir, names := range map[string][]string{ |
| 120 | p.runDir: {"exit.gen.7.inv7.json", "exit.run.inv7.json"}, |
| 121 | p.dataDir: {"launch.gen.7.json", "launch.gen.8.json"}, |
| 122 | } { |
| 123 | for _, name := range names { |
| 124 | if err := os.WriteFile(filepath.Join(dir, name), rec, 0o600); err != nil { |
| 125 | t.Fatal(err) |
| 126 | } |
| 127 | } |
| 128 | } |
| 129 | unit.set("active", "inv7", true, true) |
| 130 | up := b.newUnitProcFor(rt, instancectl.RoleRun, 7, p, "inv7", nil, nil) |
| 131 | unit.set("inactive", "inv7", false, false) |
| 132 | waitFor(t, "the generation to exit", up.Exited) |
| 133 | for path, want := range map[string]bool{ |
| 134 | filepath.Join(p.runDir, "exit.gen.7.inv7.json"): false, |
| 135 | filepath.Join(p.runDir, "exit.run.inv7.json"): true, |
| 136 | filepath.Join(p.dataDir, "launch.gen.7.json"): false, |
| 137 | filepath.Join(p.dataDir, "launch.gen.8.json"): true, |
| 138 | } { |
| 139 | if _, err := os.Stat(path); (err == nil) != want { |
| 140 | t.Errorf("%s present=%v, want %v", path, err == nil, want) |
| 141 | } |
| 142 | } |
| 143 | } |
| 144 | |
| 145 | func TestStartRoleDispatchesByGeneration(t *testing.T) { |
| 146 | ctl := &fakeControl{inv: "inv"} |
| 147 | b, rt := nestedBackend(t, ctl, &fakeUnit{}) |
| 148 | b.Control = fakeGenControl{ctl} |
| 149 | if _, err := b.startRole(context.Background(), rt, instancectl.RoleRun, 0, nil, cgroup.Limit{}); err != nil { |
| 150 | t.Fatal(err) |
| 151 | } |
| 152 | if _, err := b.startRole(context.Background(), rt, instancectl.RoleRun, 7, nil, cgroup.Limit{}); err != nil { |
| 153 | t.Fatal(err) |
| 154 | } |
| 155 | if want := "start h.example.com run\nstartgen h.example.com 7 536870912"; ctl.joined() != want { |
| 156 | t.Errorf("calls =\n%s\nwant\n%s", ctl.joined(), want) |
| 157 | } |
| 158 | } |
| 159 | |
| 160 | func TestUnitGen(t *testing.T) { |
| 161 | b := testExecBackend(t) |
| 162 | if n, err := b.unitGen(7); n != 0 || err != nil { |
| 163 | t.Errorf("containment off: %d %v, want 0", n, err) |
| 164 | } |
| 165 | b.GenContainment = true |
| 166 | if n, err := b.unitGen(7); n != 7 || err != nil { |
| 167 | t.Errorf("containment on: %d %v, want 7", n, err) |
| 168 | } |
| 169 | if _, err := b.unitGen(0); err == nil { |
| 170 | t.Error("generation 0 accepted") |
| 171 | } |
| 172 | } |
| 173 | |
| 174 | // Flag off: a nested wake starts the original template, with the original |
| 175 | // launch file, log socket and unit name. |
| 176 | func TestStartUnitWithContainmentOffKeepsTheOriginalTemplate(t *testing.T) { |
| 177 | t.Parallel() |
| 178 | ctl := &fakeControl{inv: "inv1"} |
| 179 | unit := &fakeUnit{} |
| 180 | b, rt := nestedBackend(t, ctl, unit) |
| 181 | var stopServing func() |
| 182 | ctl.onStart = func() { stopServing = serveOK(t, rt.Socket) } |
| 183 | defer func() { |
| 184 | if stopServing != nil { |
| 185 | stopServing() |
| 186 | } |
| 187 | }() |
| 188 | unit.set("active", "inv1", true, true) |
| 189 | p, _, err := b.Start(context.Background(), rt) |
| 190 | if err != nil { |
| 191 | t.Fatalf("Start: %v", err) |
| 192 | } |
| 193 | up := p.(*unitProc) |
| 194 | if up.unit != "carlos-instance@h.example.com.service" || up.gen != 0 { |
| 195 | t.Errorf("unit = %q gen %d", up.unit, up.gen) |
| 196 | } |
| 197 | if strings.Contains(ctl.joined(), "startgen") { |
| 198 | t.Errorf("a flag-off wake used the generation verb:\n%s", ctl.joined()) |
| 199 | } |
| 200 | if _, err := os.Stat(b.logSocket(rt.Host, instancectl.RoleRun)); err != nil { |
| 201 | t.Errorf("legacy log socket: %v", err) |
| 202 | } |
| 203 | unit.set("inactive", "inv1", false, false) |
| 204 | waitFor(t, "the unit to exit", p.Exited) |
| 205 | } |
| 206 | |
| 207 | // genStartFixture is a nested backend with containment on, ready for a |
| 208 | // generation-7 start whose answer the test chooses. |
| 209 | func genStartFixture(t *testing.T, ctl *fakeControl) (*ExecBackend, registry.Route, string, string) { |
| 210 | t.Helper() |
| 211 | b, rt := nestedBackend(t, ctl, &fakeUnit{}) |
| 212 | b.GenContainment = true |
| 213 | b.Control = fakeGenControl{ctl} |
| 214 | launch := filepath.Join(b.pathsFor(rt).dataDir, instancectl.GenLaunchFileName(7)) |
| 215 | return b, rt, launch, b.logSocketFor(rt.Host, instancectl.RoleRun, 7) |
| 216 | } |
| 217 | |
| 218 | // A generation start that failed may have started the unit (its answer |
| 219 | // was lost): the backend kills it through StopGen on its own budget, and |
| 220 | // only then closes its listeners and removes its launch file. |
| 221 | func TestStartGenFailureKillsTheGenerationBeforeReleasingIt(t *testing.T) { |
| 222 | ctl := &fakeControl{startErr: errors.New("instance control /start: 502: show: bus timeout")} |
| 223 | b, rt, launch, logSock := genStartFixture(t, ctl) |
| 224 | var launchAtStop bool |
| 225 | ctl.stopHook = func(string, instancectl.Role) { _, err := os.Stat(launch); launchAtStop = err == nil } |
| 226 | _, _, err := b.StartGen(context.Background(), rt, 7, false) |
| 227 | if err == nil || errors.Is(err, errGenUnproven) { |
| 228 | t.Fatalf("StartGen = %v, want an ordinary start failure", err) |
| 229 | } |
| 230 | if want := "startgen h.example.com 7 536870912\nstopgen h.example.com 7 kill=true"; ctl.joined() != want { |
| 231 | t.Errorf("calls =\n%s\nwant\n%s", ctl.joined(), want) |
| 232 | } |
| 233 | if !launchAtStop { |
| 234 | t.Error("the launch file was gone before the kill") |
| 235 | } |
| 236 | if _, err := os.Stat(launch); !os.IsNotExist(err) { |
| 237 | t.Errorf("launch file left after a proven stop: %v", err) |
| 238 | } |
| 239 | if _, err := os.Stat(logSock); !os.IsNotExist(err) { |
| 240 | t.Errorf("log listener still bound after a proven stop: %v", err) |
| 241 | } |
| 242 | } |
| 243 | |
| 244 | // The kill that follows a failed start fails too: termination is |
| 245 | // unproven, so the generation's files stay and the error says so. |
| 246 | func TestStartGenFailureWithAnUnprovenKillKeepsTheGeneration(t *testing.T) { |
| 247 | ctl := &fakeControl{startErr: errors.New("instance control socket: context deadline exceeded"), stopGenErr: errors.New("instance control /stop: 502: stop: processes remain")} |
| 248 | b, rt, launch, logSock := genStartFixture(t, ctl) |
| 249 | _, _, err := b.StartGen(context.Background(), rt, 7, false) |
| 250 | if !errors.Is(err, errGenUnproven) { |
| 251 | t.Fatalf("StartGen = %v, want errGenUnproven", err) |
| 252 | } |
| 253 | if _, err := os.Stat(launch); err != nil { |
| 254 | t.Errorf("launch file removed while termination is unproven: %v", err) |
| 255 | } |
| 256 | if _, err := os.Stat(logSock); err != nil { |
| 257 | t.Errorf("log listener released while termination is unproven: %v", err) |
| 258 | } |
| 259 | } |
| 260 | |
| 261 | // A start that answered no invocation is just as uncertain for a |
| 262 | // generation: it is killed and proven gone before its files go. |
| 263 | func TestStartGenWithNoInvocationKillsTheGeneration(t *testing.T) { |
| 264 | ctl := &fakeControl{} |
| 265 | b, rt, launch, _ := genStartFixture(t, ctl) |
| 266 | if _, _, err := b.StartGen(context.Background(), rt, 7, false); err == nil { |
| 267 | t.Fatal("a start with no invocation succeeded") |
| 268 | } |
| 269 | if !strings.Contains(ctl.joined(), "stopgen h.example.com 7 kill=true") { |
| 270 | t.Errorf("calls =\n%s", ctl.joined()) |
| 271 | } |
| 272 | if _, err := os.Stat(launch); !os.IsNotExist(err) { |
| 273 | t.Errorf("launch file left after a proven stop: %v", err) |
| 274 | } |
| 275 | } |
| 276 | |
| 277 | // genProcFixture is a generation-7 unitProc over fakes, with its launch |
| 278 | // file on disk. |
| 279 | func genProcFixture(t *testing.T) (*fakeControl, *fakeUnit, *ExecBackend, registry.Route, string) { |
| 280 | t.Helper() |
| 281 | ctl, unit := &fakeControl{}, &fakeUnit{} |
| 282 | b, rt := nestedBackend(t, ctl, unit) |
| 283 | b.GenContainment = true |
| 284 | b.Control = fakeGenControl{ctl} |
| 285 | p := b.pathsFor(rt) |
| 286 | if err := os.MkdirAll(p.dataDir, 0o750); err != nil { |
| 287 | t.Fatal(err) |
| 288 | } |
| 289 | launch := filepath.Join(p.dataDir, instancectl.GenLaunchFileName(7)) |
| 290 | if err := os.WriteFile(launch, []byte("{}"), 0o600); err != nil { |
| 291 | t.Fatal(err) |
| 292 | } |
| 293 | return ctl, unit, b, rt, launch |
| 294 | } |
| 295 | |
| 296 | // A generation that exits on its own is finalised once observe proves it |
| 297 | // gone: its launch file here, and the control service's drop-in and exe |
| 298 | // layout through a stop of the inactive unit. A later Stop sends nothing. |
| 299 | func TestGenerationNaturalExitFinalisesItsFiles(t *testing.T) { |
| 300 | t.Parallel() |
| 301 | ctl, unit, b, rt, launch := genProcFixture(t) |
| 302 | unit.set("active", "inv7", true, true) |
| 303 | up := b.newUnitProcFor(rt, instancectl.RoleRun, 7, b.pathsFor(rt), "inv7", nil, nil) |
| 304 | unit.set("inactive", "inv7", false, false) |
| 305 | waitFor(t, "the generation to exit", up.Exited) |
| 306 | if want := "stopgen h.example.com 7 kill=false"; ctl.joined() != want { |
| 307 | t.Errorf("calls =\n%s\nwant\n%s", ctl.joined(), want) |
| 308 | } |
| 309 | if _, err := os.Stat(launch); !os.IsNotExist(err) { |
| 310 | t.Errorf("launch file left after a natural exit: %v", err) |
| 311 | } |
| 312 | if err := up.Stop(context.Background()); err != nil { |
| 313 | t.Fatal(err) |
| 314 | } |
| 315 | if want := "stopgen h.example.com 7 kill=false"; ctl.joined() != want { |
| 316 | t.Errorf("a stop after the exit sent more:\n%s", ctl.joined()) |
| 317 | } |
| 318 | } |
| 319 | |
| 320 | // A generation restarted outside the activator is stopped, and once that |
| 321 | // is proven its files go too. The stop that proved it already removed the |
| 322 | // control service's files, so it is not asked twice. |
| 323 | func TestGenerationForeignInvocationExitFinalisesItsFiles(t *testing.T) { |
| 324 | t.Parallel() |
| 325 | ctl, unit, b, rt, launch := genProcFixture(t) |
| 326 | ctl.stopHook = func(string, instancectl.Role) { unit.set("inactive", "", false, false) } |
| 327 | unit.set("active", "inv7", true, true) |
| 328 | up := b.newUnitProcFor(rt, instancectl.RoleRun, 7, b.pathsFor(rt), "inv7", nil, nil) |
| 329 | unit.set("active", "someone-else", true, true) |
| 330 | waitFor(t, "the generation to exit", up.Exited) |
| 331 | if want := "stopgen h.example.com 7 kill=false"; ctl.joined() != want { |
| 332 | t.Errorf("calls =\n%s\nwant\n%s", ctl.joined(), want) |
| 333 | } |
| 334 | if _, err := os.Stat(launch); !os.IsNotExist(err) { |
| 335 | t.Errorf("launch file left after a foreign invocation was stopped: %v", err) |
| 336 | } |
| 337 | } |
| 338 | |
| 339 | // With no generation channel StartGen is never sent, so nothing ran: the |
| 340 | // launch file written for it goes. |
| 341 | func TestStartGenRefusedBeforeSendingRemovesTheLaunchFile(t *testing.T) { |
| 342 | ctl := &fakeControl{inv: "inv7"} |
| 343 | b, rt, launch, _ := genStartFixture(t, ctl) |
| 344 | b.Control = ctl // a Control without the generation verbs |
| 345 | if _, _, err := b.StartGen(context.Background(), rt, 7, false); !errors.Is(err, errNoGenControl) { |
| 346 | t.Fatalf("StartGen = %v, want errNoGenControl", err) |
| 347 | } |
| 348 | if _, err := os.Stat(launch); !os.IsNotExist(err) { |
| 349 | t.Errorf("launch file left by a start never sent: %v", err) |
| 350 | } |
| 351 | } |
| 352 | |
amadan