| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "errors" |
| 5 | "io/fs" |
| 6 | "net" |
| 7 | "net/http" |
| 8 | "net/http/httptest" |
| 9 | "os" |
| 10 | "path/filepath" |
| 11 | "strconv" |
| 12 | "strings" |
| 13 | "syscall" |
| 14 | "testing" |
| 15 | "time" |
| 16 | |
| 17 | "github.com/carlosframework/platform/internal/registry" |
| 18 | "github.com/carlosframework/platform/internal/release" |
| 19 | ) |
| 20 | |
| 21 | type fakeUnits struct { |
| 22 | shows []unitProps // returned in order; the last repeats |
| 23 | n int |
| 24 | exe map[int]string |
| 25 | exeErr map[int]error // optional: errors to return from Exe |
| 26 | exeSeq []string // optional: Exe's answers in order, whatever the pid; the last repeats |
| 27 | exeN int |
| 28 | showErr map[int]error // optional: the Nth Show call (0-based) fails with this |
| 29 | journal string |
| 30 | restarts []string |
| 31 | } |
| 32 | |
| 33 | func (f *fakeUnits) Show(string) (unitProps, error) { |
| 34 | if err, ok := f.showErr[f.n]; ok { |
| 35 | f.n++ |
| 36 | return unitProps{}, err |
| 37 | } |
| 38 | p := f.shows[min(f.n, len(f.shows)-1)] |
| 39 | f.n++ |
| 40 | return p, nil |
| 41 | } |
| 42 | func (f *fakeUnits) TryRestart(u string) error { |
| 43 | f.restarts = append(f.restarts, "try-restart "+u) |
| 44 | return nil |
| 45 | } |
| 46 | func (f *fakeUnits) Restart(u string) error { |
| 47 | f.restarts = append(f.restarts, "restart "+u) |
| 48 | return nil |
| 49 | } |
| 50 | func (f *fakeUnits) ResetFailed(u string) error { |
| 51 | f.restarts = append(f.restarts, "reset-failed "+u) |
| 52 | return nil |
| 53 | } |
| 54 | func (f *fakeUnits) JournalTail(string, int) string { return f.journal } |
| 55 | func (f *fakeUnits) StartTime(pid int) (string, error) { return strconv.Itoa(pid), nil } |
| 56 | func (f *fakeUnits) Exe(pid int) (string, error) { |
| 57 | if err, ok := f.exeErr[pid]; ok { |
| 58 | return "", err |
| 59 | } |
| 60 | if len(f.exeSeq) > 0 { |
| 61 | e := f.exeSeq[min(f.exeN, len(f.exeSeq)-1)] |
| 62 | f.exeN++ |
| 63 | return e, nil |
| 64 | } |
| 65 | if e, ok := f.exe[pid]; ok { |
| 66 | return e, nil |
| 67 | } |
| 68 | return "", errors.New("no such pid") |
| 69 | } |
| 70 | |
| 71 | // settleClock, not fakeClock: stage_relay_test.go already declares a |
| 72 | // fakeClock type in this package. |
| 73 | func settleClock() (func() time.Time, func(time.Duration)) { |
| 74 | t := time.Unix(0, 0) |
| 75 | return func() time.Time { return t }, func(d time.Duration) { t = t.Add(d) } |
| 76 | } |
| 77 | |
| 78 | func opts(ready func() error) settleOpts { |
| 79 | now, sleep := settleClock() |
| 80 | return settleOpts{Timeout: 30 * time.Second, Poll: 500 * time.Millisecond, Hold: 5 * time.Second, |
| 81 | WantDir: "/opt/carlos/versions/new", Ready: ready, Now: now, Sleep: sleep} |
| 82 | } |
| 83 | |
| 84 | func TestParseShow(t *testing.T) { |
| 85 | p := parseShow("ActiveState=active\nSubState=running\nInvocationID=abc\nMainPID=42\nNRestarts=0\n" + |
| 86 | "ExecStart={ path=/opt/carlos/live/platform.example.com ; argv[]=/opt/carlos/live/platform.example.com serve ; ignore_errors=no }\n") |
| 87 | if p.ActiveState != "active" || p.SubState != "running" || p.InvocationID != "abc" || p.MainPID != 42 || |
| 88 | p.ExecPath != "/opt/carlos/live/platform.example.com" { |
| 89 | t.Fatalf("parseShow = %+v", p) |
| 90 | } |
| 91 | } |
| 92 | |
| 93 | func TestSettleHealthyNewInvocation(t *testing.T) { |
| 94 | f := &fakeUnits{shows: []unitProps{ |
| 95 | {ActiveState: "activating", InvocationID: "new"}, |
| 96 | {ActiveState: "active", InvocationID: "new", MainPID: 7}, |
| 97 | }, exe: map[int]string{7: "/opt/carlos/versions/new/console"}} |
| 98 | if err := settleUnit(f, "console.service", "old", opts(func() error { return nil })); err != nil { |
| 99 | t.Fatalf("settle = %v, want healthy", err) |
| 100 | } |
| 101 | } |
| 102 | |
| 103 | // systemd resets NRestarts on a manual start: a healthy unit that had |
| 104 | // auto-restarted before must still pass (spec §2.3, codex 4). |
| 105 | func TestSettleIgnoresThePreviousInvocationsCounter(t *testing.T) { |
| 106 | f := &fakeUnits{shows: []unitProps{{ActiveState: "active", InvocationID: "new", MainPID: 7, NRestarts: 0}}, |
| 107 | exe: map[int]string{7: "/opt/carlos/versions/new/console"}} |
| 108 | if err := settleUnit(f, "console.service", "old", opts(func() error { return nil })); err != nil { |
| 109 | t.Fatalf("settle = %v", err) |
| 110 | } |
| 111 | } |
| 112 | |
| 113 | // Today's outage: the process exits in 55 ms and systemd cycles it. |
| 114 | func TestSettleCatchesARestartLoop(t *testing.T) { |
| 115 | f := &fakeUnits{shows: []unitProps{ |
| 116 | {ActiveState: "active", InvocationID: "new1", MainPID: 7}, |
| 117 | {ActiveState: "activating", InvocationID: "new2", NRestarts: 1}, |
| 118 | }, exe: map[int]string{7: "/opt/carlos/versions/new/console"}, journal: "usage: carlos <command> [flags]"} |
| 119 | err := settleUnit(f, "console.service", "old", opts(func() error { return errors.New("connection refused") })) |
| 120 | var nu *release.UnitNotUpError |
| 121 | if !errors.As(err, &nu) || !strings.Contains(nu.Journal, "usage: carlos") { |
| 122 | t.Fatalf("settle = %v, want UnitNotUpError carrying the journal", err) |
| 123 | } |
| 124 | } |
| 125 | |
| 126 | func TestSettleCatchesFailed(t *testing.T) { |
| 127 | f := &fakeUnits{shows: []unitProps{{ActiveState: "failed", InvocationID: "new"}}} |
| 128 | var nu *release.UnitNotUpError |
| 129 | if err := settleUnit(f, "console.service", "old", opts(nil)); !errors.As(err, &nu) { |
| 130 | t.Fatalf("settle = %v, want UnitNotUpError", err) |
| 131 | } |
| 132 | } |
| 133 | |
| 134 | func TestSettleDeadlineSaysNotReady(t *testing.T) { |
| 135 | f := &fakeUnits{shows: []unitProps{{ActiveState: "active", InvocationID: "new", MainPID: 7}}, |
| 136 | exe: map[int]string{7: "/opt/carlos/versions/new/console"}} |
| 137 | err := settleUnit(f, "console.service", "old", opts(func() error { return errors.New("dial: refused") })) |
| 138 | var nu *release.UnitNotUpError |
| 139 | if !errors.As(err, &nu) || !strings.Contains(nu.Reason, "not ready after 30s") { |
| 140 | t.Fatalf("settle = %v, want a not-ready deadline", err) |
| 141 | } |
| 142 | } |
| 143 | |
| 144 | func TestSettleRejectsTheOldBinary(t *testing.T) { |
| 145 | f := &fakeUnits{shows: []unitProps{{ActiveState: "active", InvocationID: "new", MainPID: 7}}, |
| 146 | exe: map[int]string{7: "/opt/carlos/versions/old/console"}} |
| 147 | var nu *release.UnitNotUpError |
| 148 | if err := settleUnit(f, "console.service", "old", opts(func() error { return nil })); !errors.As(err, &nu) || |
| 149 | !strings.Contains(nu.Reason, "running /opt/carlos/versions/old/console, not the new build in /opt/carlos/versions/new") { |
| 150 | t.Fatalf("settle = %v, want not-up at the deadline naming the old build's exe", err) |
| 151 | } |
| 152 | } |
| 153 | |
| 154 | // Final review F1: for Type=simple, systemd reports the new invocation active |
| 155 | // once it has forked and before it execs ExecStart, so the first poll can see |
| 156 | // systemd's executor as MainPID's exe. That is not-ready, not a failure. |
| 157 | func TestSettleWaitsOutTheExecWindow(t *testing.T) { |
| 158 | f := &fakeUnits{shows: []unitProps{{ActiveState: "active", InvocationID: "new", MainPID: 7}}, |
| 159 | exeSeq: []string{"/usr/lib/systemd/systemd-executor", "/opt/carlos/versions/new/console"}} |
| 160 | readies := 0 |
| 161 | err := settleUnit(f, "console.service", "old", opts(func() error { readies++; return nil })) |
| 162 | if err != nil { |
| 163 | t.Fatalf("settle = %v, want healthy once the new binary is exec'd", err) |
| 164 | } |
| 165 | if f.exeN != 2 || readies != 1 { |
| 166 | t.Fatalf("exe reads %d, ready calls %d; want 2 and 1 (no readiness judged on the executor)", f.exeN, readies) |
| 167 | } |
| 168 | } |
| 169 | |
| 170 | func TestSettleExecutorAtTheDeadlineNamesTheExe(t *testing.T) { |
| 171 | f := &fakeUnits{shows: []unitProps{{ActiveState: "active", InvocationID: "new", MainPID: 7}}, |
| 172 | exeSeq: []string{"/usr/lib/systemd/systemd-executor"}} |
| 173 | var nu *release.UnitNotUpError |
| 174 | err := settleUnit(f, "console.service", "old", opts(func() error { return nil })) |
| 175 | if !errors.As(err, &nu) || |
| 176 | !strings.Contains(nu.Reason, "running /usr/lib/systemd/systemd-executor, not the new build in /opt/carlos/versions/new") { |
| 177 | t.Fatalf("settle = %v, want a deadline UnitNotUpError naming the exe seen", err) |
| 178 | } |
| 179 | } |
| 180 | |
| 181 | // Final review F5: one failed `systemctl show` is retried, not a verdict. |
| 182 | func TestSettleRetriesATransientShowError(t *testing.T) { |
| 183 | f := &fakeUnits{shows: []unitProps{{ActiveState: "active", InvocationID: "new", MainPID: 7}}, |
| 184 | exe: map[int]string{7: "/opt/carlos/versions/new/console"}, |
| 185 | showErr: map[int]error{0: errors.New("Failed to get properties: Connection timed out")}} |
| 186 | if err := settleUnit(f, "console.service", "old", opts(func() error { return nil })); err != nil { |
| 187 | t.Fatalf("settle = %v, want healthy after one show error", err) |
| 188 | } |
| 189 | } |
| 190 | |
| 191 | func TestSettleShowErrorAtTheDeadline(t *testing.T) { |
| 192 | sys := &showFails{fakeUnits: &fakeUnits{}} |
| 193 | var nu *release.UnitNotUpError |
| 194 | err := settleUnit(sys, "console.service", "old", opts(func() error { return nil })) |
| 195 | if !errors.As(err, &nu) || !strings.Contains(nu.Reason, "cannot observe console.service: Access denied") { |
| 196 | t.Fatalf("settle = %v, want a deadline UnitNotUpError saying the unit could not be observed", err) |
| 197 | } |
| 198 | } |
| 199 | |
| 200 | // Final review F2: inactive/dead after the restart is an operator's stop. |
| 201 | func TestSettleStoppedUnitIsErrUnitStopped(t *testing.T) { |
| 202 | f := &fakeUnits{shows: []unitProps{ |
| 203 | {ActiveState: "active", InvocationID: "new", MainPID: 7}, |
| 204 | {ActiveState: "inactive", SubState: "dead", InvocationID: "new"}, |
| 205 | }, exe: map[int]string{7: "/opt/carlos/versions/new/console"}} |
| 206 | err := settleUnit(f, "console.service", "old", opts(func() error { return errors.New("refused") })) |
| 207 | if !errors.Is(err, errUnitStopped) { |
| 208 | t.Fatalf("settle = %v, want errUnitStopped", err) |
| 209 | } |
| 210 | } |
| 211 | |
| 212 | func TestSettleWithoutReadyHoldsFiveSeconds(t *testing.T) { |
| 213 | f := &fakeUnits{shows: []unitProps{{ActiveState: "active", InvocationID: "new", MainPID: 7}}, |
| 214 | exe: map[int]string{7: "/opt/carlos/versions/new/console"}} |
| 215 | o := opts(nil) |
| 216 | start := o.Now() |
| 217 | if err := settleUnit(f, "console.service", "old", o); err != nil { |
| 218 | t.Fatalf("settle = %v", err) |
| 219 | } |
| 220 | if got := o.Now().Sub(start); got < 5*time.Second { |
| 221 | t.Fatalf("settled after %v, want at least the 5s hold", got) |
| 222 | } |
| 223 | } |
| 224 | |
| 225 | func TestSettleSkipsAUnitThatWasNotRunning(t *testing.T) { |
| 226 | // beforeInvocation "" means the caller saw the unit inactive; settleUnit |
| 227 | // is not called at all in that case (Task 6). This pins the caller's |
| 228 | // contract from this side: an inactive unit that stays inactive after |
| 229 | // try-restart is "nothing started", not a failure. |
| 230 | f := &fakeUnits{shows: []unitProps{{ActiveState: "inactive"}}} |
| 231 | if err := settleUnit(f, "console.service", "", opts(nil)); err != nil { |
| 232 | t.Fatalf("settle on a stopped unit = %v, want nil", err) |
| 233 | } |
| 234 | } |
| 235 | |
| 236 | // Fix round 1, issue 1: NRestarts unchecked on the poll that fixes the new invocation. |
| 237 | func TestSettleCatchesRestartLoopOnFreshInvocation(t *testing.T) { |
| 238 | f := &fakeUnits{shows: []unitProps{ |
| 239 | {ActiveState: "active", InvocationID: "new", MainPID: 7, NRestarts: 1}, |
| 240 | }, exe: map[int]string{7: "/opt/carlos/versions/new/console"}} |
| 241 | var nu *release.UnitNotUpError |
| 242 | if err := settleUnit(f, "console.service", "old", opts(func() error { return nil })); !errors.As(err, &nu) { |
| 243 | t.Fatalf("settle = %v, want UnitNotUpError for restart loop on fresh invocation", err) |
| 244 | } |
| 245 | } |
| 246 | |
| 247 | // Fix round 1, issue 2a: Exe error (not permission) should be remembered and deadline should mention it. |
| 248 | func TestSettleExeErrorDeadline(t *testing.T) { |
| 249 | f := &fakeUnits{shows: []unitProps{ |
| 250 | {ActiveState: "active", InvocationID: "new", MainPID: 7}, |
| 251 | }, exe: map[int]string{}} |
| 252 | var nu *release.UnitNotUpError |
| 253 | if err := settleUnit(f, "console.service", "old", opts(nil)); !errors.As(err, &nu) || !strings.Contains(nu.Reason, "exe unreadable") { |
| 254 | t.Fatalf("settle = %v, want UnitNotUpError with 'exe unreadable' in reason", err) |
| 255 | } |
| 256 | } |
| 257 | |
| 258 | // Fix round 1, issue 2b: Permission error on Exe should be logged once and skipped, allowing settle to succeed. |
| 259 | func TestSettleExePermissionErrorSkipped(t *testing.T) { |
| 260 | f := &fakeUnits{ |
| 261 | shows: []unitProps{ |
| 262 | {ActiveState: "active", InvocationID: "new", MainPID: 7}, |
| 263 | }, |
| 264 | exeErr: map[int]error{7: &fs.PathError{Op: "readlink", Path: "/proc/7/exe", Err: syscall.EACCES}}, |
| 265 | } |
| 266 | if err := settleUnit(f, "console.service", "old", opts(nil)); err != nil { |
| 267 | t.Fatalf("settle = %v, want nil (permission error on Exe should be skipped)", err) |
| 268 | } |
| 269 | } |
| 270 | |
| 271 | func TestUnitSettleTimeoutFloor(t *testing.T) { |
| 272 | env := func(v string) func(string) string { return func(string) string { return v } } |
| 273 | for in, want := range map[string]time.Duration{"": 30 * time.Second, "1s": 5 * time.Second, |
| 274 | "45s": 45 * time.Second, "junk": 30 * time.Second} { |
| 275 | if got := unitSettleTimeout(env(in)); got != want { |
| 276 | t.Fatalf("CARLOS_UNIT_SETTLE=%q -> %v, want %v", in, got, want) |
| 277 | } |
| 278 | } |
| 279 | } |
| 280 | |
| 281 | func TestRouteReadyProbesTheUnixSocket(t *testing.T) { |
| 282 | dir, err := os.MkdirTemp("", "rr") |
| 283 | if err != nil { |
| 284 | t.Fatal(err) |
| 285 | } |
| 286 | t.Cleanup(func() { os.RemoveAll(dir) }) |
| 287 | sock := filepath.Join(dir, "s.sock") |
| 288 | ln, err := net.Listen("unix", sock) |
| 289 | if err != nil { |
| 290 | t.Fatal(err) |
| 291 | } |
| 292 | srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 293 | if r.URL.Path == "/healthz" && r.Host == "console.example.com" { |
| 294 | w.WriteHeader(200) |
| 295 | return |
| 296 | } |
| 297 | w.WriteHeader(500) |
| 298 | })} |
| 299 | go srv.Serve(ln) |
| 300 | t.Cleanup(func() { srv.Close() }) |
| 301 | rt := registry.Route{Host: "console.example.com", Socket: sock, Addr: "127.0.0.1:1", |
| 302 | Health: `{"path":"/healthz","status":200}`} |
| 303 | if err := routeReady(rt, time.Second)(); err != nil { |
| 304 | t.Fatalf("ready over the socket = %v (Addr must be ignored when Socket is set)", err) |
| 305 | } |
| 306 | } |
| 307 | |
| 308 | func TestRouteReadyWrongStatus(t *testing.T) { |
| 309 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(503) })) |
| 310 | t.Cleanup(srv.Close) |
| 311 | rt := registry.Route{Host: "x.example.com", Addr: strings.TrimPrefix(srv.URL, "http://"), |
| 312 | Health: `{"path":"/healthz","status":200}`} |
| 313 | if err := routeReady(rt, time.Second)(); err == nil || !strings.Contains(err.Error(), "503") { |
| 314 | t.Fatalf("ready = %v, want a 503 failure", err) |
| 315 | } |
| 316 | } |
| 317 | |
| 318 | func TestRouteReadyNoProbeIsAConnect(t *testing.T) { |
| 319 | ln, err := net.Listen("tcp", "127.0.0.1:0") |
| 320 | if err != nil { |
| 321 | t.Fatal(err) |
| 322 | } |
| 323 | t.Cleanup(func() { ln.Close() }) |
| 324 | if err := routeReady(registry.Route{Host: "x", Addr: ln.Addr().String()}, time.Second)(); err != nil { |
| 325 | t.Fatalf("connect readiness = %v", err) |
| 326 | } |
| 327 | } |
| 328 | |
| 329 | func TestRouteReadyNoUpstreamIsNil(t *testing.T) { |
| 330 | if routeReady(registry.Route{Host: "x"}, time.Second) != nil { |
| 331 | t.Fatal("a route with no upstream got a ready check; settle must use the hold instead") |
| 332 | } |
| 333 | } |
| 334 | |
| 335 | func TestRouteReadyDoesNotFollowRedirects(t *testing.T) { |
| 336 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 337 | if r.URL.Path == "/healthz" { |
| 338 | w.Header().Set("Location", "/ok") |
| 339 | w.WriteHeader(302) |
| 340 | return |
| 341 | } |
| 342 | if r.URL.Path == "/ok" { |
| 343 | w.WriteHeader(200) |
| 344 | return |
| 345 | } |
| 346 | w.WriteHeader(500) |
| 347 | })) |
| 348 | t.Cleanup(srv.Close) |
| 349 | // Case 1: expect 200 at /healthz, but get 302; should error |
| 350 | rt1 := registry.Route{Host: "x.example.com", Addr: strings.TrimPrefix(srv.URL, "http://"), |
| 351 | Health: `{"path":"/healthz","status":200}`} |
| 352 | if err := routeReady(rt1, time.Second)(); err == nil || !strings.Contains(err.Error(), "302") { |
| 353 | t.Fatalf("ready with redirect = %v, want a 302 error", err) |
| 354 | } |
| 355 | // Case 2: expect 302 at /healthz; should succeed |
| 356 | rt2 := registry.Route{Host: "x.example.com", Addr: strings.TrimPrefix(srv.URL, "http://"), |
| 357 | Health: `{"path":"/healthz","status":302}`} |
| 358 | if err := routeReady(rt2, time.Second)(); err != nil { |
| 359 | t.Fatalf("ready expecting 302 = %v, want nil", err) |
| 360 | } |
| 361 | } |
| 362 | |
| 363 | func boundRoute() registry.Route { |
| 364 | return registry.Route{Host: "platform.example.com", App: "console", Unit: "console.service", Addr: "127.0.0.1:1"} |
| 365 | } |
| 366 | |
| 367 | func liveTo(t *testing.T, opt, host, ver string) { |
| 368 | t.Helper() |
| 369 | dir := filepath.Join(opt, "versions", ver) |
| 370 | if err := os.MkdirAll(dir, 0o755); err != nil { |
| 371 | t.Fatal(err) |
| 372 | } |
| 373 | if err := os.WriteFile(filepath.Join(dir, "console"), []byte("x"), 0o755); err != nil { |
| 374 | t.Fatal(err) |
| 375 | } |
| 376 | if err := os.MkdirAll(filepath.Join(opt, "live"), 0o755); err != nil { |
| 377 | t.Fatal(err) |
| 378 | } |
| 379 | os.Remove(filepath.Join(opt, "live", host)) |
| 380 | if err := os.Symlink(filepath.Join(dir, "console"), filepath.Join(opt, "live", host)); err != nil { |
| 381 | t.Fatal(err) |
| 382 | } |
| 383 | } |
| 384 | |
| 385 | func TestNamedRestartBindingMismatchKeepsTheError(t *testing.T) { |
| 386 | opt := t.TempDir() |
| 387 | liveTo(t, opt, "platform.example.com", "new") |
| 388 | f := &fakeUnits{shows: []unitProps{{ActiveState: "active", InvocationID: "old", |
| 389 | ExecPath: filepath.Join(opt, "live", "console.example.com")}}} |
| 390 | failing := &failingTry{fakeUnits: f} |
| 391 | err := agentCheckedUnitRestart(failing, opt, func(string) string { return "" }, nil)(boundRoute()) |
| 392 | if err == nil || !strings.Contains(err.Error(), "try-restart failed") { |
| 393 | t.Fatalf("err = %v, want the try-restart error propagated (codex 12)", err) |
| 394 | } |
| 395 | } |
| 396 | |
| 397 | type failingTry struct{ *fakeUnits } |
| 398 | |
| 399 | func (failingTry) TryRestart(string) error { return errors.New("try-restart failed") } |
| 400 | |
| 401 | func TestNamedRestartShowFailsFallsBack(t *testing.T) { |
| 402 | opt := t.TempDir() |
| 403 | sys := &showFails{fakeUnits: &fakeUnits{}} |
| 404 | if err := agentCheckedUnitRestart(sys, opt, func(string) string { return "" }, nil)(boundRoute()); err != nil { |
| 405 | t.Fatalf("err = %v, want today's behaviour (plain try-restart, its nil)", err) |
| 406 | } |
| 407 | if len(sys.restarts) != 1 || sys.restarts[0] != "try-restart console.service" { |
| 408 | t.Fatalf("restarts = %v", sys.restarts) |
| 409 | } |
| 410 | } |
| 411 | |
| 412 | type showFails struct{ *fakeUnits } |
| 413 | |
| 414 | func (showFails) Show(string) (unitProps, error) { return unitProps{}, errors.New("Access denied") } |
| 415 | |
| 416 | func TestNamedRestartStoppedUnitIsNotChecked(t *testing.T) { |
| 417 | opt := t.TempDir() |
| 418 | liveTo(t, opt, "platform.example.com", "new") |
| 419 | f := &fakeUnits{shows: []unitProps{{ActiveState: "inactive", ExecPath: filepath.Join(opt, "live", "platform.example.com")}}} |
| 420 | if err := agentCheckedUnitRestart(f, opt, func(string) string { return "" }, nil)(boundRoute()); err != nil { |
| 421 | t.Fatalf("err = %v, want nil for an operator-stopped unit", err) |
| 422 | } |
| 423 | } |
| 424 | |
| 425 | func TestRollbackRestartResetsThenRestarts(t *testing.T) { |
| 426 | t.Parallel() |
| 427 | opt := t.TempDir() |
| 428 | liveTo(t, opt, "platform.example.com", "old") |
| 429 | reg := restartTestRegistry(t) |
| 430 | r := boundRoute() |
| 431 | r.Addr = "" // no upstream: readiness is nil and settle holds instead of dialling |
| 432 | if err := reg.Upsert(r); err != nil { |
| 433 | t.Fatal(err) |
| 434 | } |
| 435 | f := &fakeUnits{shows: []unitProps{ |
| 436 | {ActiveState: "active", InvocationID: "bad"}, |
| 437 | {ActiveState: "active", InvocationID: "restored", MainPID: 9}, |
| 438 | }, exe: map[int]string{9: filepath.Join(opt, "versions", "old", "console")}} |
| 439 | rb := agentRollbackUnitRestart(reg, f, opt, func(string) string { return "" }) |
| 440 | if err := rb("platform.example.com"); err != nil { |
| 441 | t.Fatalf("rollback = %v", err) |
| 442 | } |
| 443 | if strings.Join(f.restarts, ",") != "reset-failed console.service,restart console.service" { |
| 444 | t.Fatalf("restarts = %v, want reset-failed then restart (never start)", f.restarts) |
| 445 | } |
| 446 | } |
| 447 | |
| 448 | // newBuildDir is where checkedUnitRestart expects the new process to run |
| 449 | // from: the directory of the live link's resolved target, the same |
| 450 | // EvalSymlinks the production code does (t.TempDir can sit under a symlink). |
| 451 | func newBuildDir(t *testing.T, opt, host string) string { |
| 452 | t.Helper() |
| 453 | want, err := filepath.EvalSymlinks(filepath.Join(opt, "live", host)) |
| 454 | if err != nil { |
| 455 | t.Fatal(err) |
| 456 | } |
| 457 | return filepath.Dir(want) |
| 458 | } |
| 459 | |
| 460 | // The production wiring, end to end: checkedUnitRestart reads the unit, |
| 461 | // try-restarts it, and settles a healthy forward restart on the new build |
| 462 | // (ledger Task 6 minor). Real clock; the first poll is healthy. |
| 463 | func TestNamedRestartSettlesAHealthyForwardRestart(t *testing.T) { |
| 464 | opt := t.TempDir() |
| 465 | liveTo(t, opt, "platform.example.com", "new") |
| 466 | ln, err := net.Listen("tcp", "127.0.0.1:0") |
| 467 | if err != nil { |
| 468 | t.Fatal(err) |
| 469 | } |
| 470 | t.Cleanup(func() { ln.Close() }) |
| 471 | rt := boundRoute() |
| 472 | rt.Addr = ln.Addr().String() |
| 473 | f := &fakeUnits{shows: []unitProps{ |
| 474 | {ActiveState: "active", SubState: "running", InvocationID: "old", MainPID: 6, |
| 475 | ExecPath: filepath.Join(opt, "live", "platform.example.com")}, |
| 476 | {ActiveState: "active", SubState: "running", InvocationID: "new", MainPID: 7}, |
| 477 | }, exe: map[int]string{7: filepath.Join(newBuildDir(t, opt, "platform.example.com"), "console")}} |
| 478 | if err := agentCheckedUnitRestart(f, opt, func(string) string { return "" }, nil)(rt); err != nil { |
| 479 | t.Fatalf("forward restart = %v, want settled", err) |
| 480 | } |
| 481 | if strings.Join(f.restarts, ",") != "try-restart console.service" { |
| 482 | t.Fatalf("restarts = %v, want exactly the try-restart", f.restarts) |
| 483 | } |
| 484 | } |
| 485 | |
| 486 | // Final review F2: the unit read active, then an operator stopped it (before |
| 487 | // or during the settle). checkedUnitRestart reports nil, so adoptHost does |
| 488 | // not roll back, refuse or restart it, and the only systemctl verb issued is |
| 489 | // the try-restart itself. |
| 490 | func TestNamedRestartStopDuringSettleIsNotJudged(t *testing.T) { |
| 491 | opt := t.TempDir() |
| 492 | liveTo(t, opt, "platform.example.com", "new") |
| 493 | f := &fakeUnits{shows: []unitProps{ |
| 494 | {ActiveState: "active", SubState: "running", InvocationID: "old", MainPID: 6, |
| 495 | ExecPath: filepath.Join(opt, "live", "platform.example.com")}, |
| 496 | {ActiveState: "inactive", SubState: "dead", InvocationID: "old"}, |
| 497 | }} |
| 498 | if err := agentCheckedUnitRestart(f, opt, func(string) string { return "" }, nil)(boundRoute()); err != nil { |
| 499 | t.Fatalf("err = %v, want nil: a stop is not a verdict on the build", err) |
| 500 | } |
| 501 | if strings.Join(f.restarts, ",") != "try-restart console.service" { |
| 502 | t.Fatalf("restarts = %v, want only the try-restart (no restart, no reset-failed)", f.restarts) |
| 503 | } |
| 504 | } |
| 505 | |
| 506 | // The other side of F2: failed is a crash, never a stop. |
| 507 | func TestNamedRestartFailedStillFails(t *testing.T) { |
| 508 | opt := t.TempDir() |
| 509 | liveTo(t, opt, "platform.example.com", "new") |
| 510 | f := &fakeUnits{shows: []unitProps{ |
| 511 | {ActiveState: "active", SubState: "running", InvocationID: "old", MainPID: 6, |
| 512 | ExecPath: filepath.Join(opt, "live", "platform.example.com")}, |
| 513 | {ActiveState: "failed", SubState: "failed", InvocationID: "new"}, |
| 514 | }} |
| 515 | var nu *release.UnitNotUpError |
| 516 | if err := agentCheckedUnitRestart(f, opt, func(string) string { return "" }, nil)(boundRoute()); !errors.As(err, &nu) { |
| 517 | t.Fatalf("err = %v, want UnitNotUpError for a failed unit", err) |
| 518 | } |
| 519 | } |
| 520 | |
| 521 | // A stop during the rollback's own settle is the operator's too: the link |
| 522 | // names the restored build, and nothing restarts it a second time. |
| 523 | func TestRollbackRestartStopDuringSettleIsNotJudged(t *testing.T) { |
| 524 | opt := t.TempDir() |
| 525 | liveTo(t, opt, "platform.example.com", "old") |
| 526 | reg := restartTestRegistry(t) |
| 527 | r := boundRoute() |
| 528 | r.Addr = "" |
| 529 | if err := reg.Upsert(r); err != nil { |
| 530 | t.Fatal(err) |
| 531 | } |
| 532 | f := &fakeUnits{shows: []unitProps{ |
| 533 | {ActiveState: "active", InvocationID: "bad"}, |
| 534 | {ActiveState: "inactive", SubState: "dead", InvocationID: "bad"}, |
| 535 | }} |
| 536 | if err := agentRollbackUnitRestart(reg, f, opt, func(string) string { return "" })("platform.example.com"); err != nil { |
| 537 | t.Fatalf("rollback = %v, want nil for a stop during its settle", err) |
| 538 | } |
| 539 | if strings.Join(f.restarts, ",") != "reset-failed console.service,restart console.service" { |
| 540 | t.Fatalf("restarts = %v, want the one reset-failed and restart only", f.restarts) |
| 541 | } |
| 542 | } |
| 543 | |
amadan