| 1 | package instancectl |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "net/http" |
| 7 | "net/http/httptest" |
| 8 | "os" |
| 9 | "path/filepath" |
| 10 | "strconv" |
| 11 | "strings" |
| 12 | "sync" |
| 13 | "testing" |
| 14 | "time" |
| 15 | |
| 16 | "github.com/carlosframework/platform/internal/cgroup" |
| 17 | "github.com/carlosframework/platform/internal/provision" |
| 18 | ) |
| 19 | |
| 20 | // unitCtl is a stateful systemd stand-in for one run unit. |
| 21 | type unitCtl struct { |
| 22 | mu sync.Mutex |
| 23 | calls []string |
| 24 | state, invocation string |
| 25 | pid int |
| 26 | group string |
| 27 | startInvocation string |
| 28 | stopLeavesActive bool |
| 29 | failStart error |
| 30 | events string |
| 31 | } |
| 32 | |
| 33 | func (u *unitCtl) run(ctx context.Context, args ...string) (string, error) { |
| 34 | u.mu.Lock() |
| 35 | defer u.mu.Unlock() |
| 36 | u.calls = append(u.calls, strings.Join(args, " ")) |
| 37 | if err := ctx.Err(); err != nil { |
| 38 | return "", err |
| 39 | } |
| 40 | switch args[0] { |
| 41 | case "start": |
| 42 | if u.failStart != nil { |
| 43 | return "failed", u.failStart |
| 44 | } |
| 45 | u.state, u.invocation, u.pid = "active", u.startInvocation, 4242 |
| 46 | u.setPopulated(true) |
| 47 | case "stop": |
| 48 | if !u.stopLeavesActive { |
| 49 | u.state = "inactive" |
| 50 | u.setPopulated(false) |
| 51 | } |
| 52 | case "show": |
| 53 | var out []string |
| 54 | for _, p := range strings.Split(args[2], ",") { |
| 55 | switch p { |
| 56 | case "ActiveState": |
| 57 | out = append(out, "ActiveState="+u.state) |
| 58 | case "InvocationID": |
| 59 | out = append(out, "InvocationID="+u.invocation) |
| 60 | case "MainPID": |
| 61 | out = append(out, "MainPID="+strconv.Itoa(u.pid)) |
| 62 | case "ControlGroup": |
| 63 | out = append(out, "ControlGroup="+u.group) |
| 64 | } |
| 65 | } |
| 66 | return strings.Join(out, "\n") + "\n", nil |
| 67 | } |
| 68 | return "", nil |
| 69 | } |
| 70 | |
| 71 | func (u *unitCtl) setPopulated(on bool) { |
| 72 | v := "populated 0\n" |
| 73 | if on { |
| 74 | v = "populated 1\n" |
| 75 | } |
| 76 | _ = os.WriteFile(filepath.Join(u.events, "cgroup.events"), []byte(v), 0o644) |
| 77 | } |
| 78 | |
| 79 | func (u *unitCtl) joined() string { |
| 80 | u.mu.Lock() |
| 81 | defer u.mu.Unlock() |
| 82 | return strings.Join(u.calls, "\n") |
| 83 | } |
| 84 | |
| 85 | func claimUnitFixture(t *testing.T) (ClaimUnits, *unitCtl, ClaimUnit, string) { |
| 86 | t.Helper() |
| 87 | host := "shop.example" |
| 88 | root := t.TempDir() |
| 89 | events := filepath.Join(root, parentSlices, SliceName(host), UnitName(RoleRun, host)) |
| 90 | if err := os.MkdirAll(events, 0o755); err != nil { |
| 91 | t.Fatal(err) |
| 92 | } |
| 93 | if err := os.WriteFile(filepath.Join(events, "cgroup.events"), []byte("populated 0\nfrozen 0\n"), 0o644); err != nil { |
| 94 | t.Fatal(err) |
| 95 | } |
| 96 | ctl := &unitCtl{state: "inactive", invocation: "old-invocation", startInvocation: "new-invocation", group: "/" + ClaimUnitControlGroup(host), events: events} |
| 97 | build := filepath.Join(t.TempDir(), "versions", "v1", "app") |
| 98 | if err := os.MkdirAll(filepath.Dir(build), 0o755); err != nil { |
| 99 | t.Fatal(err) |
| 100 | } |
| 101 | if err := os.WriteFile(build, []byte("app"), 0o555); err != nil { |
| 102 | t.Fatal(err) |
| 103 | } |
| 104 | u := ClaimUnits{Systemctl: ctl.run, CgroupRoot: root, ExeRoot: filepath.Join(t.TempDir(), "exe"), WaitBudget: time.Second, Poll: time.Millisecond} |
| 105 | return u, ctl, ClaimUnit{Host: host, Executable: build, MemoryMax: cgroup.TierDefault.MemoryMax}, events |
| 106 | } |
| 107 | |
| 108 | func populate(t *testing.T, events string, on bool) { |
| 109 | t.Helper() |
| 110 | v := "populated 0\n" |
| 111 | if on { |
| 112 | v = "populated 1\n" |
| 113 | } |
| 114 | if err := os.WriteFile(filepath.Join(events, "cgroup.events"), []byte(v), 0o644); err != nil { |
| 115 | t.Fatal(err) |
| 116 | } |
| 117 | } |
| 118 | |
| 119 | func TestClaimUnitStartsMainsIsolatedRunUnitOnThePinnedBuild(t *testing.T) { |
| 120 | u, ctl, in, _ := claimUnitFixture(t) |
| 121 | got, err := u.Start(context.Background(), in) |
| 122 | if err != nil { |
| 123 | t.Fatal(err) |
| 124 | } |
| 125 | unit := UnitName(RoleRun, in.Host) |
| 126 | if got.Unit != unit || got.InvocationID != "new-invocation" || got.MainPID != 4242 || got.ControlGroup != ClaimUnitControlGroup(in.Host) { |
| 127 | t.Fatalf("started=%+v", got) |
| 128 | } |
| 129 | calls := ctl.joined() |
| 130 | want := []string{"set-property --runtime " + SliceName(in.Host) + " MemoryMax=" + strconv.FormatInt(in.MemoryMax, 10), "start " + unit} |
| 131 | at := 0 |
| 132 | for _, w := range want { |
| 133 | i := strings.Index(calls[at:], w) |
| 134 | if i < 0 { |
| 135 | t.Fatalf("missing %q in order:\n%s", w, calls) |
| 136 | } |
| 137 | at += i + len(w) |
| 138 | } |
| 139 | // The unit's /opt/carlos/app source is the exact build root verified, |
| 140 | // not the host's retargetable live link. |
| 141 | iid := provision.InstanceID(in.Host) |
| 142 | if target, err := os.Readlink(filepath.Join(u.ExeRoot, iid, "app")); err != nil || target != in.Executable { |
| 143 | t.Fatalf("app source=%q %v", target, err) |
| 144 | } |
| 145 | if target, err := os.Readlink(filepath.Join(u.ExeRoot, iid, "live", in.Host)); err != nil || target != "../app" { |
| 146 | t.Fatalf("live source=%q %v", target, err) |
| 147 | } |
| 148 | } |
| 149 | |
| 150 | func TestClaimUnitRefusesBeforeAnyEffect(t *testing.T) { |
| 151 | for name, setup := range map[string]func(*ClaimUnits, *unitCtl, *ClaimUnit, string){ |
| 152 | "active unit": func(_ *ClaimUnits, c *unitCtl, _ *ClaimUnit, _ string) { c.state = "active" }, |
| 153 | "activating unit": func(_ *ClaimUnits, c *unitCtl, _ *ClaimUnit, _ string) { c.state = "activating" }, |
| 154 | "populated group": func(_ *ClaimUnits, _ *unitCtl, _ *ClaimUnit, ev string) { populate(t, ev, true) }, |
| 155 | "not a tier": func(_ *ClaimUnits, _ *unitCtl, in *ClaimUnit, _ string) { in.MemoryMax = 12345 }, |
| 156 | "relative build": func(_ *ClaimUnits, _ *unitCtl, in *ClaimUnit, _ string) { in.Executable = "versions/app" }, |
| 157 | "unclean build": func(_ *ClaimUnits, _ *unitCtl, in *ClaimUnit, _ string) { in.Executable = "/opt/carlos/../x" }, |
| 158 | "not a host": func(_ *ClaimUnits, _ *unitCtl, in *ClaimUnit, _ string) { in.Host = "../etc" }, |
| 159 | "disk full": func(u *ClaimUnits, _ *unitCtl, _ *ClaimUnit, _ string) { u.Room = func() error { return ErrDiskFull } }, |
| 160 | "no exe root": func(u *ClaimUnits, _ *unitCtl, _ *ClaimUnit, _ string) { u.ExeRoot = "" }, |
| 161 | "no cgroup root": func(u *ClaimUnits, _ *unitCtl, _ *ClaimUnit, _ string) { u.CgroupRoot = "" }, |
| 162 | "no systemctl": func(u *ClaimUnits, _ *unitCtl, _ *ClaimUnit, _ string) { u.Systemctl = nil }, |
| 163 | "platform binary": func(_ *ClaimUnits, _ *unitCtl, in *ClaimUnit, _ string) { in.Executable = SystemBinary }, |
| 164 | "host live link": func(_ *ClaimUnits, _ *unitCtl, in *ClaimUnit, _ string) { |
| 165 | in.Executable = filepath.Join(LiveDir, in.Host) |
| 166 | }, |
| 167 | "probe build view": func(_ *ClaimUnits, _ *unitCtl, in *ClaimUnit, _ string) { in.Host = ProbeDirName }, |
| 168 | } { |
| 169 | u, ctl, in, events := claimUnitFixture(t) |
| 170 | setup(&u, ctl, &in, events) |
| 171 | if _, err := u.Start(context.Background(), in); err == nil { |
| 172 | t.Fatalf("%s started", name) |
| 173 | } |
| 174 | if ctl != nil && strings.Contains(ctl.joined(), "start ") { |
| 175 | t.Fatalf("%s reached systemctl start:\n%s", name, ctl.joined()) |
| 176 | } |
| 177 | } |
| 178 | } |
| 179 | |
| 180 | func TestClaimUnitStopsAnInvocationItCannotVouchFor(t *testing.T) { |
| 181 | for name, setup := range map[string]func(*unitCtl){ |
| 182 | "same invocation": func(c *unitCtl) { c.startInvocation = "old-invocation" }, |
| 183 | "empty invocation": func(c *unitCtl) { c.startInvocation = "" }, |
| 184 | "other group": func(c *unitCtl) { c.group = "/carlos.slice/elsewhere.service" }, |
| 185 | "start failed": func(c *unitCtl) { c.failStart = errors.New("boom") }, |
| 186 | } { |
| 187 | u, ctl, in, _ := claimUnitFixture(t) |
| 188 | setup(ctl) |
| 189 | if _, err := u.Start(context.Background(), in); err == nil { |
| 190 | t.Fatalf("%s accepted", name) |
| 191 | } |
| 192 | if !strings.Contains(ctl.joined(), "stop "+UnitName(RoleRun, in.Host)) { |
| 193 | t.Fatalf("%s left the unit running:\n%s", name, ctl.joined()) |
| 194 | } |
| 195 | } |
| 196 | } |
| 197 | |
| 198 | func TestClaimUnitStopAndGoneProveTheInvocationEnded(t *testing.T) { |
| 199 | ctx := context.Background() |
| 200 | u, ctl, in, events := claimUnitFixture(t) |
| 201 | if _, err := u.Start(ctx, in); err != nil { |
| 202 | t.Fatal(err) |
| 203 | } |
| 204 | populate(t, events, true) |
| 205 | if err := u.Gone(ctx, in.Host, "new-invocation"); err == nil { |
| 206 | t.Fatal("running invocation reported gone") |
| 207 | } |
| 208 | if err := u.Gone(ctx, in.Host, "older-invocation"); err != nil { |
| 209 | t.Fatalf("a later invocation means the older one ended: %v", err) |
| 210 | } |
| 211 | ctl.stopLeavesActive = true |
| 212 | stopCtx, cancel := context.WithTimeout(ctx, 50*time.Millisecond) |
| 213 | if err := u.Stop(stopCtx, in.Host); err == nil { |
| 214 | t.Fatal("stop returned while the unit stayed active") |
| 215 | } |
| 216 | cancel() |
| 217 | ctl.stopLeavesActive = false |
| 218 | populate(t, events, false) |
| 219 | if err := u.Stop(ctx, in.Host); err != nil { |
| 220 | t.Fatal(err) |
| 221 | } |
| 222 | if err := u.Gone(ctx, in.Host, "new-invocation"); err != nil { |
| 223 | t.Fatalf("stopped invocation: %v", err) |
| 224 | } |
| 225 | if err := u.Gone(ctx, in.Host, ""); err != nil { |
| 226 | t.Fatalf("idle unit with no named invocation: %v", err) |
| 227 | } |
| 228 | populate(t, events, true) |
| 229 | if err := u.Gone(ctx, in.Host, ""); err == nil { |
| 230 | t.Fatal("populated group reported gone") |
| 231 | } |
| 232 | } |
| 233 | |
| 234 | func TestClaimLaunchRouteExistsOnlyInClaimMode(t *testing.T) { |
| 235 | legacy := (&Server{Systemctl: func(context.Context, ...string) (string, error) { return "", nil }}).Handler() |
| 236 | rec := httptest.NewRecorder() |
| 237 | legacy.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/claim/launch", strings.NewReader("{}"))) |
| 238 | if rec.Code != http.StatusNotFound { |
| 239 | t.Fatalf("flag-off mux answered /claim/launch with %d", rec.Code) |
| 240 | } |
| 241 | reached := "" |
| 242 | claim := (&Server{ClaimStop: http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { reached = r.URL.Path })}).Handler() |
| 243 | rec = httptest.NewRecorder() |
| 244 | claim.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/claim/launch", strings.NewReader("{}"))) |
| 245 | if reached != "/claim/launch" { |
| 246 | t.Fatalf("claim mux did not route /claim/launch: %q", reached) |
| 247 | } |
| 248 | rec = httptest.NewRecorder() |
| 249 | claim.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/claim/launch", nil)) |
| 250 | if rec.Code != http.StatusMethodNotAllowed { |
| 251 | t.Fatalf("GET /claim/launch=%d", rec.Code) |
| 252 | } |
| 253 | } |
| 254 | |
| 255 | // TestClaimHeldReleaseRouteExistsOnlyInClaimMode (moves Task 7b): the |
| 256 | // root's held-release route is the claim handler's, and a flag-off mux |
| 257 | // has none, so an agent's request there reads as unavailable. |
| 258 | func TestClaimHeldReleaseRouteExistsOnlyInClaimMode(t *testing.T) { |
| 259 | legacy := (&Server{Systemctl: func(context.Context, ...string) (string, error) { return "", nil }}).Handler() |
| 260 | rec := httptest.NewRecorder() |
| 261 | legacy.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/claim/held-release", strings.NewReader("{}"))) |
| 262 | if rec.Code != http.StatusNotFound { |
| 263 | t.Fatalf("flag-off mux answered /claim/held-release with %d", rec.Code) |
| 264 | } |
| 265 | // Claim mode with moves off (whole-branch fixes, item 6): the route is |
| 266 | // not registered, so main's 404 stands rather than the claim handler's |
| 267 | // refusal. |
| 268 | reached := "" |
| 269 | stop := http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { reached = r.URL.Path }) |
| 270 | claimOnly := (&Server{ClaimStop: stop}).Handler() |
| 271 | rec = httptest.NewRecorder() |
| 272 | claimOnly.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/claim/held-release", strings.NewReader("{}"))) |
| 273 | if rec.Code != http.StatusNotFound || reached != "" { |
| 274 | t.Fatalf("claim mode with moves off answered /claim/held-release with %d (reached %q)", rec.Code, reached) |
| 275 | } |
| 276 | // Both flags on: the held-release handler is registered and routed. |
| 277 | claim := (&Server{ClaimStop: stop, ClaimHeldRelease: stop}).Handler() |
| 278 | rec = httptest.NewRecorder() |
| 279 | claim.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/claim/held-release", strings.NewReader("{}"))) |
| 280 | if reached != "/claim/held-release" { |
| 281 | t.Fatalf("claim mux did not route /claim/held-release: %q", reached) |
| 282 | } |
| 283 | rec = httptest.NewRecorder() |
| 284 | claim.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/claim/held-release", nil)) |
| 285 | if rec.Code != http.StatusMethodNotAllowed { |
| 286 | t.Fatalf("GET /claim/held-release=%d", rec.Code) |
| 287 | } |
| 288 | } |
| 289 | |
amadan