| 1 | //go:build linux |
| 2 | |
| 3 | package instancectl |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "crypto/sha256" |
| 8 | "encoding/hex" |
| 9 | "errors" |
| 10 | "net/http" |
| 11 | "os" |
| 12 | "path/filepath" |
| 13 | "strconv" |
| 14 | "strings" |
| 15 | "testing" |
| 16 | "time" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/cgroup" |
| 19 | "github.com/carlosframework/platform/internal/provision" |
| 20 | "github.com/carlosframework/platform/internal/release" |
| 21 | ) |
| 22 | |
| 23 | func writeStaged(t *testing.T, path, body string) string { |
| 24 | t.Helper() |
| 25 | if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { |
| 26 | t.Fatal(err) |
| 27 | } |
| 28 | if err := os.WriteFile(path, []byte(body), 0o755); err != nil { |
| 29 | t.Fatal(err) |
| 30 | } |
| 31 | sum := sha256.Sum256([]byte(body)) |
| 32 | return hex.EncodeToString(sum[:]) |
| 33 | } |
| 34 | |
| 35 | // Root mounts only the host's own app's staged build, reached without a |
| 36 | // symlink, whose bytes are the attempt's (zero-downtime plan 3, Task 9). |
| 37 | func TestValidateStagedExe(t *testing.T) { |
| 38 | opt := t.TempDir() |
| 39 | host := "blog.example.com" |
| 40 | s := &Server{OptDir: opt, HostApp: func(h string) (string, string, error) { |
| 41 | if h != host { |
| 42 | return "", "", errors.New("no such route") |
| 43 | } |
| 44 | return release.OpsAccount, "blog", nil |
| 45 | }} |
| 46 | path := release.VersionPath(opt, release.OpsAccount, "v2", "blog") |
| 47 | digest := writeStaged(t, path, "build v2") |
| 48 | if err := s.validateStagedExe(host, StagedExe{Path: path, Version: "v2", Digest: digest}); err != nil { |
| 49 | t.Fatalf("the host's own staged build was refused: %v", err) |
| 50 | } |
| 51 | shop := release.VersionPath(opt, release.OpsAccount, "v2", "shop") |
| 52 | shopDigest := writeStaged(t, shop, "build v2") |
| 53 | // v3 of blog reached through a symlinked version directory. |
| 54 | if err := os.Symlink(filepath.Join(opt, "versions", "v2"), filepath.Join(opt, "versions", "v3")); err != nil { |
| 55 | t.Fatal(err) |
| 56 | } |
| 57 | linked := release.VersionPath(opt, release.OpsAccount, "v3", "blog") |
| 58 | for name, tc := range map[string]struct { |
| 59 | host string |
| 60 | x StagedExe |
| 61 | }{ |
| 62 | "another app's build": {host, StagedExe{Path: shop, Version: "v2", Digest: shopDigest}}, |
| 63 | "another version": {host, StagedExe{Path: path, Version: "v3", Digest: digest}}, |
| 64 | "a path outside": {host, StagedExe{Path: "/etc/passwd", Version: "v2", Digest: digest}}, |
| 65 | "bytes that differ": {host, StagedExe{Path: path, Version: "v2", Digest: strings.Repeat("0", 64)}}, |
| 66 | "a symlinked component": {host, StagedExe{Path: linked, Version: "v3", Digest: digest}}, |
| 67 | "a bad version": {host, StagedExe{Path: path, Version: "../v2", Digest: digest}}, |
| 68 | "an unknown host": {"other.example.com", StagedExe{Path: path, Version: "v2", Digest: digest}}, |
| 69 | } { |
| 70 | if err := s.validateStagedExe(tc.host, tc.x); !errors.Is(err, ErrStagedExe) { |
| 71 | t.Errorf("%s: %v, want ErrStagedExe", name, err) |
| 72 | } |
| 73 | } |
| 74 | if err := (&Server{OptDir: opt}).validateStagedExe(host, StagedExe{Path: path, Version: "v2", Digest: digest}); !errors.Is(err, ErrStagedExe) { |
| 75 | t.Errorf("no registry view: %v, want ErrStagedExe", err) |
| 76 | } |
| 77 | } |
| 78 | |
| 79 | // ctlCalled reports whether any recorded systemctl call starts with prefix. |
| 80 | func ctlCalled(c *fakeCtl, prefix string) bool { |
| 81 | c.mu.Lock() |
| 82 | defer c.mu.Unlock() |
| 83 | for _, l := range c.calls { |
| 84 | if strings.HasPrefix(l, prefix) { |
| 85 | return true |
| 86 | } |
| 87 | } |
| 88 | return false |
| 89 | } |
| 90 | |
| 91 | func (f *fakeCtl) reset() { |
| 92 | f.mu.Lock() |
| 93 | f.calls = nil |
| 94 | f.mu.Unlock() |
| 95 | } |
| 96 | |
| 97 | // The nested legacy sidecar runs the serving generation's staged build |
| 98 | // (spec §7.5), validated by root under the run unit's rule before its |
| 99 | // executable mount is pointed at it. The live link names another build |
| 100 | // throughout and makes no difference. A refused path answers 403 and |
| 101 | // leaves the mount alone. Before this task a sidecar /start could not name |
| 102 | // an executable at all, so the sidecar bound the live link. |
| 103 | func TestSidecarStartMountsTheValidatedStagedExe(t *testing.T) { |
| 104 | opt, exeRoot, cgroupRoot := t.TempDir(), t.TempDir(), t.TempDir() |
| 105 | staged := release.VersionPath(opt, release.OpsAccount, "v2", "blog") |
| 106 | digest := writeStaged(t, staged, "build v2") |
| 107 | ctl := &fakeCtl{} |
| 108 | _, sock := serveOnSocket(t, ctl, func(s *Server) { |
| 109 | s.DropInDir, s.ExeRoot, s.OptDir, s.CgroupRoot = t.TempDir(), exeRoot, opt, cgroupRoot |
| 110 | s.HostApp = func(string) (string, string, error) { return release.OpsAccount, "blog", nil } |
| 111 | }) |
| 112 | c := NewClient(sock) |
| 113 | app := filepath.Join(exeRoot, provision.InstanceID(genHost), "app") |
| 114 | if _, err := c.StartSidecarExe(context.Background(), genHost, cgroup.TierDefault.MemoryMax, |
| 115 | StagedExe{Path: staged, Version: "v2", Digest: digest}); err != nil { |
| 116 | t.Fatalf("the host's own staged build was refused for its sidecar: %v", err) |
| 117 | } |
| 118 | if got, err := os.Readlink(app); err != nil || got != staged { |
| 119 | t.Fatalf("the sidecar's exe = %q (%v), want the staged %q", got, err, staged) |
| 120 | } |
| 121 | for name, x := range map[string]StagedExe{ |
| 122 | "bytes that differ": {Path: staged, Version: "v2", Digest: strings.Repeat("0", 64)}, |
| 123 | "a path outside": {Path: "/etc/passwd", Version: "v2", Digest: digest}, |
| 124 | } { |
| 125 | if _, err := c.StartSidecarExe(context.Background(), genHost, cgroup.TierDefault.MemoryMax, x); !errors.Is(err, ErrStagedExe) { |
| 126 | t.Errorf("%s: %v, want ErrStagedExe", name, err) |
| 127 | } |
| 128 | if got, _ := os.Readlink(app); got != staged { |
| 129 | t.Errorf("%s: a refused start moved the mount to %q", name, got) |
| 130 | } |
| 131 | } |
| 132 | // The slice above was cold, so its tier was set. With a generation |
| 133 | // running (populated), a staged sidecar start leaves the slice's limit |
| 134 | // alone: during an overlap it is the raised parent of both generations |
| 135 | // (spec §7.3), and a start that set one tier would put them both under |
| 136 | // it. A flag-off start (no exe) sets the tier as today. |
| 137 | if !ctlCalled(ctl, "set-property --runtime "+SliceName(genHost)) { |
| 138 | t.Fatal("a cold slice got no tier") |
| 139 | } |
| 140 | events := filepath.Join(cgroupRoot, parentSlices, SliceName(genHost), "cgroup.events") |
| 141 | if err := os.MkdirAll(filepath.Dir(events), 0o755); err != nil { |
| 142 | t.Fatal(err) |
| 143 | } |
| 144 | if err := os.WriteFile(events, []byte("populated 1\nfrozen 0\n"), 0o644); err != nil { |
| 145 | t.Fatal(err) |
| 146 | } |
| 147 | ctl.reset() |
| 148 | if _, err := c.StartSidecarExe(context.Background(), genHost, cgroup.TierDefault.MemoryMax, |
| 149 | StagedExe{Path: staged, Version: "v2", Digest: digest}); err != nil { |
| 150 | t.Fatal(err) |
| 151 | } |
| 152 | if ctlCalled(ctl, "set-property") { |
| 153 | t.Fatal("a staged sidecar start rewrote the populated generation parent's limit") |
| 154 | } |
| 155 | if _, err := c.Start(context.Background(), genHost, RoleSidecar, cgroup.TierDefault.MemoryMax); err != nil { |
| 156 | t.Fatal(err) |
| 157 | } |
| 158 | if !ctlCalled(ctl, "set-property --runtime "+SliceName(genHost)) { |
| 159 | t.Fatal("a flag-off sidecar start no longer sets the tier") |
| 160 | } |
| 161 | // A run-role start on the original template may name an executable now |
| 162 | // (settings spec §4.2: the cold wake runs the build it recorded), and |
| 163 | // root still refuses one that is not the host's own validated build. |
| 164 | q := hostRole(genHost, RoleRun) |
| 165 | q.Set("memory_max", strconv.FormatInt(cgroup.TierDefault.MemoryMax, 10)) |
| 166 | q.Set("exe", staged) |
| 167 | q.Set("version", "v2") |
| 168 | q.Set("digest", strings.Repeat("0", 64)) |
| 169 | if code, _, _ := c.do(context.Background(), DeadlineControl, http.MethodPost, "/start", q, nil); code != http.StatusForbidden { |
| 170 | t.Errorf("a run-role start with a wrong digest answered %d, want 403", code) |
| 171 | } |
| 172 | } |
| 173 | |
| 174 | // originalTemplateServer serves root's control channel over a version store |
| 175 | // holding the host's own staged v2 and v3, with the live link on v2. It |
| 176 | // returns the client, the systemctl fake, v2 as the exe root must accept, |
| 177 | // the host's app mount and the opt dir. |
| 178 | func originalTemplateServer(t *testing.T) (*Client, *fakeCtl, StagedExe, string, string) { |
| 179 | t.Helper() |
| 180 | opt, exeRoot := t.TempDir(), t.TempDir() |
| 181 | staged := release.VersionPath(opt, release.OpsAccount, "v2", "blog") |
| 182 | digest := writeStaged(t, staged, "build v2") |
| 183 | writeStaged(t, release.VersionPath(opt, release.OpsAccount, "v3", "blog"), "build v3") |
| 184 | repointLiveLink(t, opt, genHost, staged) |
| 185 | ctl := &fakeCtl{answers: map[string]string{ |
| 186 | "show -p InvocationID,Result,ExecMainStatus,ExecMainCode": "InvocationID=feed\nResult=success\nExecMainStatus=0\nExecMainCode=exited\n", |
| 187 | }} |
| 188 | _, sock := serveOnSocket(t, ctl, func(s *Server) { |
| 189 | s.DropInDir, s.ExeRoot, s.OptDir = t.TempDir(), exeRoot, opt |
| 190 | s.HostApp = func(string) (string, string, error) { return release.OpsAccount, "blog", nil } |
| 191 | }) |
| 192 | app := filepath.Join(exeRoot, provision.InstanceID(genHost), "app") |
| 193 | return NewClient(sock), ctl, StagedExe{Path: staged, Version: "v2", Digest: digest}, app, opt |
| 194 | } |
| 195 | |
| 196 | // repointLiveLink is the agent's repoint of <opt>/live/<host>. |
| 197 | func repointLiveLink(t *testing.T, opt, host, target string) { |
| 198 | t.Helper() |
| 199 | link := release.LiveLink(opt, host) |
| 200 | if err := os.MkdirAll(filepath.Dir(link), 0o755); err != nil { |
| 201 | t.Fatal(err) |
| 202 | } |
| 203 | os.Remove(link) |
| 204 | if err := os.Symlink(target, link); err != nil { |
| 205 | t.Fatal(err) |
| 206 | } |
| 207 | } |
| 208 | |
| 209 | // Settings spec §4.2: the original run unit (generation 0) binds the |
| 210 | // recorded staged file, validated under zdd-3's rule, and a repoint of the |
| 211 | // live link after the start does not move it. A wrong digest is 403 |
| 212 | // (ErrStagedExe) before systemd or the mount is touched. A start that |
| 213 | // names no executable binds the live link, as today. |
| 214 | func TestTheRunUnitBindsAValidatedStagedExe(t *testing.T) { |
| 215 | c, ctl, x, app, opt := originalTemplateServer(t) |
| 216 | if _, err := c.StartExe(context.Background(), genHost, RoleRun, cgroup.TierDefault.MemoryMax, x); err != nil { |
| 217 | t.Fatalf("the host's own staged build was refused for its run unit: %v", err) |
| 218 | } |
| 219 | if !ctlCalled(ctl, "start "+UnitName(RoleRun, genHost)) { |
| 220 | t.Fatalf("the run unit was not started:\n%s", ctl.joined()) |
| 221 | } |
| 222 | repointLiveLink(t, opt, genHost, release.VersionPath(opt, release.OpsAccount, "v3", "blog")) |
| 223 | if got, err := os.Readlink(app); err != nil || got != x.Path { |
| 224 | t.Fatalf("the run unit's app mount = %q (%v), want the recorded %q", got, err, x.Path) |
| 225 | } |
| 226 | |
| 227 | ctl.reset() |
| 228 | bad := x |
| 229 | bad.Digest = strings.Repeat("0", 64) |
| 230 | if _, err := c.StartExe(context.Background(), genHost, RoleRun, cgroup.TierDefault.MemoryMax, bad); !errors.Is(err, ErrStagedExe) { |
| 231 | t.Fatalf("a wrong digest = %v, want ErrStagedExe", err) |
| 232 | } |
| 233 | if ctl.joined() != "" { |
| 234 | t.Fatalf("a refused start reached systemctl:\n%s", ctl.joined()) |
| 235 | } |
| 236 | if got, _ := os.Readlink(app); got != x.Path { |
| 237 | t.Fatalf("a refused start moved the mount to %q", got) |
| 238 | } |
| 239 | |
| 240 | if _, err := c.Start(context.Background(), genHost, RoleRun, cgroup.TierDefault.MemoryMax); err != nil { |
| 241 | t.Fatal(err) |
| 242 | } |
| 243 | if got, _ := os.Readlink(app); got != filepath.Join(LiveDir, genHost) { |
| 244 | t.Fatalf("a run start with no executable bound %q, want the live link", got) |
| 245 | } |
| 246 | } |
| 247 | |
| 248 | // The seed unit binds the recorded staged file the same way, through |
| 249 | // /seed, which names no role. A wrong digest is 403 before anything runs, |
| 250 | // and a seed that names no executable binds the live link, as today. |
| 251 | func TestTheSeedUnitBindsAValidatedStagedExe(t *testing.T) { |
| 252 | c, ctl, x, app, opt := originalTemplateServer(t) |
| 253 | res, err := c.SeedExe(context.Background(), genHost, cgroup.TierDefault.MemoryMax, x) |
| 254 | if err != nil || res.Result != "success" { |
| 255 | t.Fatalf("the host's own staged build was refused for its seed: %+v, %v", res, err) |
| 256 | } |
| 257 | if !ctlCalled(ctl, "start "+UnitName(RoleSeed, genHost)) { |
| 258 | t.Fatalf("the seed unit was not started:\n%s", ctl.joined()) |
| 259 | } |
| 260 | repointLiveLink(t, opt, genHost, release.VersionPath(opt, release.OpsAccount, "v3", "blog")) |
| 261 | if got, err := os.Readlink(app); err != nil || got != x.Path { |
| 262 | t.Fatalf("the seed unit's app mount = %q (%v), want the recorded %q", got, err, x.Path) |
| 263 | } |
| 264 | |
| 265 | ctl.reset() |
| 266 | bad := x |
| 267 | bad.Digest = strings.Repeat("0", 64) |
| 268 | if _, err := c.SeedExe(context.Background(), genHost, cgroup.TierDefault.MemoryMax, bad); !errors.Is(err, ErrStagedExe) { |
| 269 | t.Fatalf("a wrong digest = %v, want ErrStagedExe", err) |
| 270 | } |
| 271 | if ctl.joined() != "" { |
| 272 | t.Fatalf("a refused seed reached systemctl:\n%s", ctl.joined()) |
| 273 | } |
| 274 | if got, _ := os.Readlink(app); got != x.Path { |
| 275 | t.Fatalf("a refused seed moved the mount to %q", got) |
| 276 | } |
| 277 | |
| 278 | if _, err := c.Seed(context.Background(), genHost, cgroup.TierDefault.MemoryMax); err != nil { |
| 279 | t.Fatal(err) |
| 280 | } |
| 281 | if got, _ := os.Readlink(app); got != filepath.Join(LiveDir, genHost) { |
| 282 | t.Fatalf("a seed with no executable bound %q, want the live link", got) |
| 283 | } |
| 284 | } |
| 285 | |
| 286 | // stagedGenServer is genServer with a version store holding the host's own |
| 287 | // staged v2 build. It returns the staged exe root has to accept. |
| 288 | func stagedGenServer(t *testing.T, ctl *fakeCtl) (*Server, string, StagedExe) { |
| 289 | t.Helper() |
| 290 | opt := t.TempDir() |
| 291 | staged := release.VersionPath(opt, release.OpsAccount, "v2", "blog") |
| 292 | digest := writeStaged(t, staged, "build v2") |
| 293 | dropins, exe := t.TempDir(), t.TempDir() |
| 294 | s, sock := serveOnSocket(t, ctl, func(s *Server) { |
| 295 | s.DropInDir, s.ExeRoot, s.OptDir = dropins, exe, opt |
| 296 | s.HostApp = func(string) (string, string, error) { return release.OpsAccount, "blog", nil } |
| 297 | }) |
| 298 | return s, sock, StagedExe{Path: staged, Version: "v2", Digest: digest} |
| 299 | } |
| 300 | |
| 301 | // A validated staged generation start mounts the staged build. A refused one |
| 302 | // answers ErrStagedExe having touched neither systemd nor the disk. |
| 303 | func TestGenStartExeMountsTheValidatedBuildAndRefusalTouchesNothing(t *testing.T) { |
| 304 | ctl := genCtl() |
| 305 | s, sock, x := stagedGenServer(t, ctl) |
| 306 | c := NewClient(sock) |
| 307 | bad := x |
| 308 | bad.Digest = strings.Repeat("0", 64) |
| 309 | if _, err := c.StartGenExe(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax, bad); !errors.Is(err, ErrStagedExe) { |
| 310 | t.Fatalf("StartGenExe(bad digest) = %v, want ErrStagedExe", err) |
| 311 | } |
| 312 | if ctl.joined() != "" { |
| 313 | t.Fatalf("a refused staged start reached systemctl:\n%s", ctl.joined()) |
| 314 | } |
| 315 | if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); !os.IsNotExist(err) { |
| 316 | t.Fatal("an executable mount was laid out for a refused start") |
| 317 | } |
| 318 | if _, err := c.StartGenExe(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax, x); err != nil { |
| 319 | t.Fatalf("StartGenExe: %v", err) |
| 320 | } |
| 321 | if got, err := os.Readlink(filepath.Join(s.ExeRoot, GenKey(genHost, 7), "app")); err != nil || got != x.Path { |
| 322 | t.Fatalf("generation 7's exe = %q (%v), want %q", got, err, x.Path) |
| 323 | } |
| 324 | } |
| 325 | |
| 326 | // The staged start keeps plan 2's answers: a cleanup that cannot prove the |
| 327 | // unit gone is ErrGenUnproven and not ErrStagedExe, and its files stay. |
| 328 | func TestGenStartExeKeepsTheUnprovenAnswer(t *testing.T) { |
| 329 | ctl := genCtl() |
| 330 | ctl.fail = map[string]error{"show -p InvocationID": errors.New("bus timeout")} |
| 331 | s, sock, x := stagedGenServer(t, ctl) |
| 332 | s.cgroupWaitBudget = 100 * time.Millisecond |
| 333 | populateGenCgroup(t, s) |
| 334 | _, err := NewClient(sock).StartGenExe(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax, x) |
| 335 | if !errors.Is(err, ErrGenUnproven) || errors.Is(err, ErrStagedExe) { |
| 336 | t.Fatalf("StartGenExe = %v, want ErrGenUnproven only", err) |
| 337 | } |
| 338 | if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); err != nil { |
| 339 | t.Fatalf("the exe layout of an unproven generation was removed: %v", err) |
| 340 | } |
| 341 | } |
| 342 | |
| 343 | // exe, version and digest belong to the start and seed verbs only. On |
| 344 | // those, an executable root cannot vouch for is refused (403) before |
| 345 | // systemd is asked anything. |
| 346 | func TestStagedExeParametersRefusedWhereTheyDoNotBelong(t *testing.T) { |
| 347 | ctl := genCtl() |
| 348 | _, c := newServer(t, ctl, func(s *Server) { s.DropInDir = t.TempDir() }) |
| 349 | for _, path := range []string{ |
| 350 | "/stop?host=" + genHost + "&role=run&gen=7&exe=/x", |
| 351 | "/stop?host=" + genHost + "&role=sidecar&exe=/x", |
| 352 | "/reset?host=" + genHost + "&role=sidecar&version=v2", |
| 353 | "/seed?host=" + genHost + "&memory_max=536870912&version=v2", |
| 354 | "/start?host=" + genHost + "&role=run&memory_max=536870912&gen=7&version=v2", |
| 355 | } { |
| 356 | if resp := post(t, c, path); resp.StatusCode != 400 { |
| 357 | t.Errorf("%s: status %d, want 400", path, resp.StatusCode) |
| 358 | } |
| 359 | } |
| 360 | // Settings spec §4.2: the original run unit and the seed unit may name |
| 361 | // an executable; this one is not the host's own validated build. |
| 362 | for _, path := range []string{ |
| 363 | "/seed?host=" + genHost + "&memory_max=536870912&exe=/x", |
| 364 | "/start?host=" + genHost + "&role=run&memory_max=536870912&exe=/x&version=v2&digest=d", |
| 365 | } { |
| 366 | if resp := post(t, c, path); resp.StatusCode != http.StatusForbidden { |
| 367 | t.Errorf("%s: status %d, want 403", path, resp.StatusCode) |
| 368 | } |
| 369 | } |
| 370 | if ctl.joined() != "" { |
| 371 | t.Errorf("systemctl was called:\n%s", ctl.joined()) |
| 372 | } |
| 373 | } |
| 374 | |
amadan