| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "crypto/ed25519" |
| 7 | "encoding/base64" |
| 8 | "encoding/json" |
| 9 | "fmt" |
| 10 | "log" |
| 11 | "net" |
| 12 | "os" |
| 13 | "os/exec" |
| 14 | "path/filepath" |
| 15 | "reflect" |
| 16 | "runtime" |
| 17 | "slices" |
| 18 | "strings" |
| 19 | "testing" |
| 20 | "time" |
| 21 | |
| 22 | "github.com/carlosframework/platform/internal/domains" |
| 23 | "github.com/carlosframework/platform/internal/metrics" |
| 24 | "github.com/carlosframework/platform/internal/moves" |
| 25 | "github.com/carlosframework/platform/internal/registry" |
| 26 | "github.com/carlosframework/platform/internal/release" |
| 27 | "github.com/carlosframework/platform/internal/status" |
| 28 | ) |
| 29 | |
| 30 | // expectedUnits is the contract for what bootstrap installs — a new unit file |
| 31 | // must be added here (and a removed one taken out) or TestUnitsEmbedComplete |
| 32 | // fails, so the embed directive and the fleet's unit set can't drift apart. |
| 33 | var expectedUnits = []string{ |
| 34 | "carlos-adopt.path", |
| 35 | "carlos-adopt.service", |
| 36 | "carlos-adopt.timer", |
| 37 | "carlos-app@.service", |
| 38 | "carlos-app@.socket", |
| 39 | "carlos-config-sync.service", |
| 40 | "carlos-dbread.socket", |
| 41 | "carlos-dbread@.service", |
| 42 | "carlos-edge.service", |
| 43 | "carlos-hibernate-verify.service", |
| 44 | "carlos-hibernate-verify.timer", |
| 45 | "carlos-instance-cmd@.service", |
| 46 | "carlos-instance-control.service", |
| 47 | "carlos-instance-control.socket", |
| 48 | "carlos-instance-gen@.service", |
| 49 | "carlos-instance-probe.service", |
| 50 | "carlos-instance-seed@.service", |
| 51 | "carlos-instance-sidecar@.service", |
| 52 | "carlos-instance@.service", |
| 53 | "carlos-instances.slice", |
| 54 | "carlos-litestream.service", |
| 55 | "carlos-relay.service", |
| 56 | "carlos-restart.path", |
| 57 | "carlos-restart.service", |
| 58 | "carlos-restore-verify.service", |
| 59 | "carlos-restore-verify.timer", |
| 60 | "carlos-stage.path", |
| 61 | "carlos-stage.service", |
| 62 | "carlos-system-join.path", |
| 63 | "carlos-system-join.service", |
| 64 | "carlos-system-recover.service", |
| 65 | "carlos-system-update.path", |
| 66 | "carlos-system-update.service", |
| 67 | "carlos.slice", |
| 68 | "data-instances.mount", |
| 69 | "run-carlos-instances.mount", |
| 70 | "run-carlos-logs.mount", |
| 71 | } |
| 72 | |
| 73 | // requireUnixSockets skips a test when the sandbox forbids unix-socket |
| 74 | // creation (probe: net.Listen("unix") in a temp dir). None of the current |
| 75 | // tests need one; the helper is here for the host tests that will (agent |
| 76 | // serving loops), matching the pattern used across the repo. |
| 77 | func requireUnixSockets(t *testing.T) { |
| 78 | t.Helper() |
| 79 | dir, err := os.MkdirTemp("", "unix-probe-") |
| 80 | if err != nil { |
| 81 | t.Fatal(err) |
| 82 | } |
| 83 | t.Cleanup(func() { os.RemoveAll(dir) }) |
| 84 | l, err := net.Listen("unix", filepath.Join(dir, "probe.sock")) |
| 85 | if err != nil { |
| 86 | if strings.Contains(err.Error(), "operation not permitted") { |
| 87 | t.Skipf("sandbox forbids unix sockets: %v", err) |
| 88 | } |
| 89 | t.Fatalf("unix socket probe: %v", err) |
| 90 | } |
| 91 | l.Close() |
| 92 | } |
| 93 | |
| 94 | func TestUnitsEmbedComplete(t *testing.T) { |
| 95 | names, err := unitNames() |
| 96 | if err != nil { |
| 97 | t.Fatal(err) |
| 98 | } |
| 99 | got := map[string]bool{} |
| 100 | for _, n := range names { |
| 101 | got[n] = true |
| 102 | } |
| 103 | for _, want := range expectedUnits { |
| 104 | if !got[want] { |
| 105 | t.Errorf("unit %s missing from the embed FS", want) |
| 106 | } |
| 107 | } |
| 108 | if len(names) != len(expectedUnits) { |
| 109 | t.Errorf("embed FS has %d units %v, expected list has %d — keep them in step", len(names), names, len(expectedUnits)) |
| 110 | } |
| 111 | } |
| 112 | |
| 113 | // TestBootstrapStartsRelayUnit pins the one service bootstrap starts and the |
| 114 | // one it must not. The relay is safe to start fleet-wide because its |
| 115 | // ConditionPathExists gates it on a masters file; the edge is not, and a |
| 116 | // change that quietly promoted every service into the start line would arm |
| 117 | // half-staged boxes. |
| 118 | func TestBootstrapStartsRelayUnit(t *testing.T) { |
| 119 | plan := systemctlPlan(enabledUnits) |
| 120 | var started []string |
| 121 | for _, step := range plan { |
| 122 | if step[0] == "start" { |
| 123 | started = step[1:] |
| 124 | } |
| 125 | } |
| 126 | if !slices.Contains(started, "carlos-relay.service") { |
| 127 | t.Fatalf("relay not started by plan: %v", plan) |
| 128 | } |
| 129 | if slices.Contains(started, "carlos-edge.service") { |
| 130 | t.Fatalf("edge must stay start-by-hand: %v", plan) |
| 131 | } |
| 132 | } |
| 133 | |
| 134 | // TestRelayUnitIsGatedAndLosesFirst pins the three lines that make it safe |
| 135 | // to enable and start carlos-relay.service on every box in the fleet. The |
| 136 | // condition is what keeps an unarmed box from ever listening; the OOM |
| 137 | // adjustment and the absence of the bind capability are what keep the relay |
| 138 | // from costing the edge anything when they share a box (relay spec §3). |
| 139 | func TestRelayUnitIsGatedAndLosesFirst(t *testing.T) { |
| 140 | b, err := unitsFS.ReadFile("units/" + relayUnitName) |
| 141 | if err != nil { |
| 142 | t.Fatal(err) |
| 143 | } |
| 144 | unit := string(b) |
| 145 | for _, want := range []string{ |
| 146 | "ConditionPathExists=/etc/carlos/relay-masters", |
| 147 | "OOMScoreAdjust=1000", |
| 148 | "MemoryMax=256M", |
| 149 | "ExecStart=/opt/carlos/carlos relay serve", |
| 150 | "ReadWritePaths=/data/relay /run/carlos", |
| 151 | } { |
| 152 | if !strings.Contains(unit, want) { |
| 153 | t.Errorf("%s: missing %q", relayUnitName, want) |
| 154 | } |
| 155 | } |
| 156 | // 3478 is unprivileged; only the edge may bind below 1024. |
| 157 | if strings.Contains(unit, "CAP_NET_BIND_SERVICE") { |
| 158 | t.Errorf("%s: must not carry CAP_NET_BIND_SERVICE — 3478 needs no capability", relayUnitName) |
| 159 | } |
| 160 | } |
| 161 | |
| 162 | func TestBootstrapWritesUnitsToStagingDir(t *testing.T) { |
| 163 | root := t.TempDir() |
| 164 | if err := bootstrap(root, nil); err != nil { |
| 165 | t.Fatalf("bootstrap: %v", err) |
| 166 | } |
| 167 | |
| 168 | unitDir := filepath.Join(root, "etc", "systemd", "system") |
| 169 | for _, name := range expectedUnits { |
| 170 | b, err := os.ReadFile(filepath.Join(unitDir, name)) |
| 171 | if err != nil { |
| 172 | t.Errorf("unit not installed: %v", err) |
| 173 | continue |
| 174 | } |
| 175 | content := string(b) |
| 176 | // Only the edge binds privileged ports; every other unit must not |
| 177 | // carry the capability. |
| 178 | if name == "carlos-edge.service" { |
| 179 | if !strings.Contains(content, "AmbientCapabilities=CAP_NET_BIND_SERVICE") { |
| 180 | t.Errorf("%s: missing AmbientCapabilities=CAP_NET_BIND_SERVICE", name) |
| 181 | } |
| 182 | } else if strings.Contains(content, "CAP_NET_BIND_SERVICE") { |
| 183 | t.Errorf("%s: must not carry CAP_NET_BIND_SERVICE", name) |
| 184 | } |
| 185 | } |
| 186 | |
| 187 | // The edge unit is the agent role. |
| 188 | edge, err := os.ReadFile(filepath.Join(unitDir, "carlos-edge.service")) |
| 189 | if err != nil { |
| 190 | t.Fatal(err) |
| 191 | } |
| 192 | for _, want := range []string{ |
| 193 | "ExecStart=/opt/carlos/carlos agent", |
| 194 | "EnvironmentFile=-/etc/carlos/host.env", |
| 195 | "ProtectSystem=strict", |
| 196 | "ReadWritePaths=/data /run/carlos", |
| 197 | "NoNewPrivileges=yes", |
| 198 | "MemoryDenyWriteExecute=yes", |
| 199 | "SystemCallFilter=@system-service", |
| 200 | "User=carlos", |
| 201 | "Restart=always", |
| 202 | // Delegation is the SOLE mechanism behind exec-instance readings |
| 203 | // fleet-wide (exec-child accounting spec): without it the agent's |
| 204 | // cgroup manager degrades to nil and every exec-backed tenant goes |
| 205 | // blind on the Instances page. |
| 206 | "Delegate=cpu memory io pids", |
| 207 | // The outer wall (memory-pressure spec §2.2) is now on carlos.slice |
| 208 | // (isolated-exec design §4.3). The edge is in the slice, so it still |
| 209 | // participates in the ceiling. OOMPolicy=continue is the line that |
| 210 | // keeps one tenant's OOM kill from becoming a whole-edge restart |
| 211 | // (systemd's default is stop). |
| 212 | "Slice=carlos.slice", |
| 213 | "OOMPolicy=continue", |
| 214 | } { |
| 215 | if !strings.Contains(string(edge), want) { |
| 216 | t.Errorf("carlos-edge.service: missing %q", want) |
| 217 | } |
| 218 | } |
| 219 | |
| 220 | // The outer wall moved from the edge to carlos.slice (isolated-exec |
| 221 | // design §4.3): the slice bounds the edge and every awake instance |
| 222 | // as before, but instances are now sibling units instead of children. |
| 223 | slice, err := os.ReadFile(filepath.Join(unitDir, "carlos.slice")) |
| 224 | if err != nil { |
| 225 | t.Fatal(err) |
| 226 | } |
| 227 | for _, want := range []string{ |
| 228 | "MemoryHigh=70%", |
| 229 | "MemoryMax=80%", |
| 230 | } { |
| 231 | if !strings.Contains(string(slice), want) { |
| 232 | t.Errorf("carlos.slice: missing %q", want) |
| 233 | } |
| 234 | } |
| 235 | // The counterpart of Delegate=: ProtectControlGroups would mount the |
| 236 | // cgroup tree read-only and silently un-arm delegation. Its hardening is |
| 237 | // retained by ownership (DAC) instead — see the unit file's comment. A |
| 238 | // future "hardening cleanup" that re-adds it must fail here, not on the |
| 239 | // fleet. |
| 240 | if strings.Contains(string(edge), "ProtectControlGroups=") { |
| 241 | t.Error("carlos-edge.service: the ProtectControlGroups= directive is deliberately absent (Delegate= needs the subtree writable); do not re-add it") |
| 242 | } |
| 243 | |
| 244 | // The tenant socket unit: systemd owns the listener as carlos so the |
| 245 | // edge can connect and tenants can't reach each other (the whole point). |
| 246 | sockUnit, err := os.ReadFile(filepath.Join(unitDir, "carlos-app@.socket")) |
| 247 | if err != nil { |
| 248 | t.Fatal(err) |
| 249 | } |
| 250 | for _, want := range []string{ |
| 251 | "ListenStream=/run/carlos/apps/%i.sock", |
| 252 | "SocketUser=carlos", |
| 253 | "SocketMode=0600", |
| 254 | "WantedBy=sockets.target", |
| 255 | } { |
| 256 | if !strings.Contains(string(sockUnit), want) { |
| 257 | t.Errorf("carlos-app@.socket: missing %q", want) |
| 258 | } |
| 259 | } |
| 260 | |
| 261 | // The config fast-convergence relay (issue #44). The invariant worth a |
| 262 | // test is WHICH unit the path starts: carlos-adopt.service also runs |
| 263 | // release.Pass and the canary-TTL nag, so aiming a per-config-write |
| 264 | // trigger at it would let one app's `carlos env set` cut over another |
| 265 | // app's pending release seconds later and nag on every write instead of |
| 266 | // hourly. The relay must reach the config half alone. |
| 267 | pathUnit, err := os.ReadFile(filepath.Join(unitDir, "carlos-adopt.path")) |
| 268 | if err != nil { |
| 269 | t.Fatal(err) |
| 270 | } |
| 271 | for _, want := range []string{ |
| 272 | "PathChanged=/data/config-sync-requested", |
| 273 | "Unit=carlos-config-sync.service", |
| 274 | "WantedBy=multi-user.target", |
| 275 | } { |
| 276 | if !strings.Contains(string(pathUnit), want) { |
| 277 | t.Errorf("carlos-adopt.path: missing %q", want) |
| 278 | } |
| 279 | } |
| 280 | if strings.Contains(string(pathUnit), "Unit=carlos-adopt.service") { |
| 281 | t.Error("carlos-adopt.path must not start carlos-adopt.service — that unit adopts releases and nags on expired canaries") |
| 282 | } |
| 283 | |
| 284 | // The release fast-convergence relay, and the same invariant: which unit |
| 285 | // the path starts. carlos-adopt.service prints the canary-TTL nag, an |
| 286 | // HOURLY visibility feature; this trigger fires on agent request, so |
| 287 | // aiming it there would repeat the same expired-canary lines every time a |
| 288 | // promotion could not be staged. |
| 289 | stagePath, err := os.ReadFile(filepath.Join(unitDir, "carlos-stage.path")) |
| 290 | if err != nil { |
| 291 | t.Fatal(err) |
| 292 | } |
| 293 | for _, want := range []string{ |
| 294 | "PathChanged=/data/stage-requested", |
| 295 | "Unit=carlos-stage.service", |
| 296 | "WantedBy=multi-user.target", |
| 297 | } { |
| 298 | if !strings.Contains(string(stagePath), want) { |
| 299 | t.Errorf("carlos-stage.path: missing %q", want) |
| 300 | } |
| 301 | } |
| 302 | if strings.Contains(string(stagePath), "Unit=carlos-adopt.service") { |
| 303 | t.Error("carlos-stage.path must not start carlos-adopt.service — that unit nags on expired canaries, and this trigger is per-request") |
| 304 | } |
| 305 | |
| 306 | // The staging pass must run -stage-only, and must NOT be startable by |
| 307 | // anything but its path unit. |
| 308 | stageSvc, err := os.ReadFile(filepath.Join(unitDir, "carlos-stage.service")) |
| 309 | if err != nil { |
| 310 | t.Fatal(err) |
| 311 | } |
| 312 | if !strings.Contains(string(stageSvc), "carlos adopt --stage-only") { |
| 313 | t.Error("carlos-stage.service must run `carlos adopt --stage-only` — the full pass would nag per request") |
| 314 | } |
| 315 | // A SECTION header, not any mention: the unit's own comment explains why |
| 316 | // it has no [Install], and a substring match would fail on the |
| 317 | // explanation. Same shape as carlos-config-sync.service, which documents |
| 318 | // the same choice the same way. |
| 319 | for _, line := range strings.Split(string(stageSvc), "\n") { |
| 320 | if strings.TrimSpace(line) == "[Install]" { |
| 321 | t.Error("carlos-stage.service must have no [Install] section: only carlos-stage.path should ever start it") |
| 322 | } |
| 323 | } |
| 324 | |
| 325 | syncUnit, err := os.ReadFile(filepath.Join(unitDir, "carlos-config-sync.service")) |
| 326 | if err != nil { |
| 327 | t.Fatal(err) |
| 328 | } |
| 329 | for _, want := range []string{ |
| 330 | "ExecStart=/opt/carlos/carlos adopt --config-only", |
| 331 | "Type=oneshot", |
| 332 | "EnvironmentFile=-/etc/carlos/host.env", |
| 333 | // A path-triggered unit must never latch into start-limit-hit after a |
| 334 | // burst of config writes and then ignore every later trigger. |
| 335 | "StartLimitIntervalSec=0", |
| 336 | } { |
| 337 | if !strings.Contains(string(syncUnit), want) { |
| 338 | t.Errorf("carlos-config-sync.service: missing %q", want) |
| 339 | } |
| 340 | } |
| 341 | // Started only by the path unit: an [Install] section would invite an |
| 342 | // operator to enable a oneshot that has nothing to do at boot. Matched |
| 343 | // as a whole line, since the unit's own comment names the section. |
| 344 | for _, line := range strings.Split(string(syncUnit), "\n") { |
| 345 | if strings.TrimSpace(line) == "[Install]" { |
| 346 | t.Error("carlos-config-sync.service must have no [Install] section — carlos-adopt.path is its only starter") |
| 347 | } |
| 348 | } |
| 349 | |
| 350 | // The standard dirs. |
| 351 | for _, d := range []string{ |
| 352 | filepath.Join(root, "data"), |
| 353 | filepath.Join(root, "opt", "carlos", "versions"), |
| 354 | filepath.Join(root, "opt", "carlos", "live"), |
| 355 | filepath.Join(root, "etc", "carlos"), |
| 356 | filepath.Join(root, "run", "carlos"), |
| 357 | filepath.Join(root, "run", "carlos", "apps"), |
| 358 | } { |
| 359 | fi, err := os.Stat(d) |
| 360 | if err != nil { |
| 361 | t.Errorf("dir not created: %v", err) |
| 362 | continue |
| 363 | } |
| 364 | if !fi.IsDir() { |
| 365 | t.Errorf("%s: not a directory", d) |
| 366 | } |
| 367 | } |
| 368 | |
| 369 | // The tmpfiles.d entry that keeps /run/carlos across reboots. |
| 370 | tf, err := os.ReadFile(filepath.Join(root, "etc", "tmpfiles.d", "carlos.conf")) |
| 371 | if err != nil { |
| 372 | t.Fatalf("tmpfiles conf: %v", err) |
| 373 | } |
| 374 | if !strings.Contains(string(tf), "/run/carlos") { |
| 375 | t.Errorf("tmpfiles conf %q does not mention /run/carlos", tf) |
| 376 | } |
| 377 | if !strings.Contains(string(tf), "/run/carlos/apps") { |
| 378 | t.Errorf("tmpfiles conf %q does not keep the tenant socket dir", tf) |
| 379 | } |
| 380 | } |
| 381 | |
| 382 | // TestResolveStatusBucketDefaultsToDeploymentBucket pins the I2 fix: the |
| 383 | // status loop and the deployment (release-adoption) loop must agree on which |
| 384 | // bucket to serve status out of by default, since the edge/console read |
| 385 | // status/ from the deployment bucket — an explicit CARLOS_STATUS_BUCKET |
| 386 | // still wins when set, and a CARLOS_DEPLOYMENT_DIR-only deployment (no S3 |
| 387 | // bucket to fall back to) gets no default at all. |
| 388 | func TestResolveStatusBucketDefaultsToDeploymentBucket(t *testing.T) { |
| 389 | env := func(m map[string]string) func(string) string { |
| 390 | return func(k string) string { return m[k] } |
| 391 | } |
| 392 | |
| 393 | cases := []struct { |
| 394 | name string |
| 395 | env map[string]string |
| 396 | want string |
| 397 | }{ |
| 398 | {"explicit status bucket wins", map[string]string{ |
| 399 | "CARLOS_STATUS_BUCKET": "status-bucket", "CARLOS_DEPLOYMENT_BUCKET": "deploy-bucket"}, "status-bucket"}, |
| 400 | {"falls back to deployment bucket when status unset", map[string]string{ |
| 401 | "CARLOS_DEPLOYMENT_BUCKET": "deploy-bucket"}, "deploy-bucket"}, |
| 402 | {"neither set is still neither set", map[string]string{}, ""}, |
| 403 | {"deployment dir gives no fallback (S3-only)", map[string]string{ |
| 404 | "CARLOS_DEPLOYMENT_DIR": "/data/deploy"}, ""}, |
| 405 | } |
| 406 | for _, c := range cases { |
| 407 | t.Run(c.name, func(t *testing.T) { |
| 408 | if got := resolveStatusBucket(env(c.env)); got != c.want { |
| 409 | t.Errorf("resolveStatusBucket(%v) = %q, want %q", c.env, got, c.want) |
| 410 | } |
| 411 | }) |
| 412 | } |
| 413 | } |
| 414 | |
| 415 | // ---- Task 8: statusTick — the loop's pure part ----------------------------- |
| 416 | |
| 417 | // TestStatusTickWritesBoxAndPublishesOnlyWhenAsked is the plan's Step-1 test: |
| 418 | // statusTick against a DirStore + fixture registry must always land the |
| 419 | // box's own snapshot under status/boxes/, but only land the merged |
| 420 | // status/fleet.json (+ the embedded page assets PublishFleet ships beside |
| 421 | // it) when publish=true — publish=false is the "another box holds the |
| 422 | // lease" tick, which must still self-report without touching the fleet-wide |
| 423 | // objects. |
| 424 | func TestStatusTickWritesBoxAndPublishesOnlyWhenAsked(t *testing.T) { |
| 425 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 426 | if err != nil { |
| 427 | t.Fatal(err) |
| 428 | } |
| 429 | if err := reg.Upsert(registry.Route{Host: "a.example.com", Kind: "static"}); err != nil { |
| 430 | t.Fatal(err) |
| 431 | } |
| 432 | |
| 433 | store := release.NewDirStore(t.TempDir()) |
| 434 | probes := status.Probes{MachineID: "test-machine"} |
| 435 | ctx := context.Background() |
| 436 | |
| 437 | if err := statusTick(ctx, store, reg, probes, false, ""); err != nil { |
| 438 | t.Fatalf("statusTick (publish=false): %v", err) |
| 439 | } |
| 440 | boxes, err := store.List(ctx, "status/boxes/") |
| 441 | if err != nil { |
| 442 | t.Fatalf("List status/boxes/: %v", err) |
| 443 | } |
| 444 | if len(boxes) != 1 { |
| 445 | t.Fatalf("status/boxes/ = %v, want exactly one snapshot", boxes) |
| 446 | } |
| 447 | if _, _, err := store.Get(ctx, "status/fleet.json"); err == nil { |
| 448 | t.Error("publish=false must not write status/fleet.json") |
| 449 | } |
| 450 | |
| 451 | if err := statusTick(ctx, store, reg, probes, true, ""); err != nil { |
| 452 | t.Fatalf("statusTick (publish=true): %v", err) |
| 453 | } |
| 454 | for _, key := range []string{"status/fleet.json", "status/index.html", "status/status.css"} { |
| 455 | if _, _, err := store.Get(ctx, key); err != nil { |
| 456 | t.Errorf("publish=true must write %s: %v", key, err) |
| 457 | } |
| 458 | } |
| 459 | // The box snapshot count must stay at one: the second tick republishes |
| 460 | // the SAME box's key (WriteBox is last-writer-wins on the label), not a |
| 461 | // second file. |
| 462 | boxes, err = store.List(ctx, "status/boxes/") |
| 463 | if err != nil { |
| 464 | t.Fatalf("List status/boxes/ after publish tick: %v", err) |
| 465 | } |
| 466 | if len(boxes) != 1 { |
| 467 | t.Errorf("status/boxes/ = %v after second tick, want still exactly one snapshot", boxes) |
| 468 | } |
| 469 | } |
| 470 | |
| 471 | // ---- Task 5 (customer-fleets plan): publisher guard + fleet-prefixed store - |
| 472 | |
| 473 | // TestShouldSkipPublisherElection pins the CARLOS_FLEET contract: a fleet |
| 474 | // box (CARLOS_FLEET set to anything non-empty) must skip the publisher |
| 475 | // election, and a platform box (unset, the every-deployment-today case) |
| 476 | // must not. statusLoop checks this BEFORE calling lease.Acquire — see the |
| 477 | // call site — so this also documents that the decision needs no lease, no |
| 478 | // bucket, no network at all: it's a pure function of one env var. |
| 479 | func TestShouldSkipPublisherElection(t *testing.T) { |
| 480 | env := func(m map[string]string) func(string) string { |
| 481 | return func(k string) string { return m[k] } |
| 482 | } |
| 483 | cases := []struct { |
| 484 | name string |
| 485 | env map[string]string |
| 486 | want bool |
| 487 | }{ |
| 488 | {"fleet box skips the election", map[string]string{"CARLOS_FLEET": "pi"}, true}, |
| 489 | {"platform box still elects", map[string]string{}, false}, |
| 490 | {"empty CARLOS_FLEET is not set", map[string]string{"CARLOS_FLEET": ""}, false}, |
| 491 | } |
| 492 | for _, c := range cases { |
| 493 | t.Run(c.name, func(t *testing.T) { |
| 494 | if got := shouldSkipPublisherElection(env(c.env)); got != c.want { |
| 495 | t.Errorf("shouldSkipPublisherElection(%v) = %v, want %v", c.env, got, c.want) |
| 496 | } |
| 497 | }) |
| 498 | } |
| 499 | } |
| 500 | |
| 501 | // TestFleetBoxHeartbeatLandsUnderFleetPrefixNeverPublishes is the end-to-end |
| 502 | // pin for Task 5's whole contract, composing release.NewPrefixStore (whose |
| 503 | // own contract is pinned in isolation by internal/release's |
| 504 | // TestNewPrefixStoreRebasesKeys) with statusTick: a fleet box's tick (store |
| 505 | // wrapped in CARLOS_STATUS_PREFIX="fleets/pi/", publish forced false the way |
| 506 | // shouldSkipPublisherElection drives statusLoop) writes ITS OWN Box |
| 507 | // snapshot at "fleets/pi/status/boxes/<label>.json" in the underlying |
| 508 | // bucket — never at the shared "status/boxes/<label>.json" a platform box |
| 509 | // uses, and never status/fleet.json either. |
| 510 | // |
| 511 | // Critical 1b (Opus review, fix round, controller ruling): <label> here MUST |
| 512 | // be the fleet box's operator-chosen fleet_boxes.label — resolveFleetIdentity's |
| 513 | // result, passed as statusTick's labelOverride — NEVER |
| 514 | // status.Label(probes.MachineID)'s hashed pseudonym. The public status |
| 515 | // page's anonymization rationale for that hash doesn't apply to a path |
| 516 | // (fleets/<fleet>/status/boxes/…) no public page ever reads; using the |
| 517 | // hashed value here would leave the heartbeat filename unrelated to the |
| 518 | // name anything else (the edge's tunnel hub, a fleet binding) calls this |
| 519 | // box by. |
| 520 | func TestFleetBoxHeartbeatLandsUnderFleetPrefixNeverPublishes(t *testing.T) { |
| 521 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 522 | if err != nil { |
| 523 | t.Fatal(err) |
| 524 | } |
| 525 | if err := reg.Upsert(registry.Route{Host: "a.example.com", Kind: "static"}); err != nil { |
| 526 | t.Fatal(err) |
| 527 | } |
| 528 | |
| 529 | under := release.NewDirStore(t.TempDir()) |
| 530 | fleetStore := release.NewPrefixStore(under, "fleets/pi/") |
| 531 | ctx := context.Background() |
| 532 | // MachineID deliberately differs from the fleet-box label passed below — |
| 533 | // proving the filename tracks the identity override, not the probe. |
| 534 | probes := status.Probes{MachineID: "some-hashed-machine-id"} |
| 535 | |
| 536 | // publish=false is what shouldSkipPublisherElection forces statusLoop |
| 537 | // into for a fleet box — lease.Acquire is never even called. |
| 538 | const fleetBoxLabel = "amber-gliding" |
| 539 | if err := statusTick(ctx, fleetStore, reg, probes, false, fleetBoxLabel); err != nil { |
| 540 | t.Fatalf("statusTick: %v", err) |
| 541 | } |
| 542 | |
| 543 | // The filename is pinned to the CHANNEL label, not a hash of MachineID. |
| 544 | if _, _, err := under.Get(ctx, "fleets/pi/status/boxes/"+fleetBoxLabel+".json"); err != nil { |
| 545 | t.Fatalf("Get fleets/pi/status/boxes/%s.json: %v", fleetBoxLabel, err) |
| 546 | } |
| 547 | |
| 548 | boxes, err := under.List(ctx, "fleets/pi/status/boxes/") |
| 549 | if err != nil { |
| 550 | t.Fatal(err) |
| 551 | } |
| 552 | if len(boxes) != 1 { |
| 553 | t.Fatalf("fleets/pi/status/boxes/ = %v, want exactly one snapshot", boxes) |
| 554 | } |
| 555 | |
| 556 | // The shared platform namespace must stay untouched: no box snapshot, |
| 557 | // and definitely no fleet.json (which would only ever appear had |
| 558 | // publish been true). |
| 559 | sharedBoxes, err := under.List(ctx, "status/boxes/") |
| 560 | if err != nil { |
| 561 | t.Fatal(err) |
| 562 | } |
| 563 | if len(sharedBoxes) != 0 { |
| 564 | t.Errorf("status/boxes/ = %v, want none — a fleet box must never write the shared namespace", sharedBoxes) |
| 565 | } |
| 566 | if _, _, err := under.Get(ctx, "status/fleet.json"); err == nil { |
| 567 | t.Error("a fleet box must never publish status/fleet.json") |
| 568 | } |
| 569 | } |
| 570 | |
| 571 | // ---- Task 6 (multi-upstream plan): per-upstream probe state in the snapshot - |
| 572 | |
| 573 | // TestStatusTickCarriesProbedUpstreams is Task 6's Step-1 test: the state the |
| 574 | // edge's probers hold for a multi-upstream route must reach the box's OWN |
| 575 | // snapshot object through the real tick — statusTick over a DirStore, read |
| 576 | // back as the bytes an operator (or the console) would parse — not a |
| 577 | // hand-built Box. |
| 578 | // |
| 579 | // It drives the same seam production does: setUpstreamSource is what RunAgent |
| 580 | // hands the live *edge.Edge's UpstreamStates method to from the edge-ready |
| 581 | // callback, and upstreamStates() is what statusLoop reads on every tick. The |
| 582 | // fixture stands in for the Edge here for the same reason every other probe in |
| 583 | // this file is injected — a real prober would need a live listener and a |
| 584 | // goroutine this package cannot stop. |
| 585 | func TestStatusTickCarriesProbedUpstreams(t *testing.T) { |
| 586 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 587 | if err != nil { |
| 588 | t.Fatal(err) |
| 589 | } |
| 590 | if err := reg.Upsert(registry.Route{Host: "ha.example.com", Kind: "service", Status: "ready"}); err != nil { |
| 591 | t.Fatal(err) |
| 592 | } |
| 593 | |
| 594 | store := release.NewDirStore(t.TempDir()) |
| 595 | ctx := context.Background() |
| 596 | since := time.Now().Add(-90 * time.Second).UTC().Truncate(time.Second) |
| 597 | |
| 598 | setUpstreamSource(func() map[string][]status.UpstreamState { |
| 599 | return map[string][]status.UpstreamState{"ha.example.com": { |
| 600 | {Addr: "10.0.0.11:8080", Up: true, Since: since}, |
| 601 | {Addr: "10.0.0.12:8080", Up: false, Since: since, Fails: 3, LastError: "dial: connection refused"}, |
| 602 | }} |
| 603 | }) |
| 604 | t.Cleanup(func() { setUpstreamSource(nil) }) |
| 605 | |
| 606 | // Exactly what statusLoop builds each tick. |
| 607 | probes := status.Probes{MachineID: "test-machine", Upstreams: upstreamStates()} |
| 608 | if err := statusTick(ctx, store, reg, probes, false, ""); err != nil { |
| 609 | t.Fatalf("statusTick: %v", err) |
| 610 | } |
| 611 | |
| 612 | key := "status/boxes/" + status.Label("test-machine") + ".json" |
| 613 | raw, _, err := store.Get(ctx, key) |
| 614 | if err != nil { |
| 615 | t.Fatalf("Get %s: %v", key, err) |
| 616 | } |
| 617 | var box status.Box |
| 618 | if err := json.Unmarshal(raw, &box); err != nil { |
| 619 | t.Fatalf("unmarshal %s: %v", key, err) |
| 620 | } |
| 621 | if box.Metrics == nil { |
| 622 | t.Fatalf("metrics block missing from %s: %s", key, raw) |
| 623 | } |
| 624 | got := box.Metrics.Upstreams["ha.example.com"] |
| 625 | want := []status.UpstreamState{ |
| 626 | {Addr: "10.0.0.11:8080", Up: true, Since: since}, |
| 627 | {Addr: "10.0.0.12:8080", Up: false, Since: since, Fails: 3, LastError: "dial: connection refused"}, |
| 628 | } |
| 629 | if !reflect.DeepEqual(got, want) { |
| 630 | t.Errorf("upstreams for ha.example.com = %+v, want %+v: %s", got, want, raw) |
| 631 | } |
| 632 | |
| 633 | // Staleness: the map is rebuilt from the prober's State() every tick, so a |
| 634 | // box that stops probing a route (route deleted, addrs cleared) must leave |
| 635 | // NO ghost entry behind in the next snapshot — and with nothing probed at |
| 636 | // all the key must be absent, not an empty object. |
| 637 | setUpstreamSource(func() map[string][]status.UpstreamState { return nil }) |
| 638 | probes.Upstreams = upstreamStates() |
| 639 | if err := statusTick(ctx, store, reg, probes, false, ""); err != nil { |
| 640 | t.Fatalf("statusTick (no probers): %v", err) |
| 641 | } |
| 642 | raw, _, err = store.Get(ctx, key) |
| 643 | if err != nil { |
| 644 | t.Fatalf("Get %s after second tick: %v", key, err) |
| 645 | } |
| 646 | if strings.Contains(string(raw), "ha.example.com") || strings.Contains(string(raw), "upstreams") { |
| 647 | t.Errorf("second snapshot still carries a retired route's probe state: %s", raw) |
| 648 | } |
| 649 | } |
| 650 | |
| 651 | // TestUpstreamStatesUnsetSourceIsNil pins the honest degrade: `carlos edge` |
| 652 | // and every test that never calls setUpstreamSource must read nil, not panic |
| 653 | // — statusLoop calls upstreamStates() on every tick regardless of whether the |
| 654 | // edge-ready callback has fired yet (it runs its first tick a minute in, but |
| 655 | // the ordering is not something this loop gets to assume). |
| 656 | func TestUpstreamStatesUnsetSourceIsNil(t *testing.T) { |
| 657 | setUpstreamSource(nil) |
| 658 | if got := upstreamStates(); got != nil { |
| 659 | t.Errorf("upstreamStates() with no source = %+v, want nil", got) |
| 660 | } |
| 661 | } |
| 662 | |
| 663 | // TestWrapListeningProbeNullsUnitBackedSockets is fix round 1's regression |
| 664 | // test: listeningProbe's raw /proc/net/unix read is confidently WRONG for |
| 665 | // systemd-socket-activated (registry.BackingUnit) routes, since the |
| 666 | // carlos-app@.socket unit holds the kernel-level listener open at all times |
| 667 | // regardless of whether the paired service is actually awake — so a raw |
| 668 | // probe would report "awake" forever for a sleeping unit-backed tenant. This |
| 669 | // exercises the pure seam (unitBackedSockets + wrapListeningProbe) statusLoop |
| 670 | // wires together, with a stand-in raw probe standing in for |
| 671 | // /proc/net/unix's "yes, bound" answer on both sockets — exactly what a real |
| 672 | // kernel-table read would say, since the unit's socket is always bound. |
| 673 | func TestWrapListeningProbeNullsUnitBackedSockets(t *testing.T) { |
| 674 | routes := []registry.Route{ |
| 675 | {Host: "exec-app", Kind: "instance", Hibernate: true, Socket: "/run/carlos/exec-app.sock"}, |
| 676 | {Host: "unit-app", Kind: "instance", Hibernate: true, Socket: "/run/carlos/apps/unit-app.sock", Backing: registry.BackingUnit}, |
| 677 | } |
| 678 | |
| 679 | // rawAlwaysListening stands in for listeningProbe: both sockets read as |
| 680 | // bound/listening, which is the honest kernel-table answer for BOTH |
| 681 | // routes here (the unit-backed one's systemd .socket unit never closes |
| 682 | // its listener) — the wrapper's job is to know that answer can't be |
| 683 | // trusted for the unit-backed one. |
| 684 | rawAlwaysListening := func(string) *bool { v := true; return &v } |
| 685 | wrapped := wrapListeningProbe(unitBackedSockets(routes), rawAlwaysListening) |
| 686 | |
| 687 | if got := wrapped(routes[0].Socket); got == nil || !*got { |
| 688 | t.Errorf("exec-backed socket must pass through to the raw probe unchanged, got %v", got) |
| 689 | } |
| 690 | if got := wrapped(routes[1].Socket); got != nil { |
| 691 | t.Errorf("unit-backed socket must report nil (unknown), got %v — its .socket unit's always-open listener can't distinguish awake from asleep", got) |
| 692 | } |
| 693 | |
| 694 | // One level up, the observable effect on a published snapshot: CollectBox |
| 695 | // nulls the WHOLE box's Awake/Asleep pair the instant any hibernating |
| 696 | // route's Listening call returns nil (snapshot.go's partial-knowledge |
| 697 | // rule — "some known, some not" publishes as null, never a partial |
| 698 | // count). So even though the exec-backed route's own state IS known |
| 699 | // here, the box-wide totals still come back nil because the unit-backed |
| 700 | // route's is not. That's the correct, if blunt, behavior this fix |
| 701 | // produces: a box carrying even one systemd-socket-activated hibernating |
| 702 | // tenant can't yet publish a trustworthy awake/asleep split at all. |
| 703 | b := status.CollectBox(routes, status.Probes{MachineID: "m", Listening: wrapped}, time.Now()) |
| 704 | if b.Routes.Awake != nil || b.Routes.Asleep != nil { |
| 705 | t.Errorf("box awake=%v asleep=%v, want both nil: one unit-backed hibernating route must null the whole box's counts, not just its own", b.Routes.Awake, b.Routes.Asleep) |
| 706 | } |
| 707 | } |
| 708 | |
| 709 | // ---- domains: the reconcile loop's pure config builder ---------------------- |
| 710 | |
| 711 | // TestDomainsTickConfig pins the env contract: PLATFORM_APPS_DOMAIN unset |
| 712 | // means a zero Config (Reconcile is inert on it — the feature does not |
| 713 | // exist); set means every seam is populated, with PLATFORM_EDGE_IPS parsed |
| 714 | // comma-separated and the box's public IP as the fallback. |
| 715 | // |
| 716 | // That parsed list is a RETURN VALUE, not a Config field, and this test |
| 717 | // asserts it there. Config.EdgeIPs is now a func answering the deployment's |
| 718 | // LIVE edge set — every box's published edges file folded against its status |
| 719 | // snapshot — which needs a deployment store, and this builder must stay pure |
| 720 | // and getenv-injectable. domainsLoop unions the two once its store is open, |
| 721 | // so what is pinned here is exactly the ENV half of that union. |
| 722 | // |
| 723 | // It follows that this builder never populates EdgeIPs at all, and that |
| 724 | // Resolve no longer waits on this box knowing an edge address: a box with no |
| 725 | // PLATFORM_EDGE_IPS whose fold supplies edges must still stamp. The |
| 726 | // stand-down when nothing is knowable moved to domains.Reconcile's section 4, |
| 727 | // where the set is actually known — see TestReconcileStandsDownWithNoEdgeIPs. |
| 728 | func TestDomainsTickConfig(t *testing.T) { |
| 729 | env := func(m map[string]string) func(string) string { |
| 730 | return func(k string) string { return m[k] } |
| 731 | } |
| 732 | |
| 733 | if got, envIPs, err := domainsTickConfig(env(map[string]string{}), "9.9.9.9"); !reflect.DeepEqual(got, domains.Config{}) || envIPs != nil || err != nil { |
| 734 | t.Errorf("unset PLATFORM_APPS_DOMAIN: config = %+v, envIPs = %v, err = %v, want zero, nil, nil", got, envIPs, err) |
| 735 | } |
| 736 | |
| 737 | got, envIPs, err := domainsTickConfig(env(map[string]string{ |
| 738 | "PLATFORM_APPS_DOMAIN": "oncarlos.test", |
| 739 | "PLATFORM_EDGE_IPS": "203.0.113.7, 198.51.100.9", |
| 740 | "CARLOS_DATA_DIR": "/custom", |
| 741 | }), "9.9.9.9") |
| 742 | if err != nil { |
| 743 | t.Errorf("configured deployment with no release key: err = %v, want nil", err) |
| 744 | } |
| 745 | if got.AppsDomain != "oncarlos.test" { |
| 746 | t.Errorf("AppsDomain = %q", got.AppsDomain) |
| 747 | } |
| 748 | if !reflect.DeepEqual(envIPs, []string{"203.0.113.7", "198.51.100.9"}) { |
| 749 | t.Errorf("envIPs = %v, want the parsed PLATFORM_EDGE_IPS (public IP must not override an explicit list)", envIPs) |
| 750 | } |
| 751 | if got.EdgeIPs != nil { |
| 752 | t.Error("EdgeIPs must be left nil here: only domainsLoop, which has the store, can answer the live set") |
| 753 | } |
| 754 | if got.AcmeDir != filepath.Join("/custom", "acme") { |
| 755 | t.Errorf("AcmeDir = %q, want the edge's cache dir under CARLOS_DATA_DIR", got.AcmeDir) |
| 756 | } |
| 757 | if got.Resolve == nil || got.Now == nil { |
| 758 | t.Error("Resolve/Now seams not populated on a configured deployment") |
| 759 | } |
| 760 | |
| 761 | // No explicit edge IPs: the box's own public IP (IMDS) is the default. |
| 762 | got, envIPs, _ = domainsTickConfig(env(map[string]string{"PLATFORM_APPS_DOMAIN": "oncarlos.test"}), "9.9.9.9") |
| 763 | if !reflect.DeepEqual(envIPs, []string{"9.9.9.9"}) { |
| 764 | t.Errorf("envIPs = %v, want the public-IP fallback", envIPs) |
| 765 | } |
| 766 | if got.AcmeDir != filepath.Join("/data", "acme") { |
| 767 | t.Errorf("AcmeDir = %q, want the /data default", got.AcmeDir) |
| 768 | } |
| 769 | |
| 770 | // No edge IP knowable from the ENVIRONMENT at all. The env half is empty |
| 771 | // — but Resolve is still armed, because the fold may well know edges this |
| 772 | // box's variables never named. Minting runs regardless. |
| 773 | got, envIPs, _ = domainsTickConfig(env(map[string]string{"PLATFORM_APPS_DOMAIN": "oncarlos.test"}), "") |
| 774 | if envIPs != nil { |
| 775 | t.Errorf("envIPs = %v, want nothing: neither PLATFORM_EDGE_IPS nor IMDS answered", envIPs) |
| 776 | } |
| 777 | if got.Resolve == nil { |
| 778 | t.Error("Resolve must stay armed with no env edge IPs — the fold, not this box's variables, decides whether a pass can stamp") |
| 779 | } |
| 780 | if got.AppsDomain != "oncarlos.test" { |
| 781 | t.Error("minting must stay configured even without edge IPs") |
| 782 | } |
| 783 | } |
| 784 | |
| 785 | // TestDomainsTickConfigReleasePubKey pins the auto-purge's arming condition. |
| 786 | // The reconcile pass destroys objects only when it can VERIFY an app's |
| 787 | // production flag, so the key is what switches purging on — and a missing or |
| 788 | // unparseable one leaves it off rather than failing the loop, because hiding |
| 789 | // a trashed app (reversible) must keep working on a box that may not destroy |
| 790 | // anything (irreversible). |
| 791 | func TestDomainsTickConfigReleasePubKey(t *testing.T) { |
| 792 | env := func(m map[string]string) func(string) string { |
| 793 | return func(k string) string { return m[k] } |
| 794 | } |
| 795 | _, pubB64, err := release.GenerateKey() |
| 796 | if err != nil { |
| 797 | t.Fatal(err) |
| 798 | } |
| 799 | got, _, err := domainsTickConfig(env(map[string]string{ |
| 800 | "PLATFORM_APPS_DOMAIN": "oncarlos.test", |
| 801 | "CARLOS_RELEASE_PUBKEY": pubB64, |
| 802 | }), "9.9.9.9") |
| 803 | if err != nil { |
| 804 | t.Fatalf("a good key returned err = %v", err) |
| 805 | } |
| 806 | want, err := release.ParsePublicKey(pubB64, "test") |
| 807 | if err != nil { |
| 808 | t.Fatal(err) |
| 809 | } |
| 810 | if !got.ReleasePub.Equal(want) { |
| 811 | t.Errorf("ReleasePub = %x, want %x", got.ReleasePub, want) |
| 812 | } |
| 813 | |
| 814 | // UNSET and UNPARSEABLE both leave purging off, and the caller must be |
| 815 | // able to tell them apart — release.ParsePublicKey says the same sentence |
| 816 | // for both, so the distinction lives here: unset is (nil, nil), a typo is |
| 817 | // (nil, error). domainsLoop logs the error verbatim, which is the one line |
| 818 | // an operator reads when purging silently is not happening. |
| 819 | for _, key := range []string{"", " "} { |
| 820 | got, _, err = domainsTickConfig(env(map[string]string{ |
| 821 | "PLATFORM_APPS_DOMAIN": "oncarlos.test", |
| 822 | "CARLOS_RELEASE_PUBKEY": key, |
| 823 | }), "9.9.9.9") |
| 824 | if got.ReleasePub != nil || err != nil { |
| 825 | t.Errorf("CARLOS_RELEASE_PUBKEY=%q: ReleasePub = %x, err = %v — unset must be silent, not an error", key, got.ReleasePub, err) |
| 826 | } |
| 827 | } |
| 828 | for _, key := range []string{"not-base64!!", base64.StdEncoding.EncodeToString([]byte("short"))} { |
| 829 | got, _, err = domainsTickConfig(env(map[string]string{ |
| 830 | "PLATFORM_APPS_DOMAIN": "oncarlos.test", |
| 831 | "CARLOS_RELEASE_PUBKEY": key, |
| 832 | }), "9.9.9.9") |
| 833 | if got.ReleasePub != nil { |
| 834 | t.Errorf("CARLOS_RELEASE_PUBKEY=%q armed purging with %x", key, got.ReleasePub) |
| 835 | } |
| 836 | if err == nil { |
| 837 | t.Errorf("CARLOS_RELEASE_PUBKEY=%q returned no error — a typo would be logged as \"not set\"", key) |
| 838 | } else if !strings.Contains(err.Error(), "CARLOS_RELEASE_PUBKEY") { |
| 839 | t.Errorf("CARLOS_RELEASE_PUBKEY=%q: err = %v, want it to name the variable", key, err) |
| 840 | } |
| 841 | // The rest of the pass is unaffected: hiding trashed apps is |
| 842 | // reversible and must not stop because a key is mistyped. |
| 843 | if got.AppsDomain != "oncarlos.test" || got.Resolve == nil { |
| 844 | t.Errorf("CARLOS_RELEASE_PUBKEY=%q disabled the whole pass: %+v", key, got) |
| 845 | } |
| 846 | } |
| 847 | } |
| 848 | |
| 849 | // ---- instance provisioning: the reconcile loop's pure config builder ------ |
| 850 | |
| 851 | // TestProvisionTickConfig pins the CARLOS_REGION/CARLOS_POOLS env contract. |
| 852 | // Unlike domainsTickConfig, there is no inert zero-Config case: an unset |
| 853 | // region/pools is a legal, still-armed state — see provision.Reconcile's |
| 854 | // doc comment (a box with no region still serves ""-scoped records) — so |
| 855 | // DataDir always defaults to /data regardless of whether region/pools are |
| 856 | // set. |
| 857 | func TestProvisionTickConfig(t *testing.T) { |
| 858 | env := func(m map[string]string) func(string) string { |
| 859 | return func(k string) string { return m[k] } |
| 860 | } |
| 861 | |
| 862 | got := provisionTickConfig(env(map[string]string{})) |
| 863 | if got.Region != "" { |
| 864 | t.Errorf("unset CARLOS_REGION: Region = %q, want empty", got.Region) |
| 865 | } |
| 866 | if got.Pools != nil { |
| 867 | t.Errorf("unset CARLOS_POOLS: Pools = %v, want nil", got.Pools) |
| 868 | } |
| 869 | if got.DataDir != "/data" { |
| 870 | t.Errorf("DataDir = %q, want /data default", got.DataDir) |
| 871 | } |
| 872 | if got.Now == nil { |
| 873 | t.Error("Now seam not populated") |
| 874 | } |
| 875 | |
| 876 | got = provisionTickConfig(env(map[string]string{ |
| 877 | "CARLOS_REGION": "ie", |
| 878 | "CARLOS_POOLS": "tito, shared", |
| 879 | })) |
| 880 | if got.Region != "ie" { |
| 881 | t.Errorf("Region = %q, want ie", got.Region) |
| 882 | } |
| 883 | if !reflect.DeepEqual(got.Pools, []string{"tito", "shared"}) { |
| 884 | t.Errorf("Pools = %v, want [tito shared] (whitespace trimmed)", got.Pools) |
| 885 | } |
| 886 | |
| 887 | // Whitespace-only and empty entries between commas are dropped, not |
| 888 | // preserved as "". |
| 889 | got = provisionTickConfig(env(map[string]string{ |
| 890 | "CARLOS_POOLS": " , tito ,, shared ,", |
| 891 | })) |
| 892 | if !reflect.DeepEqual(got.Pools, []string{"tito", "shared"}) { |
| 893 | t.Errorf("Pools = %v, want empties dropped and whitespace trimmed", got.Pools) |
| 894 | } |
| 895 | |
| 896 | got = provisionTickConfig(env(map[string]string{"CARLOS_DATA_DIR": "/custom"})) |
| 897 | if got.DataDir != "/custom" { |
| 898 | t.Errorf("DataDir = %q, want /custom", got.DataDir) |
| 899 | } |
| 900 | |
| 901 | // Region set with no pools: Pools stays nil (the shared "" pool only — |
| 902 | // NOT "every pool"). |
| 903 | got = provisionTickConfig(env(map[string]string{"CARLOS_REGION": "ie"})) |
| 904 | if got.Region != "ie" || got.Pools != nil { |
| 905 | t.Errorf("Region = %q Pools = %v, want ie / nil", got.Region, got.Pools) |
| 906 | } |
| 907 | |
| 908 | // The release pubkey (#340): present and parseable arms the production |
| 909 | // guard on trash-driven route retirement. |
| 910 | if got.ReleasePub != nil { |
| 911 | t.Error("ReleasePub set with CARLOS_RELEASE_PUBKEY unset") |
| 912 | } |
| 913 | _, pubB64, err := release.GenerateKey() |
| 914 | if err != nil { |
| 915 | t.Fatal(err) |
| 916 | } |
| 917 | got = provisionTickConfig(env(map[string]string{"CARLOS_RELEASE_PUBKEY": pubB64})) |
| 918 | if got.ReleasePub == nil { |
| 919 | t.Error("ReleasePub nil with a valid CARLOS_RELEASE_PUBKEY") |
| 920 | } |
| 921 | |
| 922 | // A key SET but unparseable must not be mistaken for a key that parsed. |
| 923 | // It lands on nil — the same unguarded end state domains.Config accepts |
| 924 | // for the same decision — and says so in the log rather than silently. |
| 925 | var logged bytes.Buffer |
| 926 | log.SetOutput(&logged) |
| 927 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 928 | got = provisionTickConfig(env(map[string]string{"CARLOS_RELEASE_PUBKEY": "not-a-key"})) |
| 929 | if got.ReleasePub != nil { |
| 930 | t.Error("ReleasePub set from an unparseable CARLOS_RELEASE_PUBKEY") |
| 931 | } |
| 932 | if !strings.Contains(logged.String(), "CARLOS_RELEASE_PUBKEY") { |
| 933 | t.Errorf("an unparseable key was swallowed; log was:\n%s", logged.String()) |
| 934 | } |
| 935 | } |
| 936 | |
| 937 | // TestProvisionTickConfigReadsInbound pins CARLOS_INBOUND's contract: an |
| 938 | // unset var is the SAME apps-domain-style inert gate CARLOS_REGION/ |
| 939 | // CARLOS_POOLS already use — a legal "no constraint" state, never a |
| 940 | // "feature not configured" one — so it must read back as "" (today's |
| 941 | // behaviour, byte for byte), and only an explicit "none" changes Config. |
| 942 | func TestProvisionTickConfigReadsInbound(t *testing.T) { |
| 943 | env := func(m map[string]string) func(string) string { |
| 944 | return func(k string) string { return m[k] } |
| 945 | } |
| 946 | |
| 947 | got := provisionTickConfig(env(map[string]string{ |
| 948 | "CARLOS_REGION": "ie", "CARLOS_POOLS": "eleven", "CARLOS_INBOUND": "none", |
| 949 | })) |
| 950 | if got.Inbound != "none" { |
| 951 | t.Errorf("Inbound = %q, want none", got.Inbound) |
| 952 | } |
| 953 | |
| 954 | got = provisionTickConfig(env(map[string]string{})) |
| 955 | if got.Inbound != "" { |
| 956 | t.Fatal("an unset CARLOS_INBOUND must be an inert default — today's behaviour exactly") |
| 957 | } |
| 958 | |
| 959 | // Review fix (Task 10 finding): provisionTickConfig must fold case and |
| 960 | // whitespace, and must read an unrecognised value as restrictive rather |
| 961 | // than letting it silently fall through as "any". These pin the actual |
| 962 | // production reading path (getenv -> provision.NormalizeInbound), not |
| 963 | // just the pure function in the provision package. |
| 964 | got = provisionTickConfig(env(map[string]string{"CARLOS_INBOUND": " NONE "})) |
| 965 | if got.Inbound != "none" { |
| 966 | t.Errorf("Inbound = %q, want none (case/whitespace folded)", got.Inbound) |
| 967 | } |
| 968 | got = provisionTickConfig(env(map[string]string{"CARLOS_INBOUND": "off"})) |
| 969 | if got.Inbound != "none" { |
| 970 | t.Errorf("Inbound = %q, want none (unrecognised value reads restrictive)", got.Inbound) |
| 971 | } |
| 972 | } |
| 973 | |
| 974 | func TestBootstrapIdempotent(t *testing.T) { |
| 975 | root := t.TempDir() |
| 976 | if err := bootstrap(root, nil); err != nil { |
| 977 | t.Fatalf("first bootstrap: %v", err) |
| 978 | } |
| 979 | if err := bootstrap(root, nil); err != nil { |
| 980 | t.Fatalf("second bootstrap: %v", err) |
| 981 | } |
| 982 | // Still all present after the re-run. |
| 983 | for _, name := range expectedUnits { |
| 984 | if _, err := os.Stat(filepath.Join(root, "etc", "systemd", "system", name)); err != nil { |
| 985 | t.Errorf("after re-run: %v", err) |
| 986 | } |
| 987 | } |
| 988 | } |
| 989 | |
| 990 | func TestIsCanaryRoute(t *testing.T) { |
| 991 | cases := []struct { |
| 992 | channel string |
| 993 | want bool |
| 994 | }{ |
| 995 | {"canary/x", true}, |
| 996 | {"canary/pr-142", true}, |
| 997 | {"stable", false}, |
| 998 | {"beta", false}, |
| 999 | {"canary", false}, // no slash — same convention as bake.go's DefaultHolds wildcard |
| 1000 | {"", false}, |
| 1001 | } |
| 1002 | for _, c := range cases { |
| 1003 | rt := registry.Route{Channel: c.channel} |
| 1004 | if got := isCanaryRoute(rt); got != c.want { |
| 1005 | t.Errorf("isCanaryRoute(Channel=%q) = %v, want %v", c.channel, got, c.want) |
| 1006 | } |
| 1007 | } |
| 1008 | } |
| 1009 | |
| 1010 | // ---- Pipelines Task 5: the install path arms timers ------------------------- |
| 1011 | |
| 1012 | // TestSystemctlPlanArmsTimers is the regression test for the flagship's |
| 1013 | // silent-adoption bug: carlos-adopt.timer was enabled — the .wants symlink was |
| 1014 | // right there — but never active, because `systemctl enable` only writes the |
| 1015 | // symlink and the timer does not arm until something starts it or the box |
| 1016 | // reboots. Nothing had rebooted, so no release ever adopted itself and every |
| 1017 | // deploy quietly needed a hand-run `systemctl start carlos-adopt.service`. |
| 1018 | // Every timer bootstrap enables must therefore also be started, and no |
| 1019 | // SERVICE may be (carlos-edge needs its binary and host.env staged first — |
| 1020 | // bootstrap's own closing line tells the operator to start it). The one |
| 1021 | // exemption is carlos-relay.service, whose ConditionPathExists makes the |
| 1022 | // start a no-op on any box the operator has not armed with a region master |
| 1023 | // (relay spec §3); TestBootstrapStartsRelayUnit pins that it IS started. |
| 1024 | // |
| 1025 | // PATH units are the same trap with the same fix: an enabled-but-unstarted |
| 1026 | // carlos-adopt.path watches nothing until the next reboot, so the agent's |
| 1027 | // config-sync marker would be written to a file no one reads and unit-backed |
| 1028 | // apps would silently keep only the hourly floor. |
| 1029 | func TestSystemctlPlanArmsTimers(t *testing.T) { |
| 1030 | plan := systemctlPlan(enabledUnits) |
| 1031 | if len(plan) == 0 || plan[0][0] != "daemon-reload" { |
| 1032 | t.Fatalf("plan must reload units first, got %v", plan) |
| 1033 | } |
| 1034 | |
| 1035 | started := map[string]bool{} |
| 1036 | enabled := map[string]bool{} |
| 1037 | for _, args := range plan { |
| 1038 | for _, unit := range args[1:] { |
| 1039 | switch args[0] { |
| 1040 | case "start": |
| 1041 | started[unit] = true |
| 1042 | case "enable": |
| 1043 | enabled[unit] = true |
| 1044 | } |
| 1045 | } |
| 1046 | } |
| 1047 | for _, u := range enabledUnits { |
| 1048 | if !enabled[u] { |
| 1049 | t.Errorf("%s: enabled by the plan? no — bootstrap must still enable everything it did before", u) |
| 1050 | } |
| 1051 | if (strings.HasSuffix(u, ".timer") || strings.HasSuffix(u, ".path")) && !started[u] { |
| 1052 | t.Errorf("%s: enabled but never started — that is exactly the bug (enable only symlinks; the unit arms at the next boot)", u) |
| 1053 | } |
| 1054 | if strings.HasSuffix(u, ".service") && u != relayUnitName && started[u] { |
| 1055 | t.Errorf("%s: bootstrap must not start a service — the binary and /etc/carlos/host.env may not be staged yet", u) |
| 1056 | } |
| 1057 | } |
| 1058 | } |
| 1059 | |
| 1060 | // TestBootstrapStagingRunsNoSystemctl guards the other half of the same |
| 1061 | // change: `carlos bootstrap -root <dir>` is how the unit-refresh path stages |
| 1062 | // units into a temp root on a LIVE box (see pinfra's |
| 1063 | // update-platform-binary.sh, which stages into /tmp/carlos-unitstage). Arming |
| 1064 | // timers from a staging run would touch the live system from a command whose |
| 1065 | // entire contract is "write files, change nothing" — so a staging run must |
| 1066 | // print what a live one would do and execute none of it. The fake systemctl on |
| 1067 | // PATH records any invocation; the file must never appear. |
| 1068 | func TestBootstrapStagingRunsNoSystemctl(t *testing.T) { |
| 1069 | bin := t.TempDir() |
| 1070 | log := filepath.Join(t.TempDir(), "systemctl-calls") |
| 1071 | script := "#!/bin/sh\necho \"$@\" >> " + log + "\n" |
| 1072 | if err := os.WriteFile(filepath.Join(bin, "systemctl"), []byte(script), 0o755); err != nil { |
| 1073 | t.Fatal(err) |
| 1074 | } |
| 1075 | t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) |
| 1076 | |
| 1077 | if err := bootstrap(t.TempDir(), nil); err != nil { |
| 1078 | t.Fatalf("bootstrap: %v", err) |
| 1079 | } |
| 1080 | if b, err := os.ReadFile(log); err == nil { |
| 1081 | t.Errorf("staging bootstrap ran systemctl: %s", b) |
| 1082 | } |
| 1083 | } |
| 1084 | |
| 1085 | // TestBootstrapTmpfilesBlessesConsumersGroup pins the instance-socket |
| 1086 | // consumers contract (issue #103): /run/carlos carries the dedicated |
| 1087 | // carlos-sockets group with the setgid bit — so every socket an exec child |
| 1088 | // binds inside inherits the consumers group — while /run/carlos/apps stays |
| 1089 | // carlos:carlos (unit tenants get their fd passed; enrollment deliberately |
| 1090 | // does not reach them). The group is dedicated, never `carlos`: membership |
| 1091 | // in `carlos` also grants group-read of /data, i.e. every instance's DB and |
| 1092 | // CARLOS_ADMIN_TOKEN. |
| 1093 | func TestBootstrapTmpfilesBlessesConsumersGroup(t *testing.T) { |
| 1094 | root := t.TempDir() |
| 1095 | if err := bootstrap(root, nil); err != nil { |
| 1096 | t.Fatalf("bootstrap: %v", err) |
| 1097 | } |
| 1098 | tf, err := os.ReadFile(filepath.Join(root, "etc", "tmpfiles.d", "carlos.conf")) |
| 1099 | if err != nil { |
| 1100 | t.Fatalf("tmpfiles conf: %v", err) |
| 1101 | } |
| 1102 | if want := "d /run/carlos 2750 carlos carlos-sockets -"; !strings.Contains(string(tf), want) { |
| 1103 | t.Errorf("tmpfiles conf %q missing %q", tf, want) |
| 1104 | } |
| 1105 | if want := "d /run/carlos/apps 0750 carlos carlos -"; !strings.Contains(string(tf), want) { |
| 1106 | t.Errorf("tmpfiles conf %q missing %q — the tenant socket dir must stay out of the consumers group", tf, want) |
| 1107 | } |
| 1108 | |
| 1109 | // Bootstrap's own dir handling must agree with the tmpfiles line: setgid |
| 1110 | // on /run/carlos (mode is what makes group inheritance real between |
| 1111 | // boots), and NOT on apps/. |
| 1112 | fi, err := os.Stat(filepath.Join(root, "run", "carlos")) |
| 1113 | if err != nil { |
| 1114 | t.Fatal(err) |
| 1115 | } |
| 1116 | if fi.Mode()&os.ModeSetgid == 0 { |
| 1117 | t.Errorf("/run/carlos mode %v: missing setgid — bound sockets would not inherit the consumers group", fi.Mode()) |
| 1118 | } |
| 1119 | if fi.Mode().Perm() != 0o750 { |
| 1120 | t.Errorf("/run/carlos perm %o, want 750", fi.Mode().Perm()) |
| 1121 | } |
| 1122 | afi, err := os.Stat(filepath.Join(root, "run", "carlos", "apps")) |
| 1123 | if err != nil { |
| 1124 | t.Fatal(err) |
| 1125 | } |
| 1126 | if afi.Mode()&os.ModeSetgid != 0 { |
| 1127 | t.Errorf("/run/carlos/apps mode %v: setgid must not leak onto the tenant socket dir", afi.Mode()) |
| 1128 | } |
| 1129 | |
| 1130 | // The sysusers.d snippet is what makes the tmpfiles line safe at boot on |
| 1131 | // a box whose conf arrived via a staged-tree refresh (never a live |
| 1132 | // groupadd): systemd-sysusers runs Before=systemd-tmpfiles-setup, and a |
| 1133 | // tmpfiles line whose group doesn't resolve is DROPPED — /run/carlos then |
| 1134 | // isn't created and carlos-edge (ReadWritePaths, no "-") can't start. |
| 1135 | su, err := os.ReadFile(filepath.Join(root, "etc", "sysusers.d", "carlos.conf")) |
| 1136 | if err != nil { |
| 1137 | t.Fatalf("sysusers conf: %v", err) |
| 1138 | } |
| 1139 | if want := "g carlos-sockets -"; !strings.Contains(string(su), want) { |
| 1140 | t.Errorf("sysusers conf %q missing %q", su, want) |
| 1141 | } |
| 1142 | } |
| 1143 | |
| 1144 | // TestBootstrapPrivilegedPlanEnrollsConsumers pins the printed-equals-run |
| 1145 | // contract for the group half, the same discipline systemctlPlan established: |
| 1146 | // a staging run prints exactly the commands a live run executes, so the two |
| 1147 | // can never drift. -socket-consumer users each get a usermod enrollment into |
| 1148 | // the dedicated group; no consumers means no usermod lines but the group and |
| 1149 | // dir convergence always run. |
| 1150 | func TestBootstrapPrivilegedPlanEnrollsConsumers(t *testing.T) { |
| 1151 | got := privilegedPlan([]string{"titogo", "titogo-relay"}) |
| 1152 | // The /run/carlos chown is recursive AND targets the consumers group: a |
| 1153 | // re-run is the documented enrollment path, and a recursive chown back |
| 1154 | // to carlos:carlos would reset every live instance socket's group, |
| 1155 | // stranding already-enrolled consumers until each instance rebinds. |
| 1156 | // apps/ is put back to carlos:carlos right after (unit tenants stay out |
| 1157 | // of consumer reach). |
| 1158 | want := [][]string{ |
| 1159 | {"useradd", "--system", "--user-group", "--no-create-home", "--shell", "/usr/sbin/nologin", "carlos"}, |
| 1160 | {"groupadd", "--system", "carlos-sockets"}, |
| 1161 | {"chown", "-R", "carlos:carlos", "/data"}, |
| 1162 | {"chown", "-R", "carlos:carlos-sockets", "/run/carlos"}, |
| 1163 | {"chown", "-R", "carlos:carlos", "/run/carlos/apps"}, |
| 1164 | {"chmod", "2750", "/run/carlos"}, |
| 1165 | {"usermod", "-aG", "carlos-sockets", "titogo"}, |
| 1166 | {"usermod", "-aG", "carlos-sockets", "titogo-relay"}, |
| 1167 | } |
| 1168 | if !reflect.DeepEqual(got, want) { |
| 1169 | t.Errorf("privilegedPlan = %v, want %v", got, want) |
| 1170 | } |
| 1171 | // No consumers: same plan minus the usermod tail. |
| 1172 | if got := privilegedPlan(nil); !reflect.DeepEqual(got, want[:6]) { |
| 1173 | t.Errorf("privilegedPlan(nil) = %v, want %v", got, want[:6]) |
| 1174 | } |
| 1175 | } |
| 1176 | |
| 1177 | // TestBootstrapOffcloudRequiresCredentialsFile pins the off-cloud precondition |
| 1178 | // (issue #53 point 4): a box with no EC2 instance profile must have its scoped |
| 1179 | // credentials staged — and staged at 0600 — before bootstrap proceeds. This is |
| 1180 | // the staging-root path (no root privileges available in tests), so the |
| 1181 | // root:service ownership repair never runs; that only happens on a live root. |
| 1182 | func TestBootstrapOffcloudRequiresCredentialsFile(t *testing.T) { |
| 1183 | root := t.TempDir() |
| 1184 | if err := bootstrapOffcloud(root, nil); err == nil { |
| 1185 | t.Fatal("want error when <root>/etc/carlos/credentials is absent") |
| 1186 | } |
| 1187 | credDir := filepath.Join(root, "etc", "carlos") |
| 1188 | os.MkdirAll(credDir, 0o755) |
| 1189 | credFile := filepath.Join(credDir, "credentials") |
| 1190 | os.WriteFile(credFile, []byte("[default]\naws_access_key_id=X\naws_secret_access_key=Y\n"), 0o644) |
| 1191 | if err := bootstrapOffcloud(root, nil); err == nil { |
| 1192 | t.Fatal("want error on 0644 credentials (must be 0600)") |
| 1193 | } |
| 1194 | os.Chmod(credFile, 0o600) |
| 1195 | if err := bootstrapOffcloud(root, nil); err != nil { |
| 1196 | t.Fatalf("offcloud bootstrap with staged 0600 credentials: %v", err) |
| 1197 | } |
| 1198 | // The normal bootstrap artifacts must exist too — offcloud is a superset. |
| 1199 | if _, err := os.Stat(filepath.Join(root, "etc", "systemd", "system", "carlos-edge.service")); err != nil { |
| 1200 | t.Fatalf("units not installed: %v", err) |
| 1201 | } |
| 1202 | } |
| 1203 | |
| 1204 | // TestAdoptTimerStateReading pins the probe's honesty rules at the seam that |
| 1205 | // needs no systemd to test: systemd's own word for the timer's state maps to |
| 1206 | // running/not-running, and anything this probe cannot interpret — no systemd |
| 1207 | // at all, a systemctl that failed to run, a word from a future version — |
| 1208 | // reports nil (unknown), never a reassuring true. |
| 1209 | func TestAdoptTimerStateReading(t *testing.T) { |
| 1210 | cases := []struct { |
| 1211 | out string |
| 1212 | want *bool |
| 1213 | }{ |
| 1214 | {"active\n", boolPtr(true)}, |
| 1215 | {"inactive\n", boolPtr(false)}, |
| 1216 | {"failed\n", boolPtr(false)}, |
| 1217 | {"deactivating\n", boolPtr(false)}, |
| 1218 | {"unknown\n", boolPtr(false)}, // systemd answered: it has no such unit loaded |
| 1219 | {"", nil}, // systemctl never ran (not a systemd box) |
| 1220 | {"something-new\n", nil}, // a word this probe does not understand |
| 1221 | } |
| 1222 | for _, c := range cases { |
| 1223 | got := parseIsActive(c.out) |
| 1224 | switch { |
| 1225 | case c.want == nil && got != nil: |
| 1226 | t.Errorf("parseIsActive(%q) = %v, want nil", c.out, *got) |
| 1227 | case c.want != nil && got == nil: |
| 1228 | t.Errorf("parseIsActive(%q) = nil, want %v", c.out, *c.want) |
| 1229 | case c.want != nil && *got != *c.want: |
| 1230 | t.Errorf("parseIsActive(%q) = %v, want %v", c.out, *got, *c.want) |
| 1231 | } |
| 1232 | } |
| 1233 | } |
| 1234 | |
| 1235 | func boolPtr(v bool) *bool { return &v } |
| 1236 | |
| 1237 | func TestParseMeminfoBytes(t *testing.T) { |
| 1238 | total, avail := parseMeminfoBytes(strings.NewReader( |
| 1239 | "MemTotal: 3961472 kB\nMemFree: 100 kB\nMemAvailable: 1980736 kB\n")) |
| 1240 | if total == nil || *total != 3961472*1024 || avail == nil || *avail != 1980736*1024 { |
| 1241 | t.Fatalf("got %v %v", total, avail) |
| 1242 | } |
| 1243 | if tt, aa := parseMeminfoBytes(strings.NewReader("garbage")); tt != nil || aa != nil { |
| 1244 | t.Fatal("unparseable meminfo must yield nil, not zero") |
| 1245 | } |
| 1246 | } |
| 1247 | |
| 1248 | func TestParseLoad1(t *testing.T) { |
| 1249 | if l := parseLoad1("0.42 0.36 0.30 1/234 5678\n"); l == nil || *l != 0.42 { |
| 1250 | t.Fatalf("got %v", l) |
| 1251 | } |
| 1252 | if l := parseLoad1(""); l != nil { |
| 1253 | t.Fatal("empty loadavg must yield nil") |
| 1254 | } |
| 1255 | } |
| 1256 | |
| 1257 | // TestProbedUnitsAreTheOnesBootstrapArms pins the pairing that makes both |
| 1258 | // systemd probes meaningful: each one must name a unit bootstrap actually |
| 1259 | // installs AND arms. A probe pointed at a unit no box has answers "unknown" |
| 1260 | // (systemd's own word for "no such unit"), which this package maps to a |
| 1261 | // confident false — so a typo here would report every box in the fleet as |
| 1262 | // broken, and a renamed unit would report a live box as dead. |
| 1263 | func TestProbedUnitsAreTheOnesBootstrapArms(t *testing.T) { |
| 1264 | for _, unit := range []string{adoptTimerUnit, configPathUnit, systemUpdatePathUnit} { |
| 1265 | if !slices.Contains(enabledUnits, unit) { |
| 1266 | t.Errorf("%s: probed but not in enabledUnits — bootstrap never arms it", unit) |
| 1267 | } |
| 1268 | if !slices.Contains(expectedUnits, unit) { |
| 1269 | t.Errorf("%s: probed but not an installed unit", unit) |
| 1270 | } |
| 1271 | } |
| 1272 | } |
| 1273 | |
| 1274 | func TestBootstrapPlanJSONNamesWhatALiveRunArms(t *testing.T) { |
| 1275 | p := currentBootstrapPlan() |
| 1276 | want := map[string]bool{"carlos-edge.service": true, "carlos-adopt.timer": true, "carlos-adopt.path": true, |
| 1277 | "carlos-stage.path": true, "carlos-restart.path": true, "carlos-system-update.path": true, "carlos-system-recover.service": true, |
| 1278 | "data-instances.mount": true, "run-carlos-instances.mount": true, "run-carlos-logs.mount": true} |
| 1279 | got := map[string]bool{} |
| 1280 | for _, u := range p.Enabled { |
| 1281 | got[u] = true |
| 1282 | } |
| 1283 | for u := range want { |
| 1284 | if !got[u] { |
| 1285 | t.Errorf("plan lacks %s", u) |
| 1286 | } |
| 1287 | } |
| 1288 | if got["carlos-system-update.service"] { |
| 1289 | t.Error("the path-triggered oneshot must not be enabled: it is started by its .path") |
| 1290 | } |
| 1291 | inert := map[string]bool{} |
| 1292 | for _, u := range p.Inert { |
| 1293 | inert[u] = true |
| 1294 | } |
| 1295 | if !inert["carlos-restore-verify.timer"] || !inert["carlos-hibernate-verify.timer"] { |
| 1296 | t.Errorf("inert = %v", p.Inert) |
| 1297 | } |
| 1298 | // And systemctlPlan starts no service but the relay, whatever is |
| 1299 | // enabled — the relay is safe because its ConditionPathExists gates it |
| 1300 | // on a masters file the operator stages (relay spec §3). |
| 1301 | for _, args := range systemctlPlan(p.Enabled) { |
| 1302 | if args[0] == "start" { |
| 1303 | for _, u := range args[1:] { |
| 1304 | if strings.HasSuffix(u, ".service") && u != relayUnitName { |
| 1305 | t.Errorf("plan starts a service: %v", args) |
| 1306 | } |
| 1307 | } |
| 1308 | } |
| 1309 | } |
| 1310 | } |
| 1311 | |
| 1312 | func TestSystemUnitsAreShapedLikeTheStageRelay(t *testing.T) { |
| 1313 | read := func(name string) string { |
| 1314 | b, err := unitsFS.ReadFile("units/" + name) |
| 1315 | if err != nil { |
| 1316 | t.Fatal(err) |
| 1317 | } |
| 1318 | return string(b) |
| 1319 | } |
| 1320 | p := read("carlos-system-update.path") |
| 1321 | for _, want := range []string{"PathChanged=/data/system-update-requested", "Unit=carlos-system-update.service", "WantedBy=multi-user.target"} { |
| 1322 | if !strings.Contains(p, want) { |
| 1323 | t.Errorf("path unit lacks %q", want) |
| 1324 | } |
| 1325 | } |
| 1326 | u := read("carlos-system-update.service") |
| 1327 | for _, want := range []string{"Type=oneshot", "StartLimitIntervalSec=0", "TimeoutStartSec=15min", "OnFailure=carlos-system-recover.service", |
| 1328 | "EnvironmentFile=-/etc/carlos/host.env", "system apply", "carlos.prev", "MemoryMax=256M", |
| 1329 | "floor.min-release", "carlos.prev.allowed", floorGateCall, "seeds-floor.min-release", "carlos.prev.seeds-allowed", seedsFloorGateCall} { |
| 1330 | if !strings.Contains(u, want) { |
| 1331 | t.Errorf("update unit lacks %q", want) |
| 1332 | } |
| 1333 | } |
| 1334 | if strings.Contains(u, "cmp ") { |
| 1335 | t.Errorf("update unit still compares the floor file with cmp, which follows a symlink and blocks on a FIFO") |
| 1336 | } |
| 1337 | for _, never := range []string{"ProtectSystem", "User="} { |
| 1338 | if strings.Contains(u, never) { |
| 1339 | t.Errorf("update unit must run as root with /opt writable; found %q", never) |
| 1340 | } |
| 1341 | } |
| 1342 | r := read("carlos-system-recover.service") |
| 1343 | for _, want := range []string{"Type=oneshot", "Before=carlos-edge.service", "WantedBy=multi-user.target", "TimeoutStartSec=10min", "system apply --recover", "carlos.prev", |
| 1344 | "floor.min-release", "carlos.prev.allowed", floorGateCall, "seeds-floor.min-release", "carlos.prev.seeds-allowed", seedsFloorGateCall} { |
| 1345 | if !strings.Contains(r, want) { |
| 1346 | t.Errorf("recover unit lacks %q", want) |
| 1347 | } |
| 1348 | } |
| 1349 | if strings.Contains(r, "cmp ") { |
| 1350 | t.Errorf("recover unit still compares the floor file with cmp, which follows a symlink and blocks on a FIFO") |
| 1351 | } |
| 1352 | } |
| 1353 | |
| 1354 | // TestStatusPublishNeverReadsMetricsPrefixes is the privacy tripwire the |
| 1355 | // monitoring spec commits to: operator-grade needles live under metrics/ and |
| 1356 | // alerts/, and nothing the public status publisher writes may ever be |
| 1357 | // derived from them. Sentinel values planted in metrics objects must not |
| 1358 | // appear anywhere under status/ after a full publish tick. |
| 1359 | func TestStatusPublishNeverReadsMetricsPrefixes(t *testing.T) { |
| 1360 | ctx := context.Background() |
| 1361 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 1362 | if err != nil { |
| 1363 | t.Fatal(err) |
| 1364 | } |
| 1365 | store := release.NewDirStore(t.TempDir()) |
| 1366 | sentinel := 0.987654321 |
| 1367 | chunk := metrics.RawChunk{V: 1, Box: "box-a", Hour: "2026-08-02-10", Samples: []metrics.Sample{ |
| 1368 | {At: time.Now(), Host: metrics.HostSample{CPUFrac: &sentinel}}, |
| 1369 | }} |
| 1370 | if err := metrics.WriteRaw(ctx, store, chunk); err != nil { |
| 1371 | t.Fatal(err) |
| 1372 | } |
| 1373 | |
| 1374 | // Plant sentinel in alerts/boxes/ as well |
| 1375 | alertState := metrics.AlertState{ |
| 1376 | V: 1, |
| 1377 | Box: "box-a", |
| 1378 | UpdatedAt: time.Now(), |
| 1379 | Rules: map[string]metrics.RuleState{ |
| 1380 | "disk": {Firing: true, Value: sentinel}, |
| 1381 | }, |
| 1382 | } |
| 1383 | alertBody, err := json.Marshal(alertState) |
| 1384 | if err != nil { |
| 1385 | t.Fatal(err) |
| 1386 | } |
| 1387 | if _, err := store.Put(ctx, metrics.AlertKey("box-a"), alertBody, ""); err != nil { |
| 1388 | t.Fatal(err) |
| 1389 | } |
| 1390 | |
| 1391 | if err := statusTick(ctx, store, reg, status.Probes{MachineID: "m"}, true, ""); err != nil { |
| 1392 | t.Fatal(err) |
| 1393 | } |
| 1394 | keys, err := store.List(ctx, "status/") |
| 1395 | if err != nil { |
| 1396 | t.Fatal(err) |
| 1397 | } |
| 1398 | // Without this the loop below has nothing to inspect and the test passes |
| 1399 | // no matter what statusTick publishes — a tripwire across a doorway |
| 1400 | // nobody walks through. |
| 1401 | if len(keys) == 0 { |
| 1402 | t.Fatal("no status/ keys after a publish tick — the tripwire is not armed") |
| 1403 | } |
| 1404 | for _, k := range keys { |
| 1405 | body, _, err := store.Get(ctx, k) |
| 1406 | if err != nil { |
| 1407 | t.Fatal(err) |
| 1408 | } |
| 1409 | if strings.Contains(string(body), "987654321") { |
| 1410 | t.Errorf("%s contains the metrics sentinel — the status publisher read a metrics object", k) |
| 1411 | } |
| 1412 | } |
| 1413 | } |
| 1414 | |
| 1415 | // ---- Pipelines: canary TTL, the status loop's expired-canary count -------- |
| 1416 | |
| 1417 | // wireWrapper mirrors release's unexported signedWrapper wire shape |
| 1418 | // (payload/sig) — duplicated here rather than imported (it isn't exported, |
| 1419 | // deliberately: forging a pointer's PromotedAt is not something production |
| 1420 | // code anywhere should be able to do) so this test can write a validly |
| 1421 | // SIGNED but arbitrarily-dated channel pointer straight into a DirStore, |
| 1422 | // the only way to get an actually-expired pointer under test since every |
| 1423 | // public promotion path (release.Promote) stamps the real clock. |
| 1424 | type wireWrapper struct { |
| 1425 | Payload string `json:"payload"` |
| 1426 | Sig string `json:"sig"` |
| 1427 | } |
| 1428 | |
| 1429 | // writeBackdatedPointer signs and writes a channel pointer with an |
| 1430 | // arbitrary PromotedAt directly to key release.ChannelKey(acct, app, |
| 1431 | // channel), bypassing Promote's clock stamp. |
| 1432 | func writeBackdatedPointer(t *testing.T, s release.Store, priv ed25519.PrivateKey, acct, app, channel, version string, promotedAt time.Time) { |
| 1433 | t.Helper() |
| 1434 | p := struct { |
| 1435 | Schema int `json:"schema"` |
| 1436 | App string `json:"app"` |
| 1437 | Channel string `json:"channel"` |
| 1438 | Version string `json:"version"` |
| 1439 | PromotedAt time.Time `json:"promoted_at"` |
| 1440 | }{Schema: 1, App: app, Channel: channel, Version: version, PromotedAt: promotedAt} |
| 1441 | payload, err := json.Marshal(p) |
| 1442 | if err != nil { |
| 1443 | t.Fatalf("marshal backdated pointer: %v", err) |
| 1444 | } |
| 1445 | b64 := base64.StdEncoding.EncodeToString(payload) |
| 1446 | sig := ed25519.Sign(priv, []byte(b64)) |
| 1447 | wrapper, err := json.Marshal(wireWrapper{Payload: b64, Sig: base64.StdEncoding.EncodeToString(sig)}) |
| 1448 | if err != nil { |
| 1449 | t.Fatalf("marshal backdated wrapper: %v", err) |
| 1450 | } |
| 1451 | if _, err := s.Put(context.Background(), release.ChannelKey(acct, app, channel), wrapper, ""); err != nil { |
| 1452 | t.Fatalf("write backdated pointer: %v", err) |
| 1453 | } |
| 1454 | } |
| 1455 | |
| 1456 | func testCanaryKeys(t *testing.T) (ed25519.PrivateKey, ed25519.PublicKey) { |
| 1457 | t.Helper() |
| 1458 | seedB64, pubB64, err := release.GenerateKey() |
| 1459 | if err != nil { |
| 1460 | t.Fatalf("GenerateKey: %v", err) |
| 1461 | } |
| 1462 | priv, err := release.ParseSigningKey(seedB64, "test") |
| 1463 | if err != nil { |
| 1464 | t.Fatalf("ParseSigningKey: %v", err) |
| 1465 | } |
| 1466 | pub, err := release.ParsePublicKey(pubB64, "test") |
| 1467 | if err != nil { |
| 1468 | t.Fatalf("ParsePublicKey: %v", err) |
| 1469 | } |
| 1470 | return priv, pub |
| 1471 | } |
| 1472 | |
| 1473 | // No canary routes at all needs no store to answer "none" — a confident, |
| 1474 | // known 0, not the unknown/nil path a missing store would otherwise imply. |
| 1475 | func TestCountExpiredCanariesNoCanaryRoutesIsZero(t *testing.T) { |
| 1476 | routes := []registry.Route{ |
| 1477 | {Host: "a.example.com", Kind: "instance", App: "myapp", Channel: "stable"}, |
| 1478 | {Host: "b.example.com", Kind: "static"}, |
| 1479 | } |
| 1480 | got := countExpiredCanaries(context.Background(), routes, nil, nil, time.Now()) |
| 1481 | if got == nil || *got != 0 { |
| 1482 | t.Errorf("count = %v, want 0 (no canary routes, no store needed)", got) |
| 1483 | } |
| 1484 | } |
| 1485 | |
| 1486 | // A canary route exists but there is no deployment store to resolve its |
| 1487 | // pointer against — the "routes managed by hand" box shape adoptLoop's own |
| 1488 | // doc comment names. That must publish unknown, never a guessed 0. |
| 1489 | func TestCountExpiredCanariesNilStoreIsUnknown(t *testing.T) { |
| 1490 | routes := []registry.Route{{Host: "rc.example.com", Kind: "instance", App: "myapp", Channel: "canary/x"}} |
| 1491 | if got := countExpiredCanaries(context.Background(), routes, nil, nil, time.Now()); got != nil { |
| 1492 | t.Errorf("count = %v, want nil (no store to resolve the pointer against)", *got) |
| 1493 | } |
| 1494 | } |
| 1495 | |
| 1496 | // A canary route whose channel was never actually promoted (no pointer at |
| 1497 | // that key) is the same "can't answer" case as no store at all — release. |
| 1498 | // Channel returns os.ErrNotExist, and that must null the count too, not |
| 1499 | // silently skip the route and undercount. |
| 1500 | func TestCountExpiredCanariesUnresolvablePointerIsUnknown(t *testing.T) { |
| 1501 | s := release.NewDirStore(t.TempDir()) |
| 1502 | _, pub := testCanaryKeys(t) |
| 1503 | routes := []registry.Route{{Host: "rc.example.com", Kind: "instance", App: "myapp", Channel: "canary/x", Account: "bab"}} |
| 1504 | if got := countExpiredCanaries(context.Background(), routes, s, pub, time.Now()); got != nil { |
| 1505 | t.Errorf("count = %v, want nil (channel was never promoted)", *got) |
| 1506 | } |
| 1507 | } |
| 1508 | |
| 1509 | // A fresh canary (real Promote, real clock) counts as 0 — not expired, and |
| 1510 | // known. |
| 1511 | func TestCountExpiredCanariesFreshCanaryIsZero(t *testing.T) { |
| 1512 | s := release.NewDirStore(t.TempDir()) |
| 1513 | ctx := context.Background() |
| 1514 | priv, pub := testCanaryKeys(t) |
| 1515 | shipVersion(t, s, "bab", "myapp", "v1") |
| 1516 | if err := release.Promote(ctx, s, priv, "bab", "myapp", "canary/x", "v1"); err != nil { |
| 1517 | t.Fatalf("Promote: %v", err) |
| 1518 | } |
| 1519 | routes := []registry.Route{{Host: "rc.example.com", Kind: "instance", App: "myapp", Channel: "canary/x", Account: "bab"}} |
| 1520 | got := countExpiredCanaries(ctx, routes, s, pub, time.Now()) |
| 1521 | if got == nil || *got != 0 { |
| 1522 | t.Errorf("count = %v, want 0 (freshly promoted)", got) |
| 1523 | } |
| 1524 | } |
| 1525 | |
| 1526 | // The core case: a backdated (expired) canary pointer counts as 1; a second, |
| 1527 | // unexpired canary route on a different channel does not add to it. |
| 1528 | func TestCountExpiredCanariesCountsExpiredOnly(t *testing.T) { |
| 1529 | s := release.NewDirStore(t.TempDir()) |
| 1530 | ctx := context.Background() |
| 1531 | priv, pub := testCanaryKeys(t) |
| 1532 | writeBackdatedPointer(t, s, priv, "bab", "myapp", "canary/old", "v1", |
| 1533 | time.Now().Add(-(release.CanaryTTL + 24*time.Hour))) |
| 1534 | shipVersion(t, s, "bab", "myapp", "v2") |
| 1535 | if err := release.Promote(ctx, s, priv, "bab", "myapp", "canary/fresh", "v2"); err != nil { |
| 1536 | t.Fatalf("Promote: %v", err) |
| 1537 | } |
| 1538 | routes := []registry.Route{ |
| 1539 | {Host: "old.example.com", Kind: "instance", App: "myapp", Channel: "canary/old", Account: "bab"}, |
| 1540 | {Host: "fresh.example.com", Kind: "instance", App: "myapp", Channel: "canary/fresh", Account: "bab"}, |
| 1541 | } |
| 1542 | got := countExpiredCanaries(ctx, routes, s, pub, time.Now()) |
| 1543 | if got == nil || *got != 1 { |
| 1544 | t.Errorf("count = %v, want 1 (only the old channel's route is expired)", got) |
| 1545 | } |
| 1546 | } |
| 1547 | |
| 1548 | // A platform-minted alias is a second NAME for a route already counted, not |
| 1549 | // a second canary route — same convention CollectBox uses to avoid |
| 1550 | // double-counting aliases (host_test.go's own |
| 1551 | // TestCollectBoxSkipsPlatformMintedRoutes in internal/status covers that |
| 1552 | // half; this pins the same rule for the expired-canary count). |
| 1553 | func TestCountExpiredCanariesSkipsPlatformAliases(t *testing.T) { |
| 1554 | s := release.NewDirStore(t.TempDir()) |
| 1555 | priv, pub := testCanaryKeys(t) |
| 1556 | writeBackdatedPointer(t, s, priv, "bab", "myapp", "canary/old", "v1", |
| 1557 | time.Now().Add(-(release.CanaryTTL + 24*time.Hour))) |
| 1558 | routes := []registry.Route{ |
| 1559 | {Host: "alias.example.com", Kind: "instance", App: "myapp", Channel: "canary/old", Account: "bab", Platform: true}, |
| 1560 | } |
| 1561 | got := countExpiredCanaries(context.Background(), routes, s, pub, time.Now()) |
| 1562 | if got == nil || *got != 0 { |
| 1563 | t.Errorf("count = %v, want 0 — a platform alias must not be counted", got) |
| 1564 | } |
| 1565 | } |
| 1566 | |
| 1567 | // getCountingStore wraps a real release.Store and counts Get calls — used to |
| 1568 | // prove countExpiredCanaries' per-call cache actually dedupes store reads, |
| 1569 | // not just channel-pointer resolutions in memory. |
| 1570 | type getCountingStore struct { |
| 1571 | release.Store |
| 1572 | gets int |
| 1573 | } |
| 1574 | |
| 1575 | func (s *getCountingStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 1576 | s.gets++ |
| 1577 | return s.Store.Get(ctx, key) |
| 1578 | } |
| 1579 | |
| 1580 | // TestCountExpiredCanariesReadsEachChannelOnce pins the caching half of the |
| 1581 | // 2026-08-03 review: statusLoop calls countExpiredCanaries once per |
| 1582 | // 5-minute tick, and this function must resolve a given canary channel's |
| 1583 | // pointer AT MOST ONCE per call however many of this box's routes follow |
| 1584 | // it — N routes sharing one channel costs one store.Get, not N. |
| 1585 | func TestCountExpiredCanariesReadsEachChannelOnce(t *testing.T) { |
| 1586 | real := release.NewDirStore(t.TempDir()) |
| 1587 | priv, pub := testCanaryKeys(t) |
| 1588 | writeBackdatedPointer(t, real, priv, "bab", "myapp", "canary/old", "v1", |
| 1589 | time.Now().Add(-(release.CanaryTTL + 24*time.Hour))) |
| 1590 | counting := &getCountingStore{Store: real} |
| 1591 | |
| 1592 | routes := []registry.Route{ |
| 1593 | {Host: "a.example.com", Kind: "instance", App: "myapp", Channel: "canary/old", Account: "bab"}, |
| 1594 | {Host: "b.example.com", Kind: "instance", App: "myapp", Channel: "canary/old", Account: "bab"}, |
| 1595 | {Host: "c.example.com", Kind: "instance", App: "myapp", Channel: "canary/old", Account: "bab"}, |
| 1596 | } |
| 1597 | got := countExpiredCanaries(context.Background(), routes, counting, pub, time.Now()) |
| 1598 | if got == nil || *got != 3 { |
| 1599 | t.Fatalf("count = %v, want 3 (all three routes share the one expired channel)", got) |
| 1600 | } |
| 1601 | if counting.gets != 1 { |
| 1602 | t.Errorf("store.Get called %d times for 3 routes on ONE channel, want exactly 1 — the per-tick cache must dedupe by (account, app, channel)", counting.gets) |
| 1603 | } |
| 1604 | } |
| 1605 | |
| 1606 | // shipVersion ships a minimal one-artifact release so Promote has something |
| 1607 | // real to point at — release.Promote refuses to point a channel at a |
| 1608 | // version nothing shipped. |
| 1609 | func shipVersion(t *testing.T, s release.Store, acct, app, version string) { |
| 1610 | t.Helper() |
| 1611 | m := release.Manifest{ |
| 1612 | Schema: 1, App: app, Version: version, Kind: "binary", |
| 1613 | Artifacts: map[string]release.Artifact{"linux-amd64": {File: app}}, |
| 1614 | } |
| 1615 | if err := release.Ship(context.Background(), s, acct, m, map[string][]byte{"linux-amd64": []byte("bin")}); err != nil { |
| 1616 | t.Fatalf("Ship %s/%s: %v", app, version, err) |
| 1617 | } |
| 1618 | } |
| 1619 | |
| 1620 | // Both restore drills must write their restored copies to DISK, never to the |
| 1621 | // PrivateTmp tmpfs. On tmpfs the restored database is charged to the unit's |
| 1622 | // own cgroup as memory, so the drill's ceiling has to exceed the fleet's |
| 1623 | // largest tenant — and the drill starts failing on exactly the databases it |
| 1624 | // exists to vouch for. Measured on the flagship 2026-08-15: a 412M tenant |
| 1625 | // produced a 485.6M peak against MemoryMax=512M. |
| 1626 | // |
| 1627 | // This is a one-line property that would regress silently (the drill still |
| 1628 | // passes, right up until a tenant grows), so it is pinned here. |
| 1629 | // The tenant template must deny IMDS at the unit level (issue #351): the |
| 1630 | // nftables skuid gate is operator territory and a box can simply not have |
| 1631 | // one (tito didn't), at which point any app process could read the box |
| 1632 | // instance-role credentials off 169.254.169.254. Apps get per-app scoped |
| 1633 | // IAM keys via env; the one legitimate IMDS consumer under this template |
| 1634 | // is a host_role tenant, whose drop-in re-opens it with IPAddressAllow |
| 1635 | // (allow beats deny in systemd's BPF firewall) — see caps.RenderService. |
| 1636 | // One-line property, silent regression, so it is pinned here. |
| 1637 | func TestAppTemplateDeniesIMDS(t *testing.T) { |
| 1638 | body, err := unitsFS.ReadFile("units/carlos-app@.service") |
| 1639 | if err != nil { |
| 1640 | t.Fatalf("carlos-app@.service: %v", err) |
| 1641 | } |
| 1642 | if !strings.Contains(string(body), "IPAddressDeny=169.254.169.254/32 fd00:ec2::254/128") { |
| 1643 | t.Errorf("carlos-app@.service does not deny IMDS — app code on an ungated box can take the instance role's credentials (issue #351)") |
| 1644 | } |
| 1645 | } |
| 1646 | |
| 1647 | func TestVerifyDrillsRestoreToDiskNotTmpfs(t *testing.T) { |
| 1648 | for _, unit := range []string{"carlos-restore-verify.service", "carlos-hibernate-verify.service"} { |
| 1649 | body, err := unitsFS.ReadFile("units/" + unit) |
| 1650 | if err != nil { |
| 1651 | t.Fatalf("%s: %v", unit, err) |
| 1652 | } |
| 1653 | s := string(body) |
| 1654 | if !strings.Contains(s, "CacheDirectory=carlos-verify") { |
| 1655 | t.Errorf("%s has no CacheDirectory — nothing creates the on-disk scratch dir", unit) |
| 1656 | } |
| 1657 | if !strings.Contains(s, "Environment=TMPDIR=%C/carlos-verify") { |
| 1658 | t.Errorf("%s does not point TMPDIR at the on-disk dir; restores land on the tmpfs and count as memory", unit) |
| 1659 | } |
| 1660 | if !strings.Contains(s, "MemoryMax=") { |
| 1661 | t.Errorf("%s has no MemoryMax — a runaway drill should hit its own ceiling, not the box's", unit) |
| 1662 | } |
| 1663 | } |
| 1664 | } |
| 1665 | |
| 1666 | // The journal cap ships with every bootstrap (discussion #111): uncapped, |
| 1667 | // journald's default is 10% of the filesystem, and 1.2 GB of the flagship's |
| 1668 | // 20 GB root was journal by the time it filled. |
| 1669 | func TestBootstrapWritesJournaldCap(t *testing.T) { |
| 1670 | root := t.TempDir() |
| 1671 | if err := bootstrap(root, nil); err != nil { |
| 1672 | t.Fatalf("bootstrap: %v", err) |
| 1673 | } |
| 1674 | path := filepath.Join(root, "etc", "systemd", "journald.conf.d", journaldConfName) |
| 1675 | b, err := os.ReadFile(path) |
| 1676 | if err != nil { |
| 1677 | t.Fatalf("journald drop-in not written: %v", err) |
| 1678 | } |
| 1679 | got := string(b) |
| 1680 | // The section header is not optional: journald ignores a bare assignment. |
| 1681 | for _, want := range []string{"[Journal]", "SystemMaxUse=256M", "RuntimeMaxUse="} { |
| 1682 | if !strings.Contains(got, want) { |
| 1683 | t.Errorf("drop-in missing %q:\n%s", want, got) |
| 1684 | } |
| 1685 | } |
| 1686 | // The name has to match the one placed by hand on the flagship during the |
| 1687 | // 2026-09-06 relief, or the two files sit side by side and name order |
| 1688 | // decides which cap wins. |
| 1689 | if journaldConfName != "50-cap.conf" { |
| 1690 | t.Errorf("journaldConfName = %q; it must supersede the hand-placed 50-cap.conf", journaldConfName) |
| 1691 | } |
| 1692 | // A staged root is the delivery path to boxes that predate the cap: the |
| 1693 | // self-update apply copies this file out of one by name (installUnits). |
| 1694 | if err := bootstrap(root, nil); err != nil { |
| 1695 | t.Fatalf("re-run: %v", err) |
| 1696 | } |
| 1697 | if _, err := os.Stat(path); err != nil { |
| 1698 | t.Errorf("drop-in gone after a re-run: %v", err) |
| 1699 | } |
| 1700 | } |
| 1701 | |
| 1702 | // TestTmpfilesCarriesRelayStateDir is discussion #118 as a test. The relay's |
| 1703 | // unit declares ReadWritePaths=/data/relay, and systemd refuses to start a |
| 1704 | // unit whose ReadWritePaths names a directory that is not there — "Failed to |
| 1705 | // set up mount namespacing: /data/relay: No such file or directory", which is |
| 1706 | // what the Ireland box answered on 2026-09-07. |
| 1707 | // |
| 1708 | // bootstrap's own dirs list creates it, and that is not enough: the dirs list |
| 1709 | // runs on a FRESH box or against a staging root, never against the live root |
| 1710 | // of a self-update apply. What an apply re-runs is systemd-tmpfiles. So the |
| 1711 | // directory has to be in tmpfilesConf, and this test is what keeps a future |
| 1712 | // unit's writable state dir from being added to one list and not the other. |
| 1713 | func TestTmpfilesCarriesRelayStateDir(t *testing.T) { |
| 1714 | b, err := unitsFS.ReadFile("units/" + relayUnitName) |
| 1715 | if err != nil { |
| 1716 | t.Fatal(err) |
| 1717 | } |
| 1718 | var rw string |
| 1719 | for _, line := range strings.Split(string(b), "\n") { |
| 1720 | if v, ok := strings.CutPrefix(line, "ReadWritePaths="); ok { |
| 1721 | rw = v |
| 1722 | break |
| 1723 | } |
| 1724 | } |
| 1725 | if rw == "" { |
| 1726 | t.Fatal("carlos-relay.service declares no ReadWritePaths") |
| 1727 | } |
| 1728 | for _, p := range strings.Fields(rw) { |
| 1729 | if !strings.HasPrefix(p, "/data/") { |
| 1730 | continue // /run/carlos is tmpfs and already has its own line |
| 1731 | } |
| 1732 | want := "d " + p + " " |
| 1733 | if !strings.Contains(tmpfilesConf, want) { |
| 1734 | t.Errorf("tmpfilesConf has no %q line, so an apply onto a box that predates the unit leaves it unable to start:\n%s", want, tmpfilesConf) |
| 1735 | } |
| 1736 | } |
| 1737 | // The mode must match bootstrap's dirs list, or a re-run of one flips |
| 1738 | // what the other just set (the /data/instances lesson, one line up). |
| 1739 | // Read the mode bootstrap actually uses rather than restating it: a |
| 1740 | // hardcoded "0750" here passes happily while the two lists disagree. |
| 1741 | root := t.TempDir() |
| 1742 | if err := bootstrapWith(root, nil, false); err != nil { |
| 1743 | t.Fatal(err) |
| 1744 | } |
| 1745 | fi, err := os.Stat(filepath.Join(root, "data", "relay")) |
| 1746 | if err != nil { |
| 1747 | t.Fatalf("bootstrap did not create /data/relay: %v", err) |
| 1748 | } |
| 1749 | want := fmt.Sprintf("d /data/relay %04o ", fi.Mode().Perm()) |
| 1750 | if !strings.Contains(tmpfilesConf, want) { |
| 1751 | t.Errorf("tmpfilesConf must carry %q to match the mode bootstrap's dirs list creates:\n%s", want, tmpfilesConf) |
| 1752 | } |
| 1753 | } |
| 1754 | |
| 1755 | // TestPlanCarriesArmedList pins the other half of discussion #118: the plan a |
| 1756 | // build prints says which units to START, not just which to enable, so an |
| 1757 | // applier running an OLDER binary arms what the new build wants rather than |
| 1758 | // what its own systemctlPlan happens to know about. |
| 1759 | func TestPlanCarriesArmedList(t *testing.T) { |
| 1760 | plan := currentBootstrapPlan() |
| 1761 | if len(plan.Armed) == 0 { |
| 1762 | t.Fatal("plan carries no armed list") |
| 1763 | } |
| 1764 | if !slices.Contains(plan.Armed, relayUnitName) { |
| 1765 | t.Errorf("armed list omits the relay: %v", plan.Armed) |
| 1766 | } |
| 1767 | if slices.Contains(plan.Armed, "carlos-edge.service") { |
| 1768 | t.Errorf("edge must stay start-by-hand: %v", plan.Armed) |
| 1769 | } |
| 1770 | for _, u := range plan.Armed { |
| 1771 | if !slices.Contains(plan.Enabled, u) { |
| 1772 | t.Errorf("armed unit %s is not in the enabled list", u) |
| 1773 | } |
| 1774 | } |
| 1775 | // The armed list and the local rule must agree for THIS build; they |
| 1776 | // diverge only across builds, which is the point of carrying it. |
| 1777 | if got, want := plan.Armed, armedUnits(enabledUnits); !slices.Equal(got, want) { |
| 1778 | t.Errorf("plan.Armed = %v, systemctlPlan would start %v", got, want) |
| 1779 | } |
| 1780 | } |
| 1781 | |
| 1782 | // floorGateCall is how both system units ask whether carlos.prev may run |
| 1783 | // against an armed floor: carlos.prev's own floor gate (root's trusted |
| 1784 | // path, never a PATH lookup), with both files named in the unit. |
| 1785 | const floorGateCall = "/opt/carlos/carlos.prev system floor-allows-prev /data/.moves/floor.min-release /opt/carlos/carlos.prev.allowed" |
| 1786 | |
| 1787 | // seedsFloorGateCall is the same gate asked about the seeds floor's two |
| 1788 | // files (seeded-instances spec §3.3): the units' second clause. |
| 1789 | const seedsFloorGateCall = "/opt/carlos/carlos.prev system floor-allows-prev /data/.moves/seeds-floor.min-release /opt/carlos/carlos.prev.seeds-allowed" |
| 1790 | |
| 1791 | // floorGateHelperEnv makes this test binary act as the floor gate for the |
| 1792 | // unit stubs in TestSystemUnitsSelectPrevByTheFloor. |
| 1793 | const floorGateHelperEnv = "CARLOS_TEST_FLOOR_GATE_HELPER" |
| 1794 | |
| 1795 | // TestFloorGateHelperProcess is not a test: run with floorGateHelperEnv |
| 1796 | // set, it is `carlos system floor-allows-prev` with the test's own uid as |
| 1797 | // the expected owner of both files (a test cannot make root's or the |
| 1798 | // service account's files). |
| 1799 | func TestFloorGateHelperProcess(t *testing.T) { |
| 1800 | if os.Getenv(floorGateHelperEnv) != "1" { |
| 1801 | return |
| 1802 | } |
| 1803 | args := os.Args |
| 1804 | for i, a := range args { |
| 1805 | if a == "--" { |
| 1806 | args = args[i+1:] |
| 1807 | break |
| 1808 | } |
| 1809 | } |
| 1810 | uid := []uint32{uint32(os.Getuid())} |
| 1811 | os.Exit(floorGateMain(args, uid, uid, os.Stderr)) |
| 1812 | } |
| 1813 | |
| 1814 | // TestSystemUnitsSelectPrevByTheFloor runs each system unit's ExecStart |
| 1815 | // shell (moves spec §11, round-6 finding 4) against a fake /opt/carlos and |
| 1816 | // /data, with stub binaries that record which one ran. Without a floor file |
| 1817 | // the selection is main's exactly; with one, carlos.prev runs only when its |
| 1818 | // verdict names the floor on disk, byte for byte, as carlos.prev's floor |
| 1819 | // gate reads them (the stubs hand the gate verb to the real Go code). |
| 1820 | func TestSystemUnitsSelectPrevByTheFloor(t *testing.T) { |
| 1821 | t.Parallel() |
| 1822 | if runtime.GOOS != "linux" { |
| 1823 | t.Skip("the floor gate refuses off Linux") |
| 1824 | } |
| 1825 | self, err := os.Executable() |
| 1826 | if err != nil { |
| 1827 | t.Fatal(err) |
| 1828 | } |
| 1829 | for _, unit := range []string{"carlos-system-recover.service", "carlos-system-update.service"} { |
| 1830 | t.Run(unit, func(t *testing.T) { |
| 1831 | b, err := unitsFS.ReadFile("units/" + unit) |
| 1832 | if err != nil { |
| 1833 | t.Fatal(err) |
| 1834 | } |
| 1835 | var line string |
| 1836 | for _, l := range strings.Split(string(b), "\n") { |
| 1837 | if strings.HasPrefix(l, "ExecStart=") { |
| 1838 | line = l |
| 1839 | } |
| 1840 | } |
| 1841 | // systemd expands %-specifiers and $VARIABLES in ExecStart before |
| 1842 | // the shell ever sees the line (systemd.service(5)). |
| 1843 | if strings.ContainsAny(line, "%$") { |
| 1844 | t.Fatalf("ExecStart carries a %% or a $, which systemd expands: %s", line) |
| 1845 | } |
| 1846 | for _, path := range []string{"/data/" + moves.FloorMinReleaseFile, "/opt/carlos/" + moves.PrevAllowedFile} { |
| 1847 | if !strings.Contains(line, path) { |
| 1848 | t.Fatalf("ExecStart does not read %s: %s", path, line) |
| 1849 | } |
| 1850 | } |
| 1851 | body, ok := strings.CutPrefix(line, "ExecStart=/bin/sh -c '") |
| 1852 | body, ok2 := strings.CutSuffix(body, "'") |
| 1853 | if !ok || !ok2 || strings.Contains(body, "'") { |
| 1854 | t.Fatalf("ExecStart is not one quoted sh -c body: %s", line) |
| 1855 | } |
| 1856 | |
| 1857 | for _, tc := range []struct { |
| 1858 | name string |
| 1859 | journal bool |
| 1860 | floor, verdict string // "" means no file |
| 1861 | floorKind string // "" a regular file; or symlink, dangling, directory, fifo |
| 1862 | noPrev bool |
| 1863 | want string |
| 1864 | gate string // which binary ran the floor gate, "" for none |
| 1865 | prevNoVerb bool // carlos.prev predates the gate and rejects its verb |
| 1866 | }{ |
| 1867 | {"no journal", false, "", "", "", false, "carlos", "", false}, |
| 1868 | {"no floor", true, "", "", "", false, "carlos.prev", "", false}, |
| 1869 | {"no floor, no prev", true, "", "", "", true, "carlos", "", false}, |
| 1870 | {"floor and a verdict naming it", true, "v0.30.0\n", "v0.30.0\n", "", false, "carlos.prev", "carlos.prev", false}, |
| 1871 | {"a verdict for an older floor", true, "v0.30.0\n", "v0.29.0\n", "", false, "carlos", "carlos.prev", false}, |
| 1872 | {"floor and no verdict", true, "v0.30.0\n", "", "", false, "carlos", "carlos.prev", false}, |
| 1873 | {"no journal, armed and allowed", false, "v0.30.0\n", "v0.30.0\n", "", false, "carlos", "", false}, |
| 1874 | {"armed, no prev", true, "v0.30.0\n", "v0.30.0\n", "", true, "carlos", "", false}, |
| 1875 | // The agent owns /data: anything but a regular file at the |
| 1876 | // floor path is refused before the gate runs (#267), and the |
| 1877 | // gate refuses one swapped in after that test (it never |
| 1878 | // follows a symlink or opens a FIFO for a blocking read). |
| 1879 | // [ -s ] is false for a dangling symlink and a FIFO, so |
| 1880 | // without the shell's test both would read as "no floor". |
| 1881 | {"symlink floor with matching verdict", true, "v0.30.0\n", "v0.30.0\n", "symlink", false, "carlos", "", false}, |
| 1882 | {"dangling symlink floor", true, "", "", "dangling", false, "carlos", "", false}, |
| 1883 | {"directory floor", true, "", "", "directory", false, "carlos", "", false}, |
| 1884 | {"fifo floor", true, "", "v0.30.0\n", "fifo", false, "carlos", "", false}, |
| 1885 | // A carlos.prev older than the gate answers its verb with |
| 1886 | // an unknown-verb exit: a mismatch, so carlos runs. |
| 1887 | {"a carlos.prev without the gate verb", true, "v0.30.0\n", "v0.30.0\n", "", false, "carlos", "carlos.prev", true}, |
| 1888 | } { |
| 1889 | t.Run(tc.name, func(t *testing.T) { |
| 1890 | root := t.TempDir() |
| 1891 | opt, data := filepath.Join(root, "opt"), filepath.Join(root, "data") |
| 1892 | ran, gate := filepath.Join(root, "ran"), filepath.Join(root, "gate") |
| 1893 | os.MkdirAll(filepath.Join(opt, "system", "router"), 0o755) |
| 1894 | os.MkdirAll(filepath.Join(data, ".moves"), 0o755) |
| 1895 | // Each stub answers the gate verb by handing it to this |
| 1896 | // test binary (TestFloorGateHelperProcess), noting which |
| 1897 | // stub was asked; anything else records the stub as run. |
| 1898 | stub := func(name string) { |
| 1899 | answer := " shift 2\n" + |
| 1900 | " exec " + self + " -test.run='^TestFloorGateHelperProcess$' -- \"$@\"\n" |
| 1901 | if name == "carlos.prev" && tc.prevNoVerb { |
| 1902 | answer = " echo 'system: unknown verb \"floor-allows-prev\"' >&2\n exit 2\n" |
| 1903 | } |
| 1904 | sh := "#!/bin/sh\n" + |
| 1905 | "if [ \"$1\" = system ] && [ \"$2\" = floor-allows-prev ]; then\n" + |
| 1906 | " echo " + name + " >> " + gate + "\n" + |
| 1907 | answer + |
| 1908 | "fi\n" + |
| 1909 | "echo " + name + " > " + ran + "\n" |
| 1910 | os.WriteFile(filepath.Join(opt, name), []byte(sh), 0o755) |
| 1911 | } |
| 1912 | stub("carlos") |
| 1913 | if !tc.noPrev { |
| 1914 | stub("carlos.prev") |
| 1915 | } |
| 1916 | if tc.journal { |
| 1917 | os.WriteFile(filepath.Join(opt, "system", "router", "journal.json"), []byte("{}"), 0o600) |
| 1918 | } |
| 1919 | floorPath := filepath.Join(data, moves.FloorMinReleaseFile) |
| 1920 | switch tc.floorKind { |
| 1921 | case "symlink", "dangling": |
| 1922 | target := filepath.Join(root, "floor-target") |
| 1923 | if tc.floorKind == "symlink" { |
| 1924 | if err := os.WriteFile(target, []byte(tc.floor), 0o644); err != nil { |
| 1925 | t.Fatal(err) |
| 1926 | } |
| 1927 | } |
| 1928 | if err := os.Symlink(target, floorPath); err != nil { |
| 1929 | t.Fatal(err) |
| 1930 | } |
| 1931 | case "directory": |
| 1932 | if err := os.Mkdir(floorPath, 0o755); err != nil { |
| 1933 | t.Fatal(err) |
| 1934 | } |
| 1935 | case "fifo": |
| 1936 | if out, err := exec.Command("mkfifo", floorPath).CombinedOutput(); err != nil { |
| 1937 | t.Fatalf("mkfifo: %v: %s", err, out) |
| 1938 | } |
| 1939 | default: |
| 1940 | if tc.floor != "" { |
| 1941 | if err := os.WriteFile(floorPath, []byte(tc.floor), 0o644); err != nil { |
| 1942 | t.Fatal(err) |
| 1943 | } |
| 1944 | } |
| 1945 | } |
| 1946 | if tc.verdict != "" { |
| 1947 | os.WriteFile(filepath.Join(opt, moves.PrevAllowedFile), []byte(tc.verdict), 0o644) |
| 1948 | } |
| 1949 | script := strings.ReplaceAll(body, "/data/", data+"/") |
| 1950 | script = strings.ReplaceAll(script, "/opt/carlos/", opt+"/") |
| 1951 | cmd := exec.Command("/bin/sh", "-c", script) |
| 1952 | cmd.Env = append(os.Environ(), floorGateHelperEnv+"=1") |
| 1953 | if out, err := cmd.CombinedOutput(); err != nil { |
| 1954 | t.Fatalf("%v: %s", err, out) |
| 1955 | } |
| 1956 | got, _ := os.ReadFile(ran) |
| 1957 | if strings.TrimSpace(string(got)) != tc.want { |
| 1958 | t.Fatalf("ran %q, want %s", strings.TrimSpace(string(got)), tc.want) |
| 1959 | } |
| 1960 | asked, _ := os.ReadFile(gate) |
| 1961 | if strings.TrimSpace(string(asked)) != tc.gate { |
| 1962 | t.Fatalf("the floor gate ran from %q, want %q", strings.TrimSpace(string(asked)), tc.gate) |
| 1963 | } |
| 1964 | }) |
| 1965 | } |
| 1966 | }) |
| 1967 | } |
| 1968 | } |
| 1969 | |
amadan