| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "crypto/sha256" |
| 5 | "encoding/hex" |
| 6 | "fmt" |
| 7 | "net" |
| 8 | "net/http" |
| 9 | "net/http/httptest" |
| 10 | "os" |
| 11 | "path/filepath" |
| 12 | "strings" |
| 13 | "sync" |
| 14 | "testing" |
| 15 | "time" |
| 16 | ) |
| 17 | |
| 18 | // selfUpdateEnv points every seam at temp doubles: a fake control binary on |
| 19 | // the same temp "filesystem" the download lands on, a fresh bake state, a |
| 20 | // recording systemctl, and the health probe aimed at a temp socket path (which |
| 21 | // a test binds via fakeControlSocket, or leaves unbound to model a refused |
| 22 | // dial). deployHealthTimeout is shrunk so an unhealthy path fails in ~a beat. |
| 23 | // By default isActive reports controld UP (so the live /healthz probe is the |
| 24 | // authoritative gate, the pre-F2 behaviour); tests exercising the |
| 25 | // inactive-controld fallback override isActive + controlBinaryReports. |
| 26 | func selfUpdateEnv(t *testing.T, newBinary []byte) (self string, entry releaseEntry, calls *[]string) { |
| 27 | t.Helper() |
| 28 | oldOpt, oldSelf, oldSysctl, oldChannel := optDir, selfPath, systemctl, os.Getenv("LCHAT_CONTROL_CHANNEL") |
| 29 | oldBuild, oldSock, oldTimeout := buildVersion, controlSocketPath, deployHealthTimeout |
| 30 | oldActive, oldBinReports := isActive, controlBinaryReports |
| 31 | buildVersion = "testold1" // the dev-build guard must not fire in these tests |
| 32 | t.Cleanup(func() { |
| 33 | optDir, selfPath, systemctl, buildVersion = oldOpt, oldSelf, oldSysctl, oldBuild |
| 34 | controlSocketPath, deployHealthTimeout = oldSock, oldTimeout |
| 35 | isActive, controlBinaryReports = oldActive, oldBinReports |
| 36 | os.Setenv("LCHAT_CONTROL_CHANNEL", oldChannel) |
| 37 | }) |
| 38 | optDir = t.TempDir() |
| 39 | self = filepath.Join(optDir, "lchat") |
| 40 | if err := os.WriteFile(self, []byte("OLD-CONTROL-BINARY"), 0o755); err != nil { |
| 41 | t.Fatal(err) |
| 42 | } |
| 43 | selfPath = func() (string, error) { return self, nil } |
| 44 | var c []string |
| 45 | calls = &c |
| 46 | systemctl = func(args ...string) error { |
| 47 | c = append(c, strings.Join(args, " ")) |
| 48 | return nil |
| 49 | } |
| 50 | os.Setenv("LCHAT_CONTROL_CHANNEL", "stable") |
| 51 | |
| 52 | // Default seams: controld is up (the /healthz probe is the gate), and the |
| 53 | // `version` integrity check is a poisoned stub — a test that expects the |
| 54 | // inactive-controld fallback must stub both, or this loud error trips. |
| 55 | isActive = func(string) bool { return true } |
| 56 | controlBinaryReports = func(string, string) error { |
| 57 | return fmt.Errorf("controlBinaryReports not stubbed for this test") |
| 58 | } |
| 59 | |
| 60 | // The health probe dials a unix socket, whose sun_path is short-limited — a |
| 61 | // dedicated short temp dir, not optDir (which can be a long macOS path). |
| 62 | sockDir, err := os.MkdirTemp("", "lch") |
| 63 | if err != nil { |
| 64 | t.Fatal(err) |
| 65 | } |
| 66 | t.Cleanup(func() { os.RemoveAll(sockDir) }) |
| 67 | controlSocketPath = filepath.Join(sockDir, "c.sock") |
| 68 | deployHealthTimeout = 400 * time.Millisecond |
| 69 | |
| 70 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 71 | w.Write(newBinary) |
| 72 | })) |
| 73 | t.Cleanup(srv.Close) |
| 74 | sum := sha256.Sum256(newBinary) |
| 75 | entry = releaseEntry{Version: "abc1234", URL: srv.URL + "/lchat", SHA256: hex.EncodeToString(sum[:]), Size: int64(len(newBinary)), |
| 76 | ARM64URL: srv.URL + "/lchat-arm64", ARM64SHA256: hex.EncodeToString(sum[:]), ARM64Size: int64(len(newBinary))} |
| 77 | return self, entry, calls |
| 78 | } |
| 79 | |
| 80 | // fakeControlSocket binds a unix socket at controlSocketPath that answers |
| 81 | // GET /healthz with a build — the same shape controld's real /healthz serves — |
| 82 | // so waitControlHealthy sees a healthy new build or a stale old one. It returns |
| 83 | // a setter so a test can flip the reported build mid-run (e.g. old→new after a |
| 84 | // channel moves). A test that wants a REFUSED dial (a crash-looping binary) |
| 85 | // simply doesn't call this, leaving the path unbound. |
| 86 | func fakeControlSocket(t *testing.T, build string) (set func(string)) { |
| 87 | t.Helper() |
| 88 | var mu sync.Mutex |
| 89 | cur := build |
| 90 | set = func(b string) { mu.Lock(); cur = b; mu.Unlock() } |
| 91 | mux := http.NewServeMux() |
| 92 | mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { |
| 93 | mu.Lock() |
| 94 | b := cur |
| 95 | mu.Unlock() |
| 96 | writeJSON(w, http.StatusOK, map[string]any{"build": b}) |
| 97 | }) |
| 98 | ln, err := net.Listen("unix", controlSocketPath) |
| 99 | if err != nil { |
| 100 | t.Fatal(err) |
| 101 | } |
| 102 | srv := &http.Server{Handler: mux} |
| 103 | go srv.Serve(ln) |
| 104 | t.Cleanup(func() { srv.Close() }) |
| 105 | return set |
| 106 | } |
| 107 | |
| 108 | // The swap end to end: verified download, staged into the version store, atomic |
| 109 | // install over the running path, a .prev rollback beside it, and the backend |
| 110 | // services bounced — router deliberately untouched. The control socket reports |
| 111 | // the new build, so the health probe passes and the swap commits. |
| 112 | func TestControlSelfUpdateSwaps(t *testing.T) { |
| 113 | newBin := []byte("NEW-CONTROL-BINARY") |
| 114 | self, entry, calls := selfUpdateEnv(t, newBin) |
| 115 | fakeControlSocket(t, entry.Version) // healthy on the new build |
| 116 | m := &releaseManifest{Schema: 1, GeneratedAt: time.Now(), Channels: map[string]releaseEntry{"stable": entry}} |
| 117 | |
| 118 | // ignoreBake=true is the deliberate override path; the bake-hold path is |
| 119 | // pinned separately below. |
| 120 | if err := controlSelfUpdate(m, loadFirstSeen(), nil, false, true); err != nil { |
| 121 | t.Fatalf("healthy swap returned error: %v", err) |
| 122 | } |
| 123 | |
| 124 | got, err := os.ReadFile(self) |
| 125 | if err != nil || string(got) != string(newBin) { |
| 126 | t.Fatalf("control binary not swapped: %q %v", got, err) |
| 127 | } |
| 128 | // Staged into the store too, so gcVersions protects it and rollback can reach it. |
| 129 | if staged, err := os.ReadFile(versionBinPath(entry.Version)); err != nil || string(staged) != string(newBin) { |
| 130 | t.Fatalf("new build not staged into the version store: %q %v", staged, err) |
| 131 | } |
| 132 | prev, err := os.ReadFile(self + ".prev-" + buildVersion) |
| 133 | if err != nil || string(prev) != "OLD-CONTROL-BINARY" { |
| 134 | t.Fatalf("rollback copy missing/wrong: %q %v", prev, err) |
| 135 | } |
| 136 | joined := strings.Join(*calls, ";") |
| 137 | if !strings.Contains(joined, "try-restart lchat-control.service") || !strings.Contains(joined, "try-restart lchat-cloud.service") { |
| 138 | t.Fatalf("backend services not bounced: %v", *calls) |
| 139 | } |
| 140 | if strings.Contains(joined, "lchat-router") { |
| 141 | t.Fatalf("the router must never restart on the timer's schedule: %v", *calls) |
| 142 | } |
| 143 | } |
| 144 | |
| 145 | // The happy path in isolation: a healthy new build commits, does NOT roll back, |
| 146 | // bounces the backends exactly once (two calls, not four), and returns nil. |
| 147 | func TestControlSelfUpdateHealthyCommits(t *testing.T) { |
| 148 | newBin := []byte("NEW-CONTROL-BINARY") |
| 149 | self, entry, calls := selfUpdateEnv(t, newBin) |
| 150 | fakeControlSocket(t, entry.Version) |
| 151 | m := &releaseManifest{Schema: 1, GeneratedAt: time.Now(), Channels: map[string]releaseEntry{"stable": entry}} |
| 152 | |
| 153 | if err := controlSelfUpdate(m, loadFirstSeen(), nil, false, true); err != nil { |
| 154 | t.Fatalf("healthy commit returned error: %v", err) |
| 155 | } |
| 156 | if got, _ := os.ReadFile(self); string(got) != string(newBin) { |
| 157 | t.Fatalf("healthy build should stay swapped in: %q", got) |
| 158 | } |
| 159 | // One restart of each backend, never a second (rollback) round. Counted |
| 160 | // against controlBinaryUnits rather than a literal, so adding a unit to the |
| 161 | // update path doesn't read as a spurious extra restart here. |
| 162 | if n, want := strings.Count(strings.Join(*calls, ";"), "try-restart"), len(controlBinaryUnits); n != want { |
| 163 | t.Fatalf("expected exactly %d restarts (no rollback), got %d: %v", want, n, *calls) |
| 164 | } |
| 165 | } |
| 166 | |
| 167 | // The whole point: a swapped-in build that never reports its version is rolled |
| 168 | // back. The control socket reports the OLD build (a restart that didn't take), |
| 169 | // so the new-build probe times out; the binary on disk is restored to .prev's |
| 170 | // content, the backends are restarted a SECOND time, and the function errors. |
| 171 | func TestControlSelfUpdateRollsBackOnUnhealthy(t *testing.T) { |
| 172 | newBin := []byte("NEW-CONTROL-BINARY") |
| 173 | self, entry, calls := selfUpdateEnv(t, newBin) |
| 174 | fakeControlSocket(t, buildVersion) // stale: reports the OLD build after the swap |
| 175 | m := &releaseManifest{Schema: 1, GeneratedAt: time.Now(), Channels: map[string]releaseEntry{"stable": entry}} |
| 176 | |
| 177 | err := controlSelfUpdate(m, loadFirstSeen(), nil, false, true) |
| 178 | if err == nil { |
| 179 | t.Fatal("an unhealthy swap must return an error") |
| 180 | } |
| 181 | if got, _ := os.ReadFile(self); string(got) != "OLD-CONTROL-BINARY" { |
| 182 | t.Fatalf("binary not restored to the last-known-good: %q", got) |
| 183 | } |
| 184 | joined := strings.Join(*calls, ";") |
| 185 | if n := strings.Count(joined, "try-restart lchat-control.service"); n != 2 { |
| 186 | t.Fatalf("expected control restarted twice (swap + rollback), got %d: %v", n, *calls) |
| 187 | } |
| 188 | if strings.Contains(joined, "lchat-router") { |
| 189 | t.Fatalf("the router must never restart, even on rollback: %v", *calls) |
| 190 | } |
| 191 | } |
| 192 | |
| 193 | // The genuine brick: the new build is unhealthy AND the rolled-back old build is |
| 194 | // unhealthy too (nothing ever answers /healthz). The swap is still physically |
| 195 | // reverted on disk, but a distinct, screaming error is surfaced — and it does |
| 196 | // not loop. |
| 197 | func TestControlSelfUpdateRollbackAlsoFailsScreams(t *testing.T) { |
| 198 | newBin := []byte("NEW-CONTROL-BINARY") |
| 199 | self, entry, _ := selfUpdateEnv(t, newBin) |
| 200 | // No fakeControlSocket at all → every /healthz dial is refused → both the |
| 201 | // post-swap and the post-rollback probes time out. |
| 202 | m := &releaseManifest{Schema: 1, GeneratedAt: time.Now(), Channels: map[string]releaseEntry{"stable": entry}} |
| 203 | |
| 204 | err := controlSelfUpdate(m, loadFirstSeen(), nil, false, true) |
| 205 | if err == nil || !strings.Contains(err.Error(), "CONTROL PLANE BRICKED") { |
| 206 | t.Fatalf("expected a distinct BRICKED error, got: %v", err) |
| 207 | } |
| 208 | // The rollback rename is a pure file op, so it still restores the good binary |
| 209 | // even when the box won't come back up on it. |
| 210 | if got, _ := os.ReadFile(self); string(got) != "OLD-CONTROL-BINARY" { |
| 211 | t.Fatalf("rollback rename should still restore the old binary: %q", got) |
| 212 | } |
| 213 | } |
| 214 | |
| 215 | // F2: when lchat-control isn't running, a failed /healthz probe is NOT |
| 216 | // authoritative — try-restart was a no-op and nothing could answer. A |
| 217 | // genuinely good binary (it execs and self-reports the target build via the |
| 218 | // `version` integrity check) must COMMIT, not be falsely rolled back, and the |
| 219 | // BRICKED path must not be taken. No socket is bound (the probe refuses). |
| 220 | func TestControlSelfUpdateInactiveControldCommits(t *testing.T) { |
| 221 | newBin := []byte("NEW-CONTROL-BINARY") |
| 222 | self, entry, calls := selfUpdateEnv(t, newBin) |
| 223 | isActive = func(string) bool { return false } // controld not running |
| 224 | controlBinaryReports = func(_, want string) error { |
| 225 | if want == entry.Version { |
| 226 | return nil // the new binary loads + self-reports the target build |
| 227 | } |
| 228 | return fmt.Errorf("unexpected integrity target %q", want) |
| 229 | } |
| 230 | m := &releaseManifest{Schema: 1, GeneratedAt: time.Now(), Channels: map[string]releaseEntry{"stable": entry}} |
| 231 | |
| 232 | if err := controlSelfUpdate(m, loadFirstSeen(), nil, false, true); err != nil { |
| 233 | t.Fatalf("inactive controld + good binary must commit, got error: %v", err) |
| 234 | } |
| 235 | if got, _ := os.ReadFile(self); string(got) != string(newBin) { |
| 236 | t.Fatalf("good binary must stay committed, not rolled back: %q", got) |
| 237 | } |
| 238 | joined := strings.Join(*calls, ";") |
| 239 | if n, want := strings.Count(joined, "try-restart"), len(controlBinaryUnits); n != want { |
| 240 | t.Fatalf("expected exactly %d restarts (commit, no rollback), got %d: %v", want, n, *calls) |
| 241 | } |
| 242 | if strings.Contains(joined, "lchat-router") { |
| 243 | t.Fatalf("router must never restart: %v", *calls) |
| 244 | } |
| 245 | if got := readFailedControlVersion(); got != "" { |
| 246 | t.Fatalf("a committed build must not be quarantined: %q", got) |
| 247 | } |
| 248 | } |
| 249 | |
| 250 | // F3: a head that fails its health check is quarantined (persisted under |
| 251 | // optDir), so the next hourly pass short-circuits it instead of reinstalling + |
| 252 | // rolling it back forever — and the quarantine lifts when the channel moves to |
| 253 | // a good build. |
| 254 | func TestControlSelfUpdateQuarantinesFailedHead(t *testing.T) { |
| 255 | newBin := []byte("NEW-CONTROL-BINARY") |
| 256 | self, entry, calls := selfUpdateEnv(t, newBin) |
| 257 | setBuild := fakeControlSocket(t, buildVersion) // stale: reports OLD after the swap |
| 258 | m := &releaseManifest{Schema: 1, GeneratedAt: time.Now(), Channels: map[string]releaseEntry{"stable": entry}} |
| 259 | |
| 260 | // First pass: the head fails its health check, is rolled back and quarantined. |
| 261 | if err := controlSelfUpdate(m, loadFirstSeen(), nil, false, true); err == nil { |
| 262 | t.Fatal("an unhealthy head must return an error") |
| 263 | } |
| 264 | if got := readFailedControlVersion(); got != entry.Version { |
| 265 | t.Fatalf("failed head not quarantined: %q", got) |
| 266 | } |
| 267 | if got, _ := os.ReadFile(self); string(got) != "OLD-CONTROL-BINARY" { |
| 268 | t.Fatalf("binary not rolled back: %q", got) |
| 269 | } |
| 270 | |
| 271 | // Second pass on the SAME head: the quarantine short-circuits it — no |
| 272 | // download, no swap, no restarts. |
| 273 | before := len(*calls) |
| 274 | if err := controlSelfUpdate(m, loadFirstSeen(), nil, false, true); err != nil { |
| 275 | t.Fatalf("quarantined head should be a silent no-op, got: %v", err) |
| 276 | } |
| 277 | if len(*calls) != before { |
| 278 | t.Fatalf("quarantined head must not restart anything: %v", (*calls)[before:]) |
| 279 | } |
| 280 | if got, _ := os.ReadFile(self); string(got) != "OLD-CONTROL-BINARY" { |
| 281 | t.Fatalf("quarantined head must not swap: %q", got) |
| 282 | } |
| 283 | |
| 284 | // The channel moves to a DIFFERENT, healthy head: the quarantine clears and |
| 285 | // the new build commits. |
| 286 | e2 := entry |
| 287 | e2.Version = "def5678" // same download bytes + sha, new version id |
| 288 | setBuild(e2.Version) // socket now reports the new build → healthy |
| 289 | m2 := &releaseManifest{Schema: 1, GeneratedAt: time.Now(), Channels: map[string]releaseEntry{"stable": e2}} |
| 290 | if err := controlSelfUpdate(m2, loadFirstSeen(), nil, false, true); err != nil { |
| 291 | t.Fatalf("healthy new head should commit, got: %v", err) |
| 292 | } |
| 293 | if got := readFailedControlVersion(); got != "" { |
| 294 | t.Fatalf("quarantine should be cleared after the channel moved on, still %q", got) |
| 295 | } |
| 296 | if got, _ := os.ReadFile(self); string(got) != string(newBin) { |
| 297 | t.Fatalf("new head not committed: %q", got) |
| 298 | } |
| 299 | } |
| 300 | |
| 301 | // F4: a head that failed a health check on an INSTANCE this pass (in the |
| 302 | // poisoned set) must never then become the control plane — no download, no |
| 303 | // swap, no quarantine write. |
| 304 | func TestControlSelfUpdateSkipsPoisonedHead(t *testing.T) { |
| 305 | newBin := []byte("NEW-CONTROL-BINARY") |
| 306 | self, entry, calls := selfUpdateEnv(t, newBin) |
| 307 | m := &releaseManifest{Schema: 1, GeneratedAt: time.Now(), Channels: map[string]releaseEntry{"stable": entry}} |
| 308 | |
| 309 | poisoned := map[string]bool{entry.Version: true} |
| 310 | if err := controlSelfUpdate(m, loadFirstSeen(), poisoned, false, true); err != nil { |
| 311 | t.Fatalf("a poisoned head should be a silent no-op, got: %v", err) |
| 312 | } |
| 313 | if got, _ := os.ReadFile(self); string(got) != "OLD-CONTROL-BINARY" { |
| 314 | t.Fatalf("poisoned head must not swap: %q", got) |
| 315 | } |
| 316 | if len(*calls) != 0 { |
| 317 | t.Fatalf("poisoned head must not restart anything: %v", *calls) |
| 318 | } |
| 319 | if got := readFailedControlVersion(); got != "" { |
| 320 | t.Fatalf("skipping a poisoned head must not write a quarantine: %q", got) |
| 321 | } |
| 322 | } |
| 323 | |
| 324 | // A head still inside its bake window is held — the trust plane is the LAST |
| 325 | // thing on a box that should take a new build. |
| 326 | func TestControlSelfUpdateHonorsBake(t *testing.T) { |
| 327 | self, entry, calls := selfUpdateEnv(t, []byte("NEW")) |
| 328 | m := &releaseManifest{Schema: 1, GeneratedAt: time.Now(), Channels: map[string]releaseEntry{"stable": entry}} |
| 329 | bake := loadFirstSeen() |
| 330 | bake.observe(m, time.Now()) // first sighting is now → inside the window |
| 331 | |
| 332 | controlSelfUpdate(m, bake, nil, false, false) |
| 333 | |
| 334 | if got, _ := os.ReadFile(self); string(got) != "OLD-CONTROL-BINARY" { |
| 335 | t.Fatalf("baking head must not install: %q", got) |
| 336 | } |
| 337 | if len(*calls) != 0 { |
| 338 | t.Fatalf("no restarts while held: %v", *calls) |
| 339 | } |
| 340 | } |
| 341 | |
| 342 | // "off" and already-at-head are silent no-ops; a sha mismatch leaves the |
| 343 | // binary untouched (the verified-download contract, pinned here for the |
| 344 | // control path specifically). |
| 345 | func TestControlSelfUpdateNoOpsAndBadSha(t *testing.T) { |
| 346 | self, entry, _ := selfUpdateEnv(t, []byte("NEW")) |
| 347 | |
| 348 | os.Setenv("LCHAT_CONTROL_CHANNEL", "off") |
| 349 | controlSelfUpdate(&releaseManifest{Channels: map[string]releaseEntry{"stable": entry}}, loadFirstSeen(), nil, false, true) |
| 350 | if got, _ := os.ReadFile(self); string(got) != "OLD-CONTROL-BINARY" { |
| 351 | t.Fatal("off must be a no-op") |
| 352 | } |
| 353 | os.Setenv("LCHAT_CONTROL_CHANNEL", "stable") |
| 354 | |
| 355 | same := entry |
| 356 | same.Version = buildVersion |
| 357 | controlSelfUpdate(&releaseManifest{Channels: map[string]releaseEntry{"stable": same}}, loadFirstSeen(), nil, false, true) |
| 358 | if got, _ := os.ReadFile(self); string(got) != "OLD-CONTROL-BINARY" { |
| 359 | t.Fatal("at-head must be a no-op") |
| 360 | } |
| 361 | |
| 362 | bad := entry |
| 363 | bad.SHA256 = strings.Repeat("00", 32) |
| 364 | bad.ARM64SHA256 = strings.Repeat("00", 32) // whichever arch runs the test |
| 365 | controlSelfUpdate(&releaseManifest{Channels: map[string]releaseEntry{"stable": bad}}, loadFirstSeen(), nil, false, true) |
| 366 | if got, _ := os.ReadFile(self); string(got) != "OLD-CONTROL-BINARY" { |
| 367 | t.Fatal("a sha mismatch must never install") |
| 368 | } |
| 369 | } |
| 370 | |
amadan