| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "net/http" |
| 5 | "os" |
| 6 | "path/filepath" |
| 7 | "testing" |
| 8 | "time" |
| 9 | ) |
| 10 | |
| 11 | // The supply-chain bake window (bake.go). These tests run with the real |
| 12 | // delays (stable 72h / beta 24h) — everything else in updater_test.go runs |
| 13 | // zeroBake()d so the two concerns stay separate. |
| 14 | |
| 15 | func TestUpdatePassBakeHoldsNewStableHead(t *testing.T) { |
| 16 | withTempOpt(t) |
| 17 | calls := stubSystemctl(t) |
| 18 | shrink := deployHealthTimeout |
| 19 | deployHealthTimeout = 2 * time.Second |
| 20 | t.Cleanup(func() { deployHealthTimeout = shrink }) |
| 21 | |
| 22 | _, setChannels := startFeed(t, map[string]string{"new1234": "held-binary"}) |
| 23 | setChannels(map[string]string{"stable": "new1234"}) |
| 24 | |
| 25 | reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) |
| 26 | if err != nil { |
| 27 | t.Fatal(err) |
| 28 | } |
| 29 | s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "") |
| 30 | seedInstance(t, reg, "h1.example.com", s1, "old1111") |
| 31 | |
| 32 | // First sighting: the clock starts and NOTHING moves — no download, no |
| 33 | // restart. The window is measured from this box's own observation. |
| 34 | if err := updatePass(reg, "lchat@", false, false); err != nil { |
| 35 | t.Fatalf("updatePass: %v", err) |
| 36 | } |
| 37 | if v, _ := liveVersion("h1.example.com"); v != "old1111" { |
| 38 | t.Fatalf("brand-new stable head installed immediately: live = %q", v) |
| 39 | } |
| 40 | if _, err := os.Stat(versionBinPath("new1234")); !os.IsNotExist(err) { |
| 41 | t.Fatal("binary downloaded during the bake window") |
| 42 | } |
| 43 | if len(*calls) != 0 { |
| 44 | t.Fatalf("restarts during the bake window: %v", *calls) |
| 45 | } |
| 46 | st := loadFirstSeen() |
| 47 | seen, ok := st.Channels["stable"]["new1234"] |
| 48 | if !ok { |
| 49 | t.Fatal("first sighting not recorded") |
| 50 | } |
| 51 | if st.Jitter < 0 || st.Jitter >= 0.25 { |
| 52 | t.Fatalf("minted jitter %v outside [0, 0.25)", st.Jitter) |
| 53 | } |
| 54 | |
| 55 | // A second pass inside the window holds too, and doesn't move the clock. |
| 56 | if err := updatePass(reg, "lchat@", false, false); err != nil { |
| 57 | t.Fatal(err) |
| 58 | } |
| 59 | if v, _ := liveVersion("h1.example.com"); v != "old1111" { |
| 60 | t.Fatalf("head installed inside the window: live = %q", v) |
| 61 | } |
| 62 | if again := loadFirstSeen().Channels["stable"]["new1234"]; !again.Equal(seen) { |
| 63 | t.Fatalf("re-observation moved the clock: %v -> %v", seen, again) |
| 64 | } |
| 65 | |
| 66 | // Once the window (72h plus any jitter, ≤ 90h) has passed, it installs. |
| 67 | st.Channels["stable"]["new1234"] = time.Now().Add(-91 * time.Hour) |
| 68 | if err := st.save(); err != nil { |
| 69 | t.Fatal(err) |
| 70 | } |
| 71 | if err := updatePass(reg, "lchat@", false, false); err != nil { |
| 72 | t.Fatal(err) |
| 73 | } |
| 74 | if v, _ := liveVersion("h1.example.com"); v != "new1234" { |
| 75 | t.Fatalf("head not installed after the window: live = %q", v) |
| 76 | } |
| 77 | } |
| 78 | |
| 79 | // Pulling a channel back to a previously-seen version applies immediately — |
| 80 | // evacuating a bad release must never itself sit in a bake window. |
| 81 | func TestUpdatePassBakePullBackIsImmediate(t *testing.T) { |
| 82 | withTempOpt(t) |
| 83 | stubSystemctl(t) |
| 84 | shrink := deployHealthTimeout |
| 85 | deployHealthTimeout = 2 * time.Second |
| 86 | t.Cleanup(func() { deployHealthTimeout = shrink }) |
| 87 | |
| 88 | _, setChannels := startFeed(t, map[string]string{"goodaaa": "good-binary", "badbbbb": "bad-binary"}) |
| 89 | setChannels(map[string]string{"stable": "badbbbb"}) |
| 90 | |
| 91 | // goodaaa sat at the stable head long ago (this box saw it); badbbbb is |
| 92 | // the fresh, about-to-be-pulled release. |
| 93 | st := loadFirstSeen() |
| 94 | st.Jitter = 0.1 |
| 95 | st.Channels = map[string]map[string]time.Time{ |
| 96 | "stable": {"goodaaa": time.Now().Add(-200 * time.Hour)}, |
| 97 | } |
| 98 | if err := st.save(); err != nil { |
| 99 | t.Fatal(err) |
| 100 | } |
| 101 | |
| 102 | reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) |
| 103 | if err != nil { |
| 104 | t.Fatal(err) |
| 105 | } |
| 106 | s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "") |
| 107 | seedInstance(t, reg, "h1.example.com", s1, "old1111") |
| 108 | |
| 109 | // The fresh head is held... |
| 110 | if err := updatePass(reg, "lchat@", false, false); err != nil { |
| 111 | t.Fatal(err) |
| 112 | } |
| 113 | if v, _ := liveVersion("h1.example.com"); v != "old1111" { |
| 114 | t.Fatalf("fresh head installed: live = %q", v) |
| 115 | } |
| 116 | |
| 117 | // ...the operator pulls stable back to the known-good version, and the |
| 118 | // next pass applies it with no further wait. |
| 119 | setChannels(map[string]string{"stable": "goodaaa"}) |
| 120 | if err := updatePass(reg, "lchat@", false, false); err != nil { |
| 121 | t.Fatal(err) |
| 122 | } |
| 123 | if v, _ := liveVersion("h1.example.com"); v != "goodaaa" { |
| 124 | t.Fatalf("pull-back target was held: live = %q", v) |
| 125 | } |
| 126 | } |
| 127 | |
| 128 | // --ignore-bake is the deliberate local override: root on the box applying a |
| 129 | // held head NOW (there is intentionally no publisher-side equivalent). |
| 130 | func TestUpdatePassIgnoreBakeOverrides(t *testing.T) { |
| 131 | withTempOpt(t) |
| 132 | stubSystemctl(t) |
| 133 | shrink := deployHealthTimeout |
| 134 | deployHealthTimeout = 2 * time.Second |
| 135 | t.Cleanup(func() { deployHealthTimeout = shrink }) |
| 136 | |
| 137 | _, setChannels := startFeed(t, map[string]string{"new1234": "content"}) |
| 138 | setChannels(map[string]string{"stable": "new1234"}) |
| 139 | |
| 140 | reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) |
| 141 | if err != nil { |
| 142 | t.Fatal(err) |
| 143 | } |
| 144 | s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "") |
| 145 | seedInstance(t, reg, "h1.example.com", s1, "old1111") |
| 146 | |
| 147 | if err := updatePass(reg, "lchat@", false, true); err != nil { |
| 148 | t.Fatal(err) |
| 149 | } |
| 150 | if v, _ := liveVersion("h1.example.com"); v != "new1234" { |
| 151 | t.Fatalf("--ignore-bake didn't install: live = %q", v) |
| 152 | } |
| 153 | } |
| 154 | |
| 155 | // edge has no window: it's the operator's own deploy path (`publish && ship` |
| 156 | // must keep converging the fleet in one pass). |
| 157 | func TestUpdatePassEdgeHasNoBake(t *testing.T) { |
| 158 | withTempOpt(t) |
| 159 | stubSystemctl(t) |
| 160 | shrink := deployHealthTimeout |
| 161 | deployHealthTimeout = 2 * time.Second |
| 162 | t.Cleanup(func() { deployHealthTimeout = shrink }) |
| 163 | |
| 164 | _, setChannels := startFeed(t, map[string]string{"edge777": "edge-binary"}) |
| 165 | setChannels(map[string]string{"edge": "edge777"}) |
| 166 | |
| 167 | reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) |
| 168 | if err != nil { |
| 169 | t.Fatal(err) |
| 170 | } |
| 171 | s1 := startFakeInstance(t, "h1.example.com", http.StatusOK, |
| 172 | updatePrefs{AutoUpdate: true, AutoDownload: true, Channel: "edge"}, "") |
| 173 | seedInstance(t, reg, "h1.example.com", s1, "old1111") |
| 174 | |
| 175 | if err := updatePass(reg, "lchat@", false, false); err != nil { |
| 176 | t.Fatal(err) |
| 177 | } |
| 178 | if v, _ := liveVersion("h1.example.com"); v != "edge777" { |
| 179 | t.Fatalf("edge head was held: live = %q", v) |
| 180 | } |
| 181 | } |
| 182 | |
| 183 | // Hibernating hosts honour the window too — their repoints have no health |
| 184 | // check, so they must never be the first to run a fresh head anyway. |
| 185 | func TestUpdatePassBakeHoldsHibernating(t *testing.T) { |
| 186 | withTempOpt(t) |
| 187 | calls := stubSystemctl(t) |
| 188 | |
| 189 | oldData := instanceDataDir |
| 190 | instanceDataDir = t.TempDir() |
| 191 | t.Cleanup(func() { instanceDataDir = oldData }) |
| 192 | |
| 193 | _, setChannels := startFeed(t, map[string]string{"new1234": "content"}) |
| 194 | setChannels(map[string]string{"stable": "new1234"}) |
| 195 | |
| 196 | reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) |
| 197 | if err != nil { |
| 198 | t.Fatal(err) |
| 199 | } |
| 200 | src := filepath.Join(t.TempDir(), "old") |
| 201 | writeFakeBinary(t, src, "old-binary") |
| 202 | if err := stageVersion(src, "old1111"); err != nil { |
| 203 | t.Fatal(err) |
| 204 | } |
| 205 | if err := pointLive("h1.hib", "old1111"); err != nil { |
| 206 | t.Fatal(err) |
| 207 | } |
| 208 | if err := reg.upsert(Route{Host: "h1.hib", Kind: "instance", Status: "ready", |
| 209 | Socket: filepath.Join(t.TempDir(), "no.sock"), Hibernate: true, Version: "old1111"}); err != nil { |
| 210 | t.Fatal(err) |
| 211 | } |
| 212 | db, err := openDB(filepath.Join(instanceDataDir, "h1.hib.db")) |
| 213 | if err != nil { |
| 214 | t.Fatal(err) |
| 215 | } |
| 216 | db.sql.Close() |
| 217 | |
| 218 | if err := updatePass(reg, "lchat@", false, false); err != nil { |
| 219 | t.Fatal(err) |
| 220 | } |
| 221 | if v, _ := liveVersion("h1.hib"); v != "old1111" { |
| 222 | t.Fatalf("hibernating host repointed during the bake window: live = %q", v) |
| 223 | } |
| 224 | if len(*calls) != 0 { |
| 225 | t.Fatalf("systemctl touched: %v", *calls) |
| 226 | } |
| 227 | |
| 228 | // After the window it repoints as usual. |
| 229 | st := loadFirstSeen() |
| 230 | st.Channels["stable"]["new1234"] = time.Now().Add(-91 * time.Hour) |
| 231 | if err := st.save(); err != nil { |
| 232 | t.Fatal(err) |
| 233 | } |
| 234 | if err := updatePass(reg, "lchat@", false, false); err != nil { |
| 235 | t.Fatal(err) |
| 236 | } |
| 237 | if v, _ := liveVersion("h1.hib"); v != "new1234" { |
| 238 | t.Fatalf("hibernating host not repointed after the window: live = %q", v) |
| 239 | } |
| 240 | } |
| 241 | |
| 242 | // Dry runs compute bake eligibility but never persist the clock — a dry run |
| 243 | // must not start windows ticking. |
| 244 | func TestUpdatePassBakeDryRunPersistsNothing(t *testing.T) { |
| 245 | withTempOpt(t) |
| 246 | stubSystemctl(t) |
| 247 | |
| 248 | _, setChannels := startFeed(t, map[string]string{"new1234": "content"}) |
| 249 | setChannels(map[string]string{"stable": "new1234"}) |
| 250 | |
| 251 | reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) |
| 252 | if err != nil { |
| 253 | t.Fatal(err) |
| 254 | } |
| 255 | s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "") |
| 256 | seedInstance(t, reg, "h1.example.com", s1, "old1111") |
| 257 | |
| 258 | if err := updatePass(reg, "lchat@", true, false); err != nil { |
| 259 | t.Fatal(err) |
| 260 | } |
| 261 | if _, err := os.Stat(firstSeenPath()); !os.IsNotExist(err) { |
| 262 | t.Fatal("dry run persisted the bake clock") |
| 263 | } |
| 264 | } |
| 265 | |
| 266 | func TestFirstSeenObservePruneAndEligibility(t *testing.T) { |
| 267 | withTempOpt(t) |
| 268 | |
| 269 | // A fresh box mints its jitter alongside the first observation. |
| 270 | st := loadFirstSeen() |
| 271 | now := time.Now() |
| 272 | if !st.observe(&releaseManifest{Channels: map[string]releaseEntry{"stable": {Version: "v0"}}}, now) { |
| 273 | t.Fatal("first observation reported no change") |
| 274 | } |
| 275 | if st.Jitter < 0 || st.Jitter >= 0.25 { |
| 276 | t.Fatalf("jitter %v outside [0, 0.25)", st.Jitter) |
| 277 | } |
| 278 | // Re-observing the same head is a no-op. |
| 279 | if st.observe(&releaseManifest{Channels: map[string]releaseEntry{"stable": {Version: "v0"}}}, now.Add(time.Hour)) { |
| 280 | t.Fatal("re-observation reported a change") |
| 281 | } |
| 282 | |
| 283 | // Eligibility: firstSeen + delay*(1+jitter), exactly. |
| 284 | st.Jitter = 0.2 // 0.25 is exclusive — loadFirstSeen clamps it as corrupt |
| 285 | seen := st.Channels["stable"]["v0"] |
| 286 | at, ok := st.bakeEligibleAt("stable", "v0") |
| 287 | if want := seen.Add(time.Duration(float64(72*time.Hour) * 1.2)); !ok || !at.Equal(want) { |
| 288 | t.Fatalf("eligibleAt = %v ok=%v, want %v", at, ok, want) |
| 289 | } |
| 290 | if _, ok := st.bakeEligibleAt("stable", "never-seen"); ok { |
| 291 | t.Fatal("unseen version reported eligible time") |
| 292 | } |
| 293 | // An unseen version is held, an old one isn't, ignore always passes. |
| 294 | if bakeHold(st, "stable", "never-seen", false, now) == "" { |
| 295 | t.Fatal("unseen version not held") |
| 296 | } |
| 297 | if bakeHold(st, "stable", "never-seen", true, now) != "" { |
| 298 | t.Fatal("ignore-bake didn't override") |
| 299 | } |
| 300 | st.Channels["stable"]["ancient"] = now.Add(-100 * time.Hour) |
| 301 | if hold := bakeHold(st, "stable", "ancient", false, now); hold != "" { |
| 302 | t.Fatalf("aged-out version still held: %s", hold) |
| 303 | } |
| 304 | |
| 305 | // History is pruned to firstSeenKeep entries, oldest first — the current |
| 306 | // head always survives. |
| 307 | for i := 0; i < firstSeenKeep+5; i++ { |
| 308 | st.observe(&releaseManifest{Channels: map[string]releaseEntry{ |
| 309 | "stable": {Version: string(rune('a' + i))}, |
| 310 | }}, now.Add(time.Duration(i)*time.Minute)) |
| 311 | } |
| 312 | if n := len(st.Channels["stable"]); n != firstSeenKeep { |
| 313 | t.Fatalf("history size = %d, want %d", n, firstSeenKeep) |
| 314 | } |
| 315 | if _, ok := st.Channels["stable"][string(rune('a'+firstSeenKeep+4))]; !ok { |
| 316 | t.Fatal("newest head was pruned") |
| 317 | } |
| 318 | |
| 319 | // The clock round-trips through disk. |
| 320 | if err := st.save(); err != nil { |
| 321 | t.Fatal(err) |
| 322 | } |
| 323 | got := loadFirstSeen() |
| 324 | if got.Jitter != st.Jitter || len(got.Channels["stable"]) != firstSeenKeep { |
| 325 | t.Fatalf("reload = jitter %v, %d entries", got.Jitter, len(got.Channels["stable"])) |
| 326 | } |
| 327 | } |
| 328 | |
amadan