| 1 | package activator |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/sha256" |
| 6 | "encoding/hex" |
| 7 | "errors" |
| 8 | "fmt" |
| 9 | "maps" |
| 10 | "os" |
| 11 | "path/filepath" |
| 12 | "slices" |
| 13 | "strconv" |
| 14 | "strings" |
| 15 | "sync" |
| 16 | "sync/atomic" |
| 17 | "testing" |
| 18 | "time" |
| 19 | |
| 20 | "github.com/carlosframework/platform/internal/cgroup" |
| 21 | "github.com/carlosframework/platform/internal/hostclaim" |
| 22 | "github.com/carlosframework/platform/internal/hostwrite" |
| 23 | "github.com/carlosframework/platform/internal/instancectl" |
| 24 | "github.com/carlosframework/platform/internal/registry" |
| 25 | "github.com/carlosframework/platform/internal/release" |
| 26 | "github.com/carlosframework/platform/internal/rolloutwords" |
| 27 | ) |
| 28 | |
| 29 | // planGenBackend is plan 3's SwapBackend (startPinnedAt's staged spawn) plus |
| 30 | // plan 2's KillGen over the package's fakeBackend. Its generations are |
| 31 | // fakeProcs a test can crash. It also starts a legacy sidecar on a |
| 32 | // generation's staged build (SidecarGenStarter), recording the generation |
| 33 | // it was handed. |
| 34 | type planGenBackend struct { |
| 35 | *fakeBackend |
| 36 | gmu sync.Mutex |
| 37 | gens []Gen |
| 38 | procs map[uint64]*fakeProc |
| 39 | killed []uint64 |
| 40 | killErr error |
| 41 | gate chan struct{} |
| 42 | sidecarGens []Gen |
| 43 | } |
| 44 | |
| 45 | func newPlanGenBackend() *planGenBackend { |
| 46 | return &planGenBackend{fakeBackend: &fakeBackend{}, procs: map[uint64]*fakeProc{}} |
| 47 | } |
| 48 | |
| 49 | func (b *planGenBackend) StartStaged(ctx context.Context, _ registry.Route, g Gen) (Gen, Proc, error) { |
| 50 | b.gmu.Lock() |
| 51 | gate := b.gate |
| 52 | b.gmu.Unlock() |
| 53 | if gate != nil { |
| 54 | <-gate |
| 55 | } |
| 56 | // The caller's exec gate at the launch point (final review F1), as the |
| 57 | // real backend asks it after its preparation. |
| 58 | if err := runExecGate(ctx); err != nil { |
| 59 | return g, nil, err |
| 60 | } |
| 61 | b.gmu.Lock() |
| 62 | defer b.gmu.Unlock() |
| 63 | g.Socket = "/run/carlos/g/test." + strconv.FormatUint(g.N, 10) + ".sock" |
| 64 | g.Control = g.Socket + ".control" |
| 65 | p := &fakeProc{label: "gen:" + g.Build, rec: b.fakeBackend} |
| 66 | b.gens = append(b.gens, g) |
| 67 | b.procs[g.N] = p |
| 68 | return g, p, nil |
| 69 | } |
| 70 | |
| 71 | func (b *planGenBackend) GenTier(registry.Route, string) cgroup.Limit { return cgroup.TierDefault } |
| 72 | func (b *planGenBackend) GenReady(registry.Route, Gen, Proc) {} |
| 73 | func (b *planGenBackend) RaiseGenParent(context.Context, registry.Route, cgroup.Limit, cgroup.Limit) error { |
| 74 | return nil |
| 75 | } |
| 76 | func (b *planGenBackend) RestoreGenParent(context.Context, registry.Route, cgroup.Limit) error { |
| 77 | return nil |
| 78 | } |
| 79 | func (b *planGenBackend) GenSwitched(registry.Route, Gen, Proc) error { return nil } |
| 80 | |
| 81 | func (b *planGenBackend) KillGen(_ context.Context, _ registry.Route, n uint64) error { |
| 82 | b.gmu.Lock() |
| 83 | defer b.gmu.Unlock() |
| 84 | b.killed = append(b.killed, n) |
| 85 | if b.killErr != nil { |
| 86 | return b.killErr |
| 87 | } |
| 88 | if p := b.procs[n]; p != nil { |
| 89 | p.exit() |
| 90 | } |
| 91 | return nil |
| 92 | } |
| 93 | |
| 94 | func (b *planGenBackend) StartSidecarGen(_ context.Context, _ registry.Route, g Gen) (Proc, error) { |
| 95 | b.gmu.Lock() |
| 96 | b.sidecarGens = append(b.sidecarGens, g) |
| 97 | b.gmu.Unlock() |
| 98 | return &fakeProc{label: "sidecar", rec: b.fakeBackend}, nil |
| 99 | } |
| 100 | |
| 101 | var ( |
| 102 | _ SwapBackend = (*planGenBackend)(nil) |
| 103 | _ genKiller = (*planGenBackend)(nil) |
| 104 | _ SidecarGenStarter = (*planGenBackend)(nil) |
| 105 | ) |
| 106 | |
| 107 | func (b *planGenBackend) started() []Gen { |
| 108 | b.gmu.Lock() |
| 109 | defer b.gmu.Unlock() |
| 110 | return append([]Gen(nil), b.gens...) |
| 111 | } |
| 112 | |
| 113 | func (b *planGenBackend) kills() []uint64 { |
| 114 | b.gmu.Lock() |
| 115 | defer b.gmu.Unlock() |
| 116 | return append([]uint64(nil), b.killed...) |
| 117 | } |
| 118 | |
| 119 | func (b *planGenBackend) procOf(n uint64) *fakeProc { |
| 120 | b.gmu.Lock() |
| 121 | defer b.gmu.Unlock() |
| 122 | return b.procs[n] |
| 123 | } |
| 124 | |
| 125 | func (b *planGenBackend) sidecarsOn() []Gen { |
| 126 | b.gmu.Lock() |
| 127 | defer b.gmu.Unlock() |
| 128 | return append([]Gen(nil), b.sidecarGens...) |
| 129 | } |
| 130 | |
| 131 | // stagedID writes a staged build and returns its identity: plan 3's |
| 132 | // startPinnedAt verifies the bytes before anything runs. |
| 133 | func stagedID(t *testing.T, dir, ver string) release.StagedIdentity { |
| 134 | t.Helper() |
| 135 | body := []byte("build " + ver) |
| 136 | path := filepath.Join(dir, ver) |
| 137 | if err := os.WriteFile(path, body, 0o755); err != nil { |
| 138 | t.Fatal(err) |
| 139 | } |
| 140 | sum := sha256.Sum256(body) |
| 141 | return release.StagedIdentity{Version: ver, StagedPath: path, Digest: hex.EncodeToString(sum[:]), Demand: "hold-" + ver} |
| 142 | } |
| 143 | |
| 144 | type genLog struct { |
| 145 | mu sync.Mutex |
| 146 | evs []GenEvent |
| 147 | } |
| 148 | |
| 149 | func (l *genLog) add(ev GenEvent) { |
| 150 | l.mu.Lock() |
| 151 | defer l.mu.Unlock() |
| 152 | l.evs = append(l.evs, ev) |
| 153 | } |
| 154 | |
| 155 | func (l *genLog) kinds() []GenEventKind { |
| 156 | l.mu.Lock() |
| 157 | defer l.mu.Unlock() |
| 158 | out := make([]GenEventKind, 0, len(l.evs)) |
| 159 | for _, ev := range l.evs { |
| 160 | out = append(out, ev.Kind) |
| 161 | } |
| 162 | return out |
| 163 | } |
| 164 | |
| 165 | func (l *genLog) wait(t *testing.T, kind GenEventKind) GenEvent { |
| 166 | t.Helper() |
| 167 | deadline := time.Now().Add(5 * time.Second) |
| 168 | for time.Now().Before(deadline) { |
| 169 | l.mu.Lock() |
| 170 | for _, ev := range l.evs { |
| 171 | if ev.Kind == kind { |
| 172 | l.mu.Unlock() |
| 173 | return ev |
| 174 | } |
| 175 | } |
| 176 | l.mu.Unlock() |
| 177 | time.Sleep(5 * time.Millisecond) |
| 178 | } |
| 179 | t.Fatalf("no %s event; saw %v", kind, l.kinds()) |
| 180 | return GenEvent{} |
| 181 | } |
| 182 | |
| 183 | func planActivator(t *testing.T, plan WakePlan) (*Service, *planGenBackend, *genLog) { |
| 184 | t.Helper() |
| 185 | b := newPlanGenBackend() |
| 186 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 187 | if err != nil { |
| 188 | t.Fatal(err) |
| 189 | } |
| 190 | a := New(Config{Bucket: "test-bucket", Idle: time.Hour}, b, reg) |
| 191 | a.gens = true // plan 1's switch, as CARLOS_SWAP=overlap sets it before New |
| 192 | evs := &genLog{} |
| 193 | a.OnGen = evs.add |
| 194 | a.Planner = func(context.Context, registry.Route) WakePlan { return plan } |
| 195 | a.genHealth = func(context.Context, Gen) error { return nil } |
| 196 | return a, b, evs |
| 197 | } |
| 198 | |
| 199 | func shrinkFirstWake(t *testing.T, budget, reserve, settle time.Duration) { |
| 200 | t.Helper() |
| 201 | b, r, s, k := firstWakeBudget, firstWakeKillReserve, firstWakeSettle, firstWakeKillBound |
| 202 | firstWakeBudget, firstWakeKillReserve, firstWakeSettle = budget, reserve, settle |
| 203 | t.Cleanup(func() { firstWakeBudget, firstWakeKillReserve, firstWakeSettle, firstWakeKillBound = b, r, s, k }) |
| 204 | } |
| 205 | |
| 206 | func TestFirstWakeFallsBackToTheGoodBuildWithinTheBudget(t *testing.T) { |
| 207 | dir := t.TempDir() |
| 208 | good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| 209 | a, b, evs := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good}) |
| 210 | var mu sync.Mutex |
| 211 | deadlines := map[string]time.Time{} |
| 212 | a.genHealth = func(ctx context.Context, g Gen) error { |
| 213 | dl, _ := ctx.Deadline() |
| 214 | mu.Lock() |
| 215 | deadlines[g.Build] = dl |
| 216 | mu.Unlock() |
| 217 | if g.Build == "v2" { |
| 218 | return errors.New("GET / answered 500") |
| 219 | } |
| 220 | return nil |
| 221 | } |
| 222 | start := time.Now() |
| 223 | done, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| 224 | if err != nil { |
| 225 | t.Fatalf("the waiting request was not served by the good build: %v", err) |
| 226 | } |
| 227 | done() |
| 228 | gens := b.started() |
| 229 | if len(gens) != 2 || gens[0].Build != "v2" || gens[1].Build != "v1" { |
| 230 | t.Fatalf("spawned %+v, want v2 then the good v1", gens) |
| 231 | } |
| 232 | mu.Lock() |
| 233 | window := deadlines["v2"].Sub(start) |
| 234 | mu.Unlock() |
| 235 | if window < 29*time.Second || window > 31*time.Second { |
| 236 | t.Fatalf("B's health window ends %v after the wake began, want 30s: the 55s deadline minus 25s", window) |
| 237 | } |
| 238 | if k := b.kills(); len(k) != 1 || k[0] != gens[0].N { |
| 239 | t.Fatalf("killed %v, want B's generation %d alone, by cgroup.kill", k, gens[0].N) |
| 240 | } |
| 241 | ev := evs.wait(t, GenFellBack) |
| 242 | if ev.Failed.Version != "v2" || ev.Gen.Build != "v1" { |
| 243 | t.Fatalf("fallback event = %+v", ev) |
| 244 | } |
| 245 | if g, ok := a.Current("one.example.com"); !ok || g.Build != "v1" { |
| 246 | t.Fatalf("Current = %+v %v, want the good build serving", g, ok) |
| 247 | } |
| 248 | } |
| 249 | |
| 250 | // Spec §6.4: once a hold build has executed, nothing falls back. |
| 251 | func TestAHoldFirstWakeNeverFallsBack(t *testing.T) { |
| 252 | dir := t.TempDir() |
| 253 | good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| 254 | a, b, evs := planActivator(t, WakePlan{Boot: next, Mode: ModeHold, FirstWake: true, Fallback: &good}) |
| 255 | a.genHealth = func(context.Context, Gen) error { return errors.New("GET / answered 500") } |
| 256 | if _, err := a.beginRequest(context.Background(), testRoute("one.example.com")); err == nil { |
| 257 | t.Fatal("a failed hold first wake was served") |
| 258 | } |
| 259 | if gens := b.started(); len(gens) != 1 { |
| 260 | t.Fatalf("spawned %+v, want B alone", gens) |
| 261 | } |
| 262 | if ev := evs.wait(t, GenFailed); ev.Mode != ModeHold || ev.Failed.Version != "v2" { |
| 263 | t.Fatalf("failure event = %+v", ev) |
| 264 | } |
| 265 | } |
| 266 | |
| 267 | // Spec §7.4, §8.1–8.4 (P5 review finding 3): a planned wake re-checks its |
| 268 | // build at the exec, after the lease and the restore. Here a sibling's |
| 269 | // failure holds v2 between the planning and the exec: the wake is planned |
| 270 | // again on the restored state and boots the good build, and v2 never |
| 271 | // execs. Without the re-check v2 ran after its hold. |
| 272 | func TestAPlannedWakeRechecksItsBuildAtTheExec(t *testing.T) { |
| 273 | dir := t.TempDir() |
| 274 | good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| 275 | a, b, evs := planActivator(t, WakePlan{}) |
| 276 | var held atomic.Bool |
| 277 | check := func(build string) error { |
| 278 | if build == "v2" && held.Load() { |
| 279 | return errors.New("v2 failed on a sibling") |
| 280 | } |
| 281 | return nil |
| 282 | } |
| 283 | var plans atomic.Int32 |
| 284 | a.Planner = func(context.Context, registry.Route) WakePlan { |
| 285 | if plans.Add(1) == 1 { |
| 286 | held.Store(true) // the sibling's failure lands after this plan is made |
| 287 | return WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good, Check: check} |
| 288 | } |
| 289 | return WakePlan{Boot: good, Mode: ModeOverlap, Check: check} |
| 290 | } |
| 291 | done, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| 292 | if err != nil { |
| 293 | t.Fatalf("the wake: %v", err) |
| 294 | } |
| 295 | done() |
| 296 | if gens := b.started(); len(gens) != 1 || gens[0].Build != "v1" { |
| 297 | t.Fatalf("spawned %+v, want the good v1 alone", gens) |
| 298 | } |
| 299 | if k := evs.kinds(); slices.Contains(k, GenFailed) || slices.Contains(k, GenFellBack) { |
| 300 | t.Fatalf("events %v: a build that never ran was recorded", k) |
| 301 | } |
| 302 | } |
| 303 | |
| 304 | // PF7: the selection gate is asked again AT the exec, inside startPinnedAt's |
| 305 | // gate (after the staged bytes are verified and the tier read), not only |
| 306 | // before the spawn. A failure recorded in that moment holds the build: it |
| 307 | // never execs, the wake is abandoned (errPlanStale), and nothing is |
| 308 | // recorded against the build. |
| 309 | func TestThePlanCheckAtTheExecRefusesWithoutRecording(t *testing.T) { |
| 310 | t.Parallel() |
| 311 | dir := t.TempDir() |
| 312 | good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| 313 | a, b, evs := planActivator(t, WakePlan{}) |
| 314 | var calls atomic.Int32 |
| 315 | check := func(build string) error { |
| 316 | if build == "v2" && calls.Add(1) >= 2 { |
| 317 | return errors.New("v2 failed on a sibling") |
| 318 | } |
| 319 | return nil |
| 320 | } |
| 321 | a.Planner = func(context.Context, registry.Route) WakePlan { |
| 322 | return WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good, Check: check} |
| 323 | } |
| 324 | _, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| 325 | if !errors.Is(err, errPlanStale) { |
| 326 | t.Fatalf("err = %v, want errPlanStale from the exec's gate", err) |
| 327 | } |
| 328 | if got := classifyStep(nil, err); got != stepStale { |
| 329 | t.Fatalf("classifyStep = %v, want stepStale: a refusal at the exec is abandonment", got) |
| 330 | } |
| 331 | if gens := b.started(); len(gens) != 0 { |
| 332 | t.Fatalf("spawned %+v: a held build execed", gens) |
| 333 | } |
| 334 | if calls.Load() < 2 { |
| 335 | t.Fatalf("Check asked %d times, want the second at the exec", calls.Load()) |
| 336 | } |
| 337 | time.Sleep(50 * time.Millisecond) |
| 338 | if k := evs.kinds(); len(k) != 0 { |
| 339 | t.Fatalf("events %v: a refusal before the exec was recorded", k) |
| 340 | } |
| 341 | } |
| 342 | |
| 343 | // PF7: the wake's own locally held lease is part of every planned exec's |
| 344 | // gate. A renewal that fails after the restore execs nothing: no |
| 345 | // generation starts, nothing is recorded, nothing falls back. |
| 346 | func TestAPlannedWakeWhoseLeaseIsLostExecsNothing(t *testing.T) { |
| 347 | t.Parallel() |
| 348 | dir := t.TempDir() |
| 349 | good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| 350 | a, b, evs := planActivator(t, WakePlan{}) |
| 351 | a.Planner = func(context.Context, registry.Route) WakePlan { |
| 352 | return WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good, |
| 353 | Check: func(string) error { |
| 354 | // Asked after the lease and the restore: lose the lease here. |
| 355 | b.lastRep().setErr(errors.New("lease renewal failed")) |
| 356 | return nil |
| 357 | }} |
| 358 | } |
| 359 | _, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| 360 | if !errors.Is(err, errSwapLeaseLost) { |
| 361 | t.Fatalf("err = %v, want the lost lease (errSwapLeaseLost)", err) |
| 362 | } |
| 363 | if gens := b.started(); len(gens) != 0 { |
| 364 | t.Fatalf("spawned %+v on a lost lease", gens) |
| 365 | } |
| 366 | time.Sleep(50 * time.Millisecond) |
| 367 | if k := evs.kinds(); len(k) != 0 { |
| 368 | t.Fatalf("events %v: a refusal before the exec was recorded", k) |
| 369 | } |
| 370 | } |
| 371 | |
| 372 | // Spec §7.4 (P5 review rule a): a wake's fallback passes the ownership |
| 373 | // check at its own exec. Here the host is fenced while B fails its health: |
| 374 | // the good build is not started. B's failure is still reported. Checked |
| 375 | // only before B's exec, the fallback ran on a fenced host. |
| 376 | func TestAFallbackOnAFencedHostIsNotStarted(t *testing.T) { |
| 377 | dir := t.TempDir() |
| 378 | good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| 379 | a, b, evs := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good}) |
| 380 | const host = "one.example.com" |
| 381 | a.genHealth = func(_ context.Context, g Gen) error { |
| 382 | if g.Build != "v2" { |
| 383 | return nil |
| 384 | } |
| 385 | a.mu.Lock() |
| 386 | a.fences = map[string]uint64{host: 1} // maintenance fences the host meanwhile |
| 387 | a.mu.Unlock() |
| 388 | return errors.New("GET / answered 500") |
| 389 | } |
| 390 | if _, err := a.beginRequest(context.Background(), testRoute(host)); err == nil { |
| 391 | t.Fatal("a fenced host's fallback served") |
| 392 | } |
| 393 | if gens := b.started(); len(gens) != 1 || gens[0].Build != "v2" { |
| 394 | t.Fatalf("spawned %+v, want B alone", gens) |
| 395 | } |
| 396 | evs.wait(t, GenFailed) |
| 397 | } |
| 398 | |
| 399 | func TestAnUnprovenKillStartsNoSecondGeneration(t *testing.T) { |
| 400 | dir := t.TempDir() |
| 401 | good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| 402 | a, b, evs := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good}) |
| 403 | a.genHealth = func(_ context.Context, g Gen) error { return errors.New("GET / answered 500") } |
| 404 | b.killErr = ErrStillRunning |
| 405 | if _, err := a.beginRequest(context.Background(), testRoute("one.example.com")); err == nil { |
| 406 | t.Fatal("served while B's end was unproven") |
| 407 | } |
| 408 | if gens := b.started(); len(gens) != 1 { |
| 409 | t.Fatalf("spawned %+v: two generations against one database while B may still run", gens) |
| 410 | } |
| 411 | evs.wait(t, GenFailed) |
| 412 | } |
| 413 | |
| 414 | func TestAnExitInsideTheFirstTenSecondsFailsTheBuild(t *testing.T) { |
| 415 | shrinkFirstWake(t, 55*time.Second, 25*time.Second, 400*time.Millisecond) |
| 416 | dir := t.TempDir() |
| 417 | good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| 418 | a, b, evs := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good}) |
| 419 | done, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| 420 | if err != nil { |
| 421 | t.Fatal(err) |
| 422 | } |
| 423 | done() |
| 424 | evs.wait(t, GenHealthy) |
| 425 | if _, ok := a.Current("one.example.com"); ok { |
| 426 | t.Fatal("Current must hide a first wake under observation, so nothing swaps it meanwhile") |
| 427 | } |
| 428 | b.procOf(b.started()[0].N).crash() |
| 429 | if ev := evs.wait(t, GenFailed); ev.Failed.Version != "v2" { |
| 430 | t.Fatalf("failure event = %+v", ev) |
| 431 | } |
| 432 | } |
| 433 | |
| 434 | func TestAnIntentionalStopInsideTheWindowRecordsNothing(t *testing.T) { |
| 435 | shrinkFirstWake(t, 55*time.Second, 25*time.Second, 400*time.Millisecond) |
| 436 | dir := t.TempDir() |
| 437 | good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| 438 | a, _, evs := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good}) |
| 439 | done, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| 440 | if err != nil { |
| 441 | t.Fatal(err) |
| 442 | } |
| 443 | done() |
| 444 | evs.wait(t, GenHealthy) |
| 445 | a.BounceFor("one.example.com", CauseConfig) |
| 446 | time.Sleep(800 * time.Millisecond) |
| 447 | // Spec §6.4: nothing about the build is recorded. Spec §9.1: the |
| 448 | // observation is still closed, by GenCancelled, or the scheduler shows |
| 449 | // the host settling for ever and the deploy watch never completes. |
| 450 | if k := evs.kinds(); !slices.Equal(k, []GenEventKind{GenHealthy, GenCancelled}) { |
| 451 | t.Fatalf("events %v: want healthy, then cancelled, and nothing recorded against the build", k) |
| 452 | } |
| 453 | } |
| 454 | |
| 455 | func TestAFirstWakeThatSurvivesSettles(t *testing.T) { |
| 456 | shrinkFirstWake(t, 55*time.Second, 25*time.Second, 200*time.Millisecond) |
| 457 | dir := t.TempDir() |
| 458 | good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| 459 | a, _, evs := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good}) |
| 460 | done, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| 461 | if err != nil { |
| 462 | t.Fatal(err) |
| 463 | } |
| 464 | done() |
| 465 | if ev := evs.wait(t, GenSettled); ev.Gen.Build != "v2" { |
| 466 | t.Fatalf("settled event = %+v", ev) |
| 467 | } |
| 468 | g, ok := a.Current("one.example.com") |
| 469 | if !ok || g.Build != "v2" { |
| 470 | t.Fatalf("Current = %+v %v after the settle", g, ok) |
| 471 | } |
| 472 | // PF10: the planned generation carries its staged identity, so the |
| 473 | // scheduler can record it as good (it needs the digest) and the legacy |
| 474 | // sidecar binds to it. |
| 475 | if g.StagedPath != next.StagedPath || g.Digest != next.Digest || g.Demand != next.Demand || g.N == 0 { |
| 476 | t.Fatalf("Current = %+v, want generation of %+v with its staged identity", g, next) |
| 477 | } |
| 478 | } |
| 479 | |
| 480 | func TestFirstWakeWaitersGetTheLongerBudget(t *testing.T) { |
| 481 | old := wakeWaitBound |
| 482 | wakeWaitBound = 100 * time.Millisecond |
| 483 | t.Cleanup(func() { wakeWaitBound = old }) |
| 484 | shrinkFirstWake(t, 600*time.Millisecond, 200*time.Millisecond, time.Millisecond) |
| 485 | dir := t.TempDir() |
| 486 | good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| 487 | |
| 488 | first, b, _ := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good}) |
| 489 | b.gate = make(chan struct{}) |
| 490 | time.AfterFunc(250*time.Millisecond, func() { close(b.gate) }) |
| 491 | done, err := first.beginRequest(context.Background(), testRoute("one.example.com")) |
| 492 | if err != nil { |
| 493 | t.Fatalf("a first wake's waiter gave up at the ordinary bound: %v", err) |
| 494 | } |
| 495 | done() |
| 496 | |
| 497 | plain, b2, _ := planActivator(t, WakePlan{Boot: good, Mode: ModeOverlap}) |
| 498 | b2.gate = make(chan struct{}) |
| 499 | time.AfterFunc(250*time.Millisecond, func() { close(b2.gate) }) |
| 500 | if _, err := plain.beginRequest(context.Background(), testRoute("two.example.com")); !errors.Is(err, errWakeTimeout) { |
| 501 | t.Fatalf("an ordinary wake's waiter = %v, want errWakeTimeout at wakeWaitBound", err) |
| 502 | } |
| 503 | } |
| 504 | |
| 505 | func TestARefusedPlanTakesNoLease(t *testing.T) { |
| 506 | a, b, _ := planActivator(t, WakePlan{Refuse: "held down after build v2 failed in hold mode"}) |
| 507 | _, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| 508 | if err == nil || !strings.Contains(err.Error(), "refusing to wake") { |
| 509 | t.Fatalf("err = %v", err) |
| 510 | } |
| 511 | if acquires, _, _ := b.counts(); acquires != 0 { |
| 512 | t.Fatal("a refused wake took the lease") |
| 513 | } |
| 514 | } |
| 515 | |
| 516 | // Claims on (plan 3 Task 10 runs the fallback inside startAdmitted): a |
| 517 | // claimed first wake that fell back reports as one, a failed one names the |
| 518 | // build that failed, and one where nothing of the build ran reports nothing. |
| 519 | func TestAClaimedFirstWakeReportsLikeAnUnclaimedOne(t *testing.T) { |
| 520 | good := release.StagedIdentity{Version: "v1", StagedPath: "/opt/v1", Digest: "d1"} |
| 521 | next := release.StagedIdentity{Version: "v2", StagedPath: "/opt/v2", Digest: "d2"} |
| 522 | a := &Service{} |
| 523 | e := &actEntry{host: "one.example.com", plan: WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good}, |
| 524 | pinFailed: &Gen{Build: "v2"}, pinFailReason: "GET / answered 500"} |
| 525 | w := a.claimedPlanResult(e, &wokenInstance{spawnVersion: "v1", gen: Gen{N: 9}}) |
| 526 | if !w.fellBack || w.failed == nil || w.failed.Version != "v2" || w.planned.Build != "v1" || w.planned.N != 9 { |
| 527 | t.Fatalf("fell back: %+v", w) |
| 528 | } |
| 529 | if w := a.claimedPlanResult(e, nil); w == nil || w.failed == nil || w.failed.Version != "v2" { |
| 530 | t.Fatalf("failed: %+v", w) |
| 531 | } |
| 532 | e.pinFailed = nil |
| 533 | if w := a.claimedPlanResult(e, nil); w != nil { |
| 534 | t.Fatalf("nothing of the build ran, yet %+v", w) |
| 535 | } |
| 536 | } |
| 537 | |
| 538 | // shrinkClaimStartBackoff sets startAdmitted's backoff base for one test. |
| 539 | func shrinkClaimStartBackoff(t *testing.T, d time.Duration) { |
| 540 | t.Helper() |
| 541 | old := claimStartBackoff |
| 542 | claimStartBackoff = d |
| 543 | t.Cleanup(func() { claimStartBackoff = old }) |
| 544 | } |
| 545 | |
| 546 | // PF17: every claims-on launch refuses on this base (RoleLaunch.Start |
| 547 | // answers ErrClaimIsolationUnavailable before its exec hook). A planned |
| 548 | // claimed wake carries the plan into the start policy (the pin, the good |
| 549 | // build, the health cap, the ownership gate and the selection gate), meets |
| 550 | // the closed launch, and records nothing: no failure event, no fallback. |
| 551 | // The refused start is retried on startAdmitted's backoff, whose schedule |
| 552 | // TestStartAdmittedLoopStopsAfterFiveStartsWithDoublingBackoff pins; here |
| 553 | // it is shrunk so the retries do not take 15 s. |
| 554 | func TestAPlannedClaimedWakeRefusesAtTheLaunchRecordingNothing(t *testing.T) { |
| 555 | shrinkClaimStartBackoff(t, 10*time.Millisecond) |
| 556 | dir := t.TempDir() |
| 557 | good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| 558 | const host = "x.example" |
| 559 | obs := claimedObs(host) |
| 560 | obs.Claim.Op.Exec = "box" |
| 561 | obs.Claim.Holder.WriterBox = "box" |
| 562 | admitted := obs |
| 563 | claim := *obs.Claim |
| 564 | holder := *claim.Holder |
| 565 | holder.Session, holder.Generation = 2, "two" |
| 566 | claim.Holder = &holder |
| 567 | admitted.Claim = &claim |
| 568 | work, _ := (hostwrite.Config{}).Select(context.Background(), host, "spawn") |
| 569 | var ( |
| 570 | pmu sync.Mutex |
| 571 | policies []startPolicy |
| 572 | reads atomic.Int32 |
| 573 | ) |
| 574 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 575 | if err != nil { |
| 576 | t.Fatal(err) |
| 577 | } |
| 578 | a := New(Config{Bucket: "test-bucket", Idle: time.Hour, HostWrites: hostwrite.Config{Enabled: true}, Claims: &ClaimRuntime{Box: "box", |
| 579 | Read: func(context.Context, string) (hostclaim.Observed, error) { |
| 580 | if reads.Add(1) == 1 { |
| 581 | return obs, nil |
| 582 | } |
| 583 | return admitted, nil |
| 584 | }, |
| 585 | StopForeign: func(context.Context, string) error { return nil }, |
| 586 | FailedAdmission: func(context.Context, hostclaim.Observed, error) error { return nil }, |
| 587 | Prerequisites: func(context.Context, registry.Route, hostclaim.Observed, startPolicy) error { return nil }, |
| 588 | Admit: func(context.Context, registry.Route, hostclaim.Observed, startPolicy) (hostclaim.Observed, error) { |
| 589 | return admitted, nil |
| 590 | }, |
| 591 | Prepare: func(_ context.Context, _ registry.Route, _ hostclaim.Observed, p startPolicy) (ClaimStart, error) { |
| 592 | pmu.Lock() |
| 593 | policies = append(policies, p) |
| 594 | pmu.Unlock() |
| 595 | return claimedStartFor(admitted, work, p.Pin.Build), nil |
| 596 | }}}, &fakeBackend{}, reg) |
| 597 | evs := &genLog{} |
| 598 | a.OnGen = evs.add |
| 599 | a.Planner = func(context.Context, registry.Route) WakePlan { |
| 600 | return WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good, Check: func(string) error { return nil }} |
| 601 | } |
| 602 | _, err = a.beginRequest(context.Background(), registry.Route{Host: host, Kind: "instance", Hibernate: true}) |
| 603 | if !errors.Is(err, ErrClaimIsolationUnavailable) { |
| 604 | t.Fatalf("err = %v, want the closed launch", err) |
| 605 | } |
| 606 | pmu.Lock() |
| 607 | defer pmu.Unlock() |
| 608 | if len(policies) != 1 { |
| 609 | t.Fatalf("prepared %d times, want once", len(policies)) |
| 610 | } |
| 611 | p := policies[0] |
| 612 | if p.Pin == nil || p.Pin.Build != "v2" || p.Pin.StagedPath != next.StagedPath || p.Pin.Digest != next.Digest { |
| 613 | t.Fatalf("policy pin = %+v, want the planned build", p.Pin) |
| 614 | } |
| 615 | if p.Fallback == nil || p.Fallback.Build != "v1" || p.HealthBy.IsZero() || p.Gate == nil || p.PinCheck == nil { |
| 616 | t.Fatalf("policy = %+v: want the good build, the health cap, the ownership and selection gates", p) |
| 617 | } |
| 618 | time.Sleep(50 * time.Millisecond) |
| 619 | if k := evs.kinds(); len(k) != 0 { |
| 620 | t.Fatalf("events %v: a launch refused before its exec recorded something", k) |
| 621 | } |
| 622 | } |
| 623 | |
| 624 | // execStagedBackend runs planned wakes through the REAL exec backend's |
| 625 | // StartStaged (real processes, plan 3's real flat and foreign spawn paths) |
| 626 | // over fakeBackend's lease and restore. Its KillGen stops the generation's |
| 627 | // process, because a test fixture is not cgroup v2. |
| 628 | type execStagedBackend struct { |
| 629 | *fakeBackend |
| 630 | eb *ExecBackend |
| 631 | smu sync.Mutex |
| 632 | procs map[uint64]Proc |
| 633 | } |
| 634 | |
| 635 | func (b *execStagedBackend) StartStaged(ctx context.Context, rt registry.Route, g Gen) (Gen, Proc, error) { |
| 636 | got, p, err := b.eb.StartStaged(ctx, rt, g) |
| 637 | if p != nil { |
| 638 | b.smu.Lock() |
| 639 | b.procs[g.N] = p |
| 640 | b.smu.Unlock() |
| 641 | } |
| 642 | return got, p, err |
| 643 | } |
| 644 | func (b *execStagedBackend) GenTier(rt registry.Route, build string) cgroup.Limit { |
| 645 | return b.eb.GenTier(rt, build) |
| 646 | } |
| 647 | func (b *execStagedBackend) GenReady(rt registry.Route, g Gen, p Proc) { b.eb.GenReady(rt, g, p) } |
| 648 | func (b *execStagedBackend) RaiseGenParent(ctx context.Context, rt registry.Route, inc, succ cgroup.Limit) error { |
| 649 | return b.eb.RaiseGenParent(ctx, rt, inc, succ) |
| 650 | } |
| 651 | func (b *execStagedBackend) RestoreGenParent(ctx context.Context, rt registry.Route, tier cgroup.Limit) error { |
| 652 | return b.eb.RestoreGenParent(ctx, rt, tier) |
| 653 | } |
| 654 | func (b *execStagedBackend) GenSwitched(rt registry.Route, g Gen, p Proc) error { |
| 655 | return b.eb.GenSwitched(rt, g, p) |
| 656 | } |
| 657 | func (b *execStagedBackend) Layout(rt registry.Route) (Layout, string) { return b.eb.Layout(rt) } |
| 658 | func (b *execStagedBackend) KillGen(ctx context.Context, _ registry.Route, n uint64) error { |
| 659 | b.smu.Lock() |
| 660 | p := b.procs[n] |
| 661 | b.smu.Unlock() |
| 662 | if p == nil { |
| 663 | return nil |
| 664 | } |
| 665 | if err := p.Stop(ctx); err != nil { |
| 666 | return err |
| 667 | } |
| 668 | if !p.Exited() { |
| 669 | return ErrStillRunning |
| 670 | } |
| 671 | return nil |
| 672 | } |
| 673 | |
| 674 | // stagedScript writes an executable staged build and returns its identity. |
| 675 | func stagedScript(t *testing.T, dir, ver, body string) release.StagedIdentity { |
| 676 | t.Helper() |
| 677 | path := filepath.Join(dir, ver) |
| 678 | if err := os.WriteFile(path, []byte(body), 0o755); err != nil { |
| 679 | t.Fatal(err) |
| 680 | } |
| 681 | sum := sha256.Sum256([]byte(body)) |
| 682 | return release.StagedIdentity{Version: ver, StagedPath: path, Digest: hex.EncodeToString(sum[:])} |
| 683 | } |
| 684 | |
| 685 | // Spec §8.2 on the real exec backend: B never answers, so its failure must |
| 686 | // be seen inside the wake's own budget, recorded, and the good build booted |
| 687 | // in its place. StartStaged hands back the running process at once (plan 3 |
| 688 | // Task 1), so startPinnedAt's health deadline (55 s − 25 s, shrunk here) is |
| 689 | // the only one, and the kill reaches a real process. A StartStaged that |
| 690 | // waited out the backend's own 30 s spawnHealthTimeout and returned a nil |
| 691 | // handle made spawnPlanned read "nothing executed": no failure recorded, no |
| 692 | // fallback, and the waiter timed out. The foreign row runs the same way on |
| 693 | // its canonical socket (plan 1's reading): refusing it failed every planned |
| 694 | // wake of a foreign row. |
| 695 | func TestAPlannedFirstWakeOnTheRealExecBackendFallsBackInTime(t *testing.T) { |
| 696 | for _, layout := range []string{"flat", "foreign"} { |
| 697 | t.Run(layout, func(t *testing.T) { |
| 698 | oldBound := wakeWaitBound |
| 699 | wakeWaitBound = time.Second |
| 700 | t.Cleanup(func() { wakeWaitBound = oldBound }) |
| 701 | shrinkFirstWake(t, 4*time.Second, 2*time.Second, time.Millisecond) // B's health window: 2 s |
| 702 | // The floor shrinks with the window, or B (2 s) would never be |
| 703 | // exec'd under the spec's 20 s floor. |
| 704 | shrinkVerdictFloor(t, time.Second) |
| 705 | eb := testExecBackend(t) |
| 706 | eb.RunDirOverride = t.TempDir() |
| 707 | host := "real.example.com" |
| 708 | stageLive(t, eb, host, "v0") |
| 709 | rt := registry.Route{Host: host, Kind: "instance", Hibernate: true, DBPath: filepath.Join(eb.DataDir, host+".db"), |
| 710 | Socket: filepath.Join(eb.RunDirOverride, host+".sock")} |
| 711 | want := LayoutFlat |
| 712 | if layout == "foreign" { |
| 713 | rt.Socket, want = filepath.Join(t.TempDir(), "hand-set.sock"), LayoutForeign |
| 714 | } |
| 715 | if got, _ := eb.Layout(rt); got != want { |
| 716 | t.Fatalf("fixture layout %v, want %v", got, want) |
| 717 | } |
| 718 | dir := t.TempDir() |
| 719 | next := stagedScript(t, dir, "v2", "#!/bin/sh\nexec sleep 30\n") // never binds |
| 720 | good := stagedScript(t, dir, "v1", "#!/bin/sh\nexec sleep 30\n") |
| 721 | b := &execStagedBackend{fakeBackend: &fakeBackend{}, eb: eb, procs: map[uint64]Proc{}} |
| 722 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 723 | if err != nil { |
| 724 | t.Fatal(err) |
| 725 | } |
| 726 | a := New(Config{Bucket: "test-bucket", Idle: time.Hour}, b, reg) |
| 727 | a.gens = true |
| 728 | evs := &genLog{} |
| 729 | a.OnGen = evs.add |
| 730 | a.Planner = func(context.Context, registry.Route) WakePlan { |
| 731 | return WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good} |
| 732 | } |
| 733 | // B gets the real check: GET / on its real socket, which nothing |
| 734 | // binds, bounded only by the caller's deadline. The good build is |
| 735 | // let through, because a shell script cannot answer HTTP. |
| 736 | a.genHealth = func(ctx context.Context, g Gen) error { |
| 737 | if g.Build == "v1" { |
| 738 | return nil |
| 739 | } |
| 740 | return waitHealthyCtx(ctx, g.Socket, spawnHealthTimeout) |
| 741 | } |
| 742 | t.Cleanup(func() { a.Shutdown(context.Background()) }) |
| 743 | start := time.Now() |
| 744 | done, err := a.beginRequest(context.Background(), rt) |
| 745 | if err != nil { |
| 746 | t.Fatalf("the waiter was not served by the good build: %v", err) |
| 747 | } |
| 748 | done() |
| 749 | if took := time.Since(start); took > 4*time.Second { |
| 750 | t.Fatalf("served after %v, past the first-wake budget", took) |
| 751 | } |
| 752 | ev := evs.wait(t, GenFellBack) |
| 753 | if ev.Failed.Version != "v2" || ev.Gen.Build != "v1" { |
| 754 | t.Fatalf("fallback event = %+v", ev) |
| 755 | } |
| 756 | b.smu.Lock() |
| 757 | procs := maps.Clone(b.procs) |
| 758 | b.smu.Unlock() |
| 759 | if len(procs) != 2 { |
| 760 | t.Fatalf("%d generations spawned, want B and the good build", len(procs)) |
| 761 | } |
| 762 | for n, p := range procs { |
| 763 | if n != ev.Gen.N && !p.Exited() { |
| 764 | t.Fatalf("B's process (generation %d) outlived its kill", n) |
| 765 | } |
| 766 | } |
| 767 | }) |
| 768 | } |
| 769 | } |
| 770 | |
| 771 | // PF10 (spec §12 "Legacy sidecar binding"): the co-spawned legacy sidecar |
| 772 | // of a planned wake runs the build the wake left serving. After a first |
| 773 | // wake fell back, that is the good build, by its staged path, never B's |
| 774 | // and never the live link. |
| 775 | func TestAFallenBackFirstWakeBindsItsSidecarToTheGoodBuild(t *testing.T) { |
| 776 | t.Parallel() |
| 777 | dir := t.TempDir() |
| 778 | good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| 779 | a, b, evs := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good}) |
| 780 | b.sidecarStaged = true |
| 781 | a.genHealth = func(_ context.Context, g Gen) error { |
| 782 | if g.Build == "v2" { |
| 783 | return errors.New("GET / answered 500") |
| 784 | } |
| 785 | return nil |
| 786 | } |
| 787 | done, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| 788 | if err != nil { |
| 789 | t.Fatal(err) |
| 790 | } |
| 791 | done() |
| 792 | ev := evs.wait(t, GenFellBack) |
| 793 | sc := b.sidecarsOn() |
| 794 | if len(sc) != 1 || sc[0].StagedPath != good.StagedPath || sc[0].Digest != good.Digest || sc[0].N != ev.Gen.N { |
| 795 | t.Fatalf("sidecar started on %+v, want the good build's generation %d at %s", sc, ev.Gen.N, good.StagedPath) |
| 796 | } |
| 797 | if n := b.sidecarStartCount(); n != 0 { |
| 798 | t.Fatalf("%d live-link sidecar starts, want none", n) |
| 799 | } |
| 800 | if ev.Gen.StagedPath != good.StagedPath || ev.Gen.Digest != good.Digest || ev.Gen.Demand != good.Demand { |
| 801 | t.Fatalf("fallback event's generation %+v lacks the good build's staged identity", ev.Gen) |
| 802 | } |
| 803 | } |
| 804 | |
| 805 | // Spec §6.5: a socketless planned wake is ready once it is still running |
| 806 | // 10 s after it started, and a clean exit inside that window is a |
| 807 | // one-shot's success: the same rule ReplaceWorker applies. The flag-off |
| 808 | // awaitWorkerSettle fails every early exit, exit 0 included, so a one-shot's |
| 809 | // first wake was recorded as a failed build (and could halt its cohort). |
| 810 | func TestAPlannedOneShotWorkerThatExitsCleanlyIsReady(t *testing.T) { |
| 811 | shrinkFirstWake(t, 55*time.Second, 25*time.Second, 200*time.Millisecond) |
| 812 | a, b, _, _, _ := stagedActivator(t) |
| 813 | a.gens = true |
| 814 | evs := &genLog{} |
| 815 | a.OnGen = evs.add |
| 816 | dir := t.TempDir() |
| 817 | good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| 818 | a.Planner = func(context.Context, registry.Route) WakePlan { |
| 819 | return WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good} |
| 820 | } |
| 821 | b.workerDeadOnStart = true // the child exits at once, cleanly |
| 822 | if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil { |
| 823 | t.Fatalf("WakeWorker: %v (a one-shot's clean exit is its success)", err) |
| 824 | } |
| 825 | ev := evs.wait(t, GenSettled) |
| 826 | if k := evs.kinds(); slices.Contains(k, GenFailed) || slices.Contains(k, GenFellBack) { |
| 827 | t.Fatalf("events %v: a clean exit was recorded against the build", k) |
| 828 | } |
| 829 | if calls := b.calls(); len(calls) != 1 { |
| 830 | t.Fatalf("spawns %v: a clean exit fell back to the good build", calls) |
| 831 | } |
| 832 | // A worker's generation is compared by its staged artifact (the |
| 833 | // scheduler's acknowledged), so the event names it, and it is a worker's. |
| 834 | if !ev.Gen.Worker || ev.Gen.StagedPath != next.StagedPath || ev.Gen.Digest != next.Digest { |
| 835 | t.Fatalf("settled event's generation = %+v, want the worker's v2 artifact", ev.Gen) |
| 836 | } |
| 837 | } |
| 838 | |
| 839 | // plannedHostKill is plan 4's stagedFake plus plan 3's KillHost, failing |
| 840 | // while bad: a worker host cgroup that still holds a process. (The name |
| 841 | // stagedHostKill is worker_retained_pin_test.go's.) |
| 842 | type plannedHostKill struct { |
| 843 | *stagedFake |
| 844 | bad atomic.Bool |
| 845 | } |
| 846 | |
| 847 | func (s *plannedHostKill) KillHost(context.Context, registry.Route) error { |
| 848 | if s.bad.Load() { |
| 849 | return errors.New("the host cgroup still holds a process") |
| 850 | } |
| 851 | return nil |
| 852 | } |
| 853 | |
| 854 | // Spec §7.4, §7.9: a failed socketless first wake falls back to the good |
| 855 | // build only once the failed worker's host cgroup is proven empty (plan 3's |
| 856 | // genTerminated on a Worker generation). Its leader's exit alone let the |
| 857 | // good build start beside a descendant of the failed one. |
| 858 | func TestAFailedWorkerFirstWakeFallsBackOnlyOnceItsCgroupIsEmpty(t *testing.T) { |
| 859 | shrinkFirstWake(t, 55*time.Second, 25*time.Second, 300*time.Millisecond) |
| 860 | a, b, _, _, _ := stagedActivator(t) |
| 861 | a.gens = true |
| 862 | hk := &plannedHostKill{stagedFake: b} |
| 863 | hk.bad.Store(true) |
| 864 | a.backend = hk |
| 865 | a.OnGen = (&genLog{}).add |
| 866 | dir := t.TempDir() |
| 867 | good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| 868 | a.Planner = func(context.Context, registry.Route) WakePlan { |
| 869 | return WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good} |
| 870 | } |
| 871 | go func() { |
| 872 | waitFor(t, "the v2 child", func() bool { return len(b.calls()) == 1 }) |
| 873 | time.Sleep(50 * time.Millisecond) |
| 874 | b.lastWorker().crash() |
| 875 | }() |
| 876 | _ = a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")) |
| 877 | if calls := b.calls(); len(calls) != 1 { |
| 878 | t.Fatalf("spawns %v: the good build started beside the failed worker's unproven host cgroup", calls) |
| 879 | } |
| 880 | } |
| 881 | |
| 882 | // The same, with the host cgroup proven empty: the good build replaces the |
| 883 | // failed worker, through startWorker's gate, and the wake reports the |
| 884 | // fallback with the good build's artifact. |
| 885 | func TestAFailedWorkerFirstWakeFallsBackToTheGoodBuild(t *testing.T) { |
| 886 | shrinkFirstWake(t, 55*time.Second, 25*time.Second, 300*time.Millisecond) |
| 887 | a, b, _, _, _ := stagedActivator(t) |
| 888 | a.gens = true |
| 889 | hk := &plannedHostKill{stagedFake: b} |
| 890 | a.backend = hk |
| 891 | evs := &genLog{} |
| 892 | a.OnGen = evs.add |
| 893 | dir := t.TempDir() |
| 894 | good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| 895 | a.Planner = func(context.Context, registry.Route) WakePlan { |
| 896 | return WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good} |
| 897 | } |
| 898 | go func() { |
| 899 | waitFor(t, "the v2 child", func() bool { return len(b.calls()) == 1 }) |
| 900 | time.Sleep(50 * time.Millisecond) |
| 901 | b.lastWorker().crash() |
| 902 | }() |
| 903 | if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil { |
| 904 | t.Fatalf("WakeWorker: %v, want the good build serving", err) |
| 905 | } |
| 906 | calls := b.calls() |
| 907 | if len(calls) != 2 || calls[0] == nil || calls[0].Version != "v2" || calls[1] == nil || calls[1].Version != "v1" { |
| 908 | t.Fatalf("spawns %v, want v2 then the good v1, each by name", calls) |
| 909 | } |
| 910 | ev := evs.wait(t, GenFellBack) |
| 911 | if ev.Failed.Version != "v2" || ev.Gen.StagedPath != good.StagedPath || !ev.Gen.Worker { |
| 912 | t.Fatalf("fallback event = %+v", ev) |
| 913 | } |
| 914 | if got := a.entryServing("bot.x.internal"); got.StagedPath != good.StagedPath { |
| 915 | t.Fatalf("serving %+v, want the good build", got) |
| 916 | } |
| 917 | } |
| 918 | |
| 919 | // breakOnAcquire is a staged backend that makes the generations file |
| 920 | // unwritable as the wake takes its lease: after the wake's own publication |
| 921 | // (doWake's step 0.8) and before startWorker's before-exec publication. |
| 922 | type breakOnAcquire struct { |
| 923 | *stagedFake |
| 924 | t *testing.T |
| 925 | path string |
| 926 | } |
| 927 | |
| 928 | func (b *breakOnAcquire) Acquire(ctx context.Context, rt registry.Route) (Lease, error) { |
| 929 | breakGenerationsFile(b.t, b.path) |
| 930 | return b.stagedFake.Acquire(ctx, rt) |
| 931 | } |
| 932 | |
| 933 | // PF9: a planned worker spawn refused because its build could not be |
| 934 | // published first (ErrBuildNotPublished) ran nothing of the build. It is |
| 935 | // abandonment: no failure recorded, nothing falls back. The wake's own |
| 936 | // publication (step 0.8) succeeds here; the file breaks afterwards, so the |
| 937 | // refusal is startWorker's before-exec publication, with the spawn attempt |
| 938 | // reaching the backend and refused inside it. |
| 939 | func TestAPlannedWorkerRefusedForPublicationRecordsNothing(t *testing.T) { |
| 940 | shrinkFirstWake(t, 55*time.Second, 25*time.Second, 200*time.Millisecond) |
| 941 | a, b, _, _, _ := stagedActivator(t) |
| 942 | a.gens = true |
| 943 | a.GenerationsFile = filepath.Join(t.TempDir(), "zdd", "generations.json") |
| 944 | a.backend = &breakOnAcquire{stagedFake: b, t: t, path: a.GenerationsFile} |
| 945 | evs := &genLog{} |
| 946 | a.OnGen = evs.add |
| 947 | dir := t.TempDir() |
| 948 | good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| 949 | a.Planner = func(context.Context, registry.Route) WakePlan { |
| 950 | return WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good} |
| 951 | } |
| 952 | err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")) |
| 953 | if !errors.Is(err, ErrBuildNotPublished) { |
| 954 | t.Fatalf("WakeWorker = %v, want ErrBuildNotPublished", err) |
| 955 | } |
| 956 | if n := b.workerStarts(); n != 0 { |
| 957 | t.Fatalf("%d worker children started for an unpublished build", n) |
| 958 | } |
| 959 | if calls := b.calls(); len(calls) != 1 { |
| 960 | t.Fatalf("spawn attempts %v: a refusal before the exec fell back", calls) |
| 961 | } |
| 962 | time.Sleep(50 * time.Millisecond) |
| 963 | if k := evs.kinds(); len(k) != 0 { |
| 964 | t.Fatalf("events %v: a refusal before the exec was recorded", k) |
| 965 | } |
| 966 | } |
| 967 | |
| 968 | // The wake's own publication (Task 12, doWake step 0.8) refuses a planned |
| 969 | // worker before the lease when the file cannot be written at all: nothing is |
| 970 | // taken, no spawn is attempted, let alone a fallback, and nothing is |
| 971 | // recorded. |
| 972 | func TestAPlannedWorkerRefusedAtItsPublicationAttemptsNoSpawn(t *testing.T) { |
| 973 | shrinkFirstWake(t, 55*time.Second, 25*time.Second, 200*time.Millisecond) |
| 974 | a, b, _, _, _ := stagedActivator(t) |
| 975 | a.gens = true |
| 976 | blocker := filepath.Join(t.TempDir(), "not-a-dir") |
| 977 | if err := os.WriteFile(blocker, nil, 0o644); err != nil { |
| 978 | t.Fatal(err) |
| 979 | } |
| 980 | a.GenerationsFile = filepath.Join(blocker, "generations.json") // unwritable |
| 981 | evs := &genLog{} |
| 982 | a.OnGen = evs.add |
| 983 | dir := t.TempDir() |
| 984 | good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| 985 | a.Planner = func(context.Context, registry.Route) WakePlan { |
| 986 | return WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good} |
| 987 | } |
| 988 | // Step 0.8 is plan 5's staged publication, flag on only, so it carries |
| 989 | // that path's sentinel (final review F9); the worker's own exec-time |
| 990 | // publication keeps ErrBuildNotPublished (the test above). |
| 991 | err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")) |
| 992 | if !errors.Is(err, errStagedBuildNotPublished) { |
| 993 | t.Fatalf("WakeWorker = %v, want errStagedBuildNotPublished", err) |
| 994 | } |
| 995 | if n := b.workerStarts(); n != 0 { |
| 996 | t.Fatalf("%d worker children started for an unpublished build", n) |
| 997 | } |
| 998 | // The plan is published before the lease (Task 12), so the refusal |
| 999 | // comes before any spawn is attempted, let alone a fallback. |
| 1000 | if calls := b.calls(); len(calls) != 0 { |
| 1001 | t.Fatalf("spawn attempts %v: an unpublished plan reached a spawn", calls) |
| 1002 | } |
| 1003 | time.Sleep(50 * time.Millisecond) |
| 1004 | if k := evs.kinds(); len(k) != 0 { |
| 1005 | t.Fatalf("events %v: a refusal before the exec was recorded", k) |
| 1006 | } |
| 1007 | } |
| 1008 | |
| 1009 | // PF9: the one publication sentinel is abandonment wherever it surfaces. |
| 1010 | func TestErrBuildNotPublishedIsAbandonment(t *testing.T) { |
| 1011 | err := fmt.Errorf("start bot.x.internal: %w", fmt.Errorf("%w: disk full", ErrBuildNotPublished)) |
| 1012 | if got := classifyStep(nil, err); got != stepStale { |
| 1013 | t.Fatalf("classifyStep = %v, want stepStale", got) |
| 1014 | } |
| 1015 | if got := verdictOf(err); got != gateProceed { |
| 1016 | t.Fatalf("verdictOf = %v: main's flag-off respawn reads verdictOf, which must stay as it was", got) |
| 1017 | } |
| 1018 | } |
| 1019 | |
| 1020 | // Final review F9: the worker path keeps main's sentinel text (origin/main |
| 1021 | // a3bf7d6c). Zero-downtime plan 6 (Task 14b) gives a worker wake whose |
| 1022 | // publication fails a member-safe event of its own, whatever the flag: the |
| 1023 | // draft below (zdd.wake.not-published), never the wrapped detail. The staged |
| 1024 | // instance path has a sentinel of its own: abandonment too, and its member |
| 1025 | // event reads as the maintenance fence's existing wording, never the raw text. |
| 1026 | func TestTheWorkerPublicationSentinelKeepsMainsText(t *testing.T) { |
| 1027 | if got, want := ErrBuildNotPublished.Error(), "worker build not published before its exec"; got != want { |
| 1028 | t.Fatalf("ErrBuildNotPublished = %q, want main's %q", got, want) |
| 1029 | } |
| 1030 | worker := fmt.Errorf("start bot.x.internal: %w", fmt.Errorf("%w: disk full", ErrBuildNotPublished)) |
| 1031 | const draft = "wake failed: the box could not record which build this instance runs, so it did not start it. the next request tries again" |
| 1032 | if got := memberSafeWakeReason(worker); got != draft { |
| 1033 | t.Fatalf("worker member text = %q, want the draft %q", got, draft) |
| 1034 | } |
| 1035 | if got := memberSafeWakeReason(fmt.Errorf("%w: disk full", ErrBuildNotPublished)); got != draft || strings.Contains(got, "disk full") { |
| 1036 | t.Fatalf("wrapped member text = %q, want the draft alone", got) |
| 1037 | } |
| 1038 | inst := fmt.Errorf("one.example.com: refusing to wake: %w", errStagedBuildNotPublished) |
| 1039 | if errors.Is(inst, ErrBuildNotPublished) || errors.Is(worker, errStagedBuildNotPublished) { |
| 1040 | t.Fatal("the worker and instance sentinels must be distinct") |
| 1041 | } |
| 1042 | if got := classifyStep(nil, inst); got != stepStale { |
| 1043 | t.Fatalf("classifyStep(instance) = %v, want stepStale", got) |
| 1044 | } |
| 1045 | if got := verdictOf(inst); got != gateProceed { |
| 1046 | t.Fatalf("verdictOf(instance) = %v, want unchanged (gateProceed)", got) |
| 1047 | } |
| 1048 | if got, want := memberSafeWakeReason(inst), memberSafeWakeReason(ErrFenced); got != want { |
| 1049 | t.Fatalf("instance member text = %q, want the fence's reviewed wording %q", got, want) |
| 1050 | } |
| 1051 | } |
| 1052 | |
| 1053 | // Spec §6.4, §7.4: a wake cancelled by its lifecycle (here Shutdown, as |
| 1054 | // maintenance or a stop would) is no failure of the build: no failure |
| 1055 | // event, no fallback, so no halted cohort and no hold-down. spawnPlanned |
| 1056 | // that marked every error with a process as a failure recorded one. |
| 1057 | func TestACancelledFirstWakeRecordsNothing(t *testing.T) { |
| 1058 | t.Parallel() |
| 1059 | dir := t.TempDir() |
| 1060 | good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| 1061 | a, b, evs := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good}) |
| 1062 | entered := make(chan struct{}) |
| 1063 | var once sync.Once |
| 1064 | a.genHealth = func(ctx context.Context, _ Gen) error { |
| 1065 | once.Do(func() { close(entered) }) |
| 1066 | <-ctx.Done() |
| 1067 | return ctx.Err() |
| 1068 | } |
| 1069 | go a.beginRequest(context.Background(), testRoute("one.example.com")) //nolint:errcheck |
| 1070 | <-entered |
| 1071 | a.Shutdown(context.Background()) |
| 1072 | time.Sleep(100 * time.Millisecond) |
| 1073 | if k := evs.kinds(); slices.Contains(k, GenFailed) || slices.Contains(k, GenFellBack) { |
| 1074 | t.Fatalf("events %v: a cancelled wake was recorded against the build", k) |
| 1075 | } |
| 1076 | if gens := b.started(); len(gens) != 1 { |
| 1077 | t.Fatalf("spawned %+v: a cancelled wake fell back", gens) |
| 1078 | } |
| 1079 | } |
| 1080 | |
| 1081 | // Spec §6.4, §8.1: a first wake's observation is registered in the awake |
| 1082 | // transition, before the generation is reachable. Here the build crashes, |
| 1083 | // and the sweep starts stopping it, in the moment between the wake becoming |
| 1084 | // ready and its observer running. Registered late (in afterWake), Current |
| 1085 | // exposed the build as stable meanwhile, and the observer, finding the |
| 1086 | // entry stopping, reported GenCancelled: a real failure of the build lost. |
| 1087 | // |
| 1088 | // Only the first healthy generation crashes. The fixed planner plans the |
| 1089 | // same first wake every time, so a hook that crashed every generation made |
| 1090 | // a livelock: the parked request got out only if it took a.mu in the gap |
| 1091 | // between ready closing and the hook's crash, which the Go scheduler rarely |
| 1092 | // gives it (thousands of recycles at about 0.4 ms each, past the 30 s wake |
| 1093 | // bound in about one run in ten, on main too). Now the request either wins |
| 1094 | // that gap or rides the recycle to a second wake that stays up. |
| 1095 | func TestAFirstWakeIsObservedFromTheMomentItIsReady(t *testing.T) { |
| 1096 | shrinkFirstWake(t, 55*time.Second, 25*time.Second, 2*time.Second) |
| 1097 | dir := t.TempDir() |
| 1098 | good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| 1099 | a, b, evs := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good}) |
| 1100 | var visible atomic.Bool |
| 1101 | var crashOnce sync.Once |
| 1102 | a.OnGen = func(ev GenEvent) { |
| 1103 | if ev.Kind == GenHealthy { |
| 1104 | if _, ok := a.Current("one.example.com"); ok { |
| 1105 | visible.Store(true) |
| 1106 | } |
| 1107 | crashOnce.Do(func() { |
| 1108 | b.procOf(ev.Gen.N).crash() |
| 1109 | a.sweepOnce() // the sweep sees the exit before the observer runs |
| 1110 | }) |
| 1111 | } |
| 1112 | evs.add(ev) |
| 1113 | } |
| 1114 | done, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| 1115 | if err != nil { |
| 1116 | t.Fatal(err) |
| 1117 | } |
| 1118 | done() |
| 1119 | if ev := evs.wait(t, GenFailed); ev.Failed.Version != "v2" { |
| 1120 | t.Fatalf("failure event = %+v", ev) |
| 1121 | } |
| 1122 | if slices.Contains(evs.kinds(), GenCancelled) { |
| 1123 | t.Fatalf("events %v: a crash was reported as a cancelled observation", evs.kinds()) |
| 1124 | } |
| 1125 | if visible.Load() { |
| 1126 | t.Fatal("Current exposed the first wake as stable before its observation was registered") |
| 1127 | } |
| 1128 | } |
| 1129 | |
| 1130 | // Spec §6.5, §8.6: a socketless first wake is settled once its readiness |
| 1131 | // window is over. A crash after it is ordinary crash recovery, not a failed |
| 1132 | // release. A second, HTTP-style observation after the readiness window |
| 1133 | // recorded a worker that crashed at 10 s plus a moment as a failed build. |
| 1134 | func TestAPlannedWorkerFirstWakeSettlesAtItsReadiness(t *testing.T) { |
| 1135 | shrinkFirstWake(t, 55*time.Second, 25*time.Second, 200*time.Millisecond) |
| 1136 | a, b, _, _, _ := stagedActivator(t) |
| 1137 | a.gens = true |
| 1138 | evs := &genLog{} |
| 1139 | a.OnGen = evs.add |
| 1140 | dir := t.TempDir() |
| 1141 | good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| 1142 | a.Planner = func(context.Context, registry.Route) WakePlan { |
| 1143 | return WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good} |
| 1144 | } |
| 1145 | if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil { |
| 1146 | t.Fatal(err) |
| 1147 | } |
| 1148 | evs.wait(t, GenSettled) |
| 1149 | b.lastWorker().crash() // after its readiness window |
| 1150 | time.Sleep(400 * time.Millisecond) |
| 1151 | if k := evs.kinds(); slices.Contains(k, GenFailed) || slices.Contains(k, GenFellBack) { |
| 1152 | t.Fatalf("events %v: a crash after readiness was recorded against the build", k) |
| 1153 | } |
| 1154 | } |
| 1155 | |
| 1156 | // The flag off: no Planner, today's wake, no events. |
| 1157 | func TestWithoutAPlannerTheWakeIsTodays(t *testing.T) { |
| 1158 | a, b, _, _ := testActivator(t, time.Hour) |
| 1159 | var fired bool |
| 1160 | a.OnGen = func(GenEvent) { fired = true } |
| 1161 | done, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| 1162 | if err != nil { |
| 1163 | t.Fatal(err) |
| 1164 | } |
| 1165 | done() |
| 1166 | if b.startCount() != 1 || fired { |
| 1167 | t.Fatalf("starts %d, events %v: want today's Start and nothing else", b.startCount(), fired) |
| 1168 | } |
| 1169 | } |
| 1170 | |
| 1171 | func TestDriftFeedsTheSchedulerInsteadOfStopping(t *testing.T) { |
| 1172 | t.Parallel() |
| 1173 | a, b, clk, _ := testActivator(t, time.Hour) |
| 1174 | a.liveVer = func(string) (string, error) { return "v2", nil } |
| 1175 | b.execVerFn = func() string { return "v1" } |
| 1176 | var drifted []string |
| 1177 | var mu sync.Mutex |
| 1178 | a.OnDrift = func(h string) { mu.Lock(); drifted = append(drifted, h); mu.Unlock() } |
| 1179 | done, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| 1180 | if err != nil { |
| 1181 | t.Fatal(err) |
| 1182 | } |
| 1183 | done() |
| 1184 | a.sweepOnce() |
| 1185 | clk.advance(ConvergeGrace + time.Second) |
| 1186 | a.sweepOnce() |
| 1187 | time.Sleep(50 * time.Millisecond) |
| 1188 | if b.lastProc().isStopped() { |
| 1189 | t.Fatal("the sweep stopped a drifted instance; with the flag on the scheduler owns the move (spec §10)") |
| 1190 | } |
| 1191 | mu.Lock() |
| 1192 | defer mu.Unlock() |
| 1193 | if len(drifted) == 0 || drifted[0] != "one.example.com" { |
| 1194 | t.Fatalf("OnDrift saw %v", drifted) |
| 1195 | } |
| 1196 | } |
| 1197 | |
| 1198 | type causeSink struct { |
| 1199 | mu sync.Mutex |
| 1200 | msgs []string |
| 1201 | } |
| 1202 | |
| 1203 | func (s *causeSink) App(string, string, string, string) {} |
| 1204 | func (s *causeSink) Event(_, _, _, _, msg string) { |
| 1205 | s.mu.Lock() |
| 1206 | defer s.mu.Unlock() |
| 1207 | s.msgs = append(s.msgs, msg) |
| 1208 | } |
| 1209 | |
| 1210 | // Spec §9.4: the bounce event names the attempt's cause. |
| 1211 | func TestBounceForNamesTheCause(t *testing.T) { |
| 1212 | a := newTestServiceAwake(t, "one.example.com") |
| 1213 | sink := &causeSink{} |
| 1214 | a.Logs = sink |
| 1215 | if !a.BounceFor("one.example.com", CauseConfig) { |
| 1216 | t.Fatal("BounceFor stopped nothing") |
| 1217 | } |
| 1218 | sink.mu.Lock() |
| 1219 | defer sink.mu.Unlock() |
| 1220 | want := fmt.Sprintf(rolloutwords.EventBounceInstance, rolloutwords.Cause(string(CauseConfig))) |
| 1221 | if len(sink.msgs) != 1 || sink.msgs[0] != want { |
| 1222 | t.Fatalf("events = %v, want %q", sink.msgs, want) |
| 1223 | } |
| 1224 | } |
| 1225 | |
| 1226 | // bounceText words both nouns through rolloutwords, and restart's cause |
| 1227 | // phrase leaves the flag-off literals in bounceLocked unchanged. |
| 1228 | func TestBounceTextIsTheApprovedWording(t *testing.T) { |
| 1229 | for _, cause := range []Cause{CauseRelease, CauseConfig, CauseRestart} { |
| 1230 | if got, want := bounceText(false, cause), fmt.Sprintf(rolloutwords.EventBounceInstance, rolloutwords.Cause(string(cause))); got != want { |
| 1231 | t.Errorf("instance, %s: %q, want %q", cause, got, want) |
| 1232 | } |
| 1233 | if got, want := bounceText(true, cause), fmt.Sprintf(rolloutwords.EventBounceSidecar, rolloutwords.Cause(string(cause))); got != want { |
| 1234 | t.Errorf("sidecar, %s: %q, want %q", cause, got, want) |
| 1235 | } |
| 1236 | } |
| 1237 | if got := bounceText(false, CauseRestart); got != "instance restarting (carlos restart)" { |
| 1238 | t.Errorf("restart bounce %q drifted from the flag-off literal", got) |
| 1239 | } |
| 1240 | if got := bounceText(true, CauseRestart); got != "sidecar restarting (carlos restart)" { |
| 1241 | t.Errorf("restart sidecar bounce %q drifted from the flag-off literal", got) |
| 1242 | } |
| 1243 | } |
| 1244 | |
| 1245 | // shrinkVerdictFloor sets the least health window a first wake's B is |
| 1246 | // exec'd with, beside shrinkFirstWake's budget, so a test drives the floor |
| 1247 | // in milliseconds rather than the spec's seconds. |
| 1248 | func shrinkVerdictFloor(t *testing.T, floor time.Duration) { |
| 1249 | t.Helper() |
| 1250 | old := firstWakeMinVerdict |
| 1251 | firstWakeMinVerdict = floor |
| 1252 | t.Cleanup(func() { firstWakeMinVerdict = old }) |
| 1253 | } |
| 1254 | |
| 1255 | // slowFirstCheck is a plan's selection gate whose first call, checkPlan's |
| 1256 | // before the exec, takes d: it stands in for a slow lease, restore or |
| 1257 | // seeding, all of which run before B's exec on the wake's own clock. |
| 1258 | func slowFirstCheck(d time.Duration) func(string) error { |
| 1259 | var once sync.Once |
| 1260 | return func(string) error { |
| 1261 | once.Do(func() { time.Sleep(d) }) |
| 1262 | return nil |
| 1263 | } |
| 1264 | } |
| 1265 | |
| 1266 | // Review fix round 1 (risk c): B's health window is fixed from the wake's |
| 1267 | // start, so a slow before-exec phase leaves B too little time for a verdict. |
| 1268 | // Below firstWakeMinVerdict B is never exec'd: the good build boots through |
| 1269 | // the same gate, nothing is recorded against B, and the wake reports as a |
| 1270 | // healthy wake of the good build that is not a first wake. |
| 1271 | func TestAFirstWakeWithNoTimeForAVerdictBootsTheGoodBuild(t *testing.T) { |
| 1272 | // B's window ends 2 s after the wake began; the floor is 1 s; the work |
| 1273 | // before the exec takes 1.5 s, leaving B about 0.5 s. |
| 1274 | shrinkFirstWake(t, 3*time.Second, time.Second, firstWakeSettle) |
| 1275 | shrinkVerdictFloor(t, time.Second) |
| 1276 | dir := t.TempDir() |
| 1277 | good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| 1278 | a, b, evs := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good, Check: slowFirstCheck(1500 * time.Millisecond)}) |
| 1279 | done, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| 1280 | if err != nil { |
| 1281 | t.Fatalf("the waiting request was not served by the good build: %v", err) |
| 1282 | } |
| 1283 | done() |
| 1284 | if gens := b.started(); len(gens) != 1 || gens[0].Build != "v1" { |
| 1285 | t.Fatalf("spawned %+v, want the good v1 alone: B must not exec without time for a verdict", gens) |
| 1286 | } |
| 1287 | if k := b.kills(); len(k) != 0 { |
| 1288 | t.Fatalf("killed %v, want nothing", k) |
| 1289 | } |
| 1290 | ev := evs.wait(t, GenHealthy) |
| 1291 | if ev.FirstWake || ev.Gen.Build != "v1" || ev.Gen.StagedPath != good.StagedPath { |
| 1292 | t.Fatalf("healthy event = %+v, want the good build, not a first wake", ev) |
| 1293 | } |
| 1294 | time.Sleep(50 * time.Millisecond) // anything afterWake would emit after GenHealthy |
| 1295 | if k := evs.kinds(); len(k) != 1 { |
| 1296 | t.Fatalf("events %v, want GenHealthy alone: nothing recorded against B, no observation", k) |
| 1297 | } |
| 1298 | if g, ok := a.Current("one.example.com"); !ok || g.Build != "v1" { |
| 1299 | t.Fatalf("Current = %+v %v, want the good build serving and not under observation", g, ok) |
| 1300 | } |
| 1301 | } |
| 1302 | |
| 1303 | // The floor needs a good build to boot in B's place. With none, B is exec'd |
| 1304 | // as a wake does today: spawnHealthTimeout from the exec, with no cap. |
| 1305 | func TestAFirstWakeWithNoGoodBuildExecsBWhateverTimeIsLeft(t *testing.T) { |
| 1306 | shrinkFirstWake(t, 3*time.Second, time.Second, 50*time.Millisecond) |
| 1307 | shrinkVerdictFloor(t, time.Second) |
| 1308 | dir := t.TempDir() |
| 1309 | next := stagedID(t, dir, "v2") |
| 1310 | a, b, evs := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Check: slowFirstCheck(1500 * time.Millisecond)}) |
| 1311 | var capped atomic.Bool |
| 1312 | a.genHealth = func(ctx context.Context, g Gen) error { |
| 1313 | if dl, ok := ctx.Deadline(); ok && time.Until(dl) < time.Second { |
| 1314 | capped.Store(true) |
| 1315 | } |
| 1316 | return ctx.Err() |
| 1317 | } |
| 1318 | done, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| 1319 | if err != nil { |
| 1320 | t.Fatalf("the wake: %v", err) |
| 1321 | } |
| 1322 | done() |
| 1323 | if gens := b.started(); len(gens) != 1 || gens[0].Build != "v2" { |
| 1324 | t.Fatalf("spawned %+v, want B: there is no good build to boot instead", gens) |
| 1325 | } |
| 1326 | if capped.Load() { |
| 1327 | t.Fatal("B's health check was capped: with no good build it keeps today's full window") |
| 1328 | } |
| 1329 | if ev := evs.wait(t, GenHealthy); !ev.FirstWake || ev.Gen.Build != "v2" { |
| 1330 | t.Fatalf("healthy event = %+v, want B's first wake", ev) |
| 1331 | } |
| 1332 | } |
| 1333 | |
| 1334 | // With at least the floor left, B is exec'd as before, and a failing B is |
| 1335 | // still a failure that falls back (the real-budget case is |
| 1336 | // TestFirstWakeFallsBackToTheGoodBuildWithinTheBudget). |
| 1337 | func TestAFirstWakeWithTimeForAVerdictStillExecsB(t *testing.T) { |
| 1338 | shrinkFirstWake(t, 3*time.Second, time.Second, firstWakeSettle) |
| 1339 | shrinkVerdictFloor(t, time.Second) |
| 1340 | dir := t.TempDir() |
| 1341 | good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| 1342 | a, b, evs := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good, Check: slowFirstCheck(0)}) |
| 1343 | a.genHealth = func(_ context.Context, g Gen) error { |
| 1344 | if g.Build == "v2" { |
| 1345 | return errors.New("GET / answered 500") |
| 1346 | } |
| 1347 | return nil |
| 1348 | } |
| 1349 | done, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| 1350 | if err != nil { |
| 1351 | t.Fatalf("the waiting request was not served by the good build: %v", err) |
| 1352 | } |
| 1353 | done() |
| 1354 | if gens := b.started(); len(gens) != 2 || gens[0].Build != "v2" || gens[1].Build != "v1" { |
| 1355 | t.Fatalf("spawned %+v, want v2 then the good v1", gens) |
| 1356 | } |
| 1357 | if ev := evs.wait(t, GenFellBack); ev.Failed.Version != "v2" || ev.Gen.Build != "v1" { |
| 1358 | t.Fatalf("fallback event = %+v", ev) |
| 1359 | } |
| 1360 | } |
| 1361 | |
| 1362 | // Astra R3 ruling 206 and plan review finding 9 (zdd-readiness Task 6): a |
| 1363 | // planned first wake whose nested start root accepted, with an uncertain |
| 1364 | // outcome, may have run its build. In hold mode that is always a failure of |
| 1365 | // the build (GenFailed, under the outcome fence), never a cancellation the |
| 1366 | // scheduler could boot the build again after. In overlap only a unit that |
| 1367 | // exited before it was observed (errStartedThenExited) is a proven crash at |
| 1368 | // the start, so it fails the build, and the good build boots in its place |
| 1369 | // only when the backend's kill proved the generation gone; unproven, the |
| 1370 | // wake serves nothing. Any other uncertain overlap start keeps its |
| 1371 | // GenCancelled with Executed and boots nothing. Each wake sends its token |
| 1372 | // exactly one terminal event. Before the fix both hold cases sent |
| 1373 | // GenCancelled and the proven overlap crash served nothing. |
| 1374 | func TestAPlannedWakeWhoseNestedStartIsUncertain(t *testing.T) { |
| 1375 | t.Parallel() |
| 1376 | unitFailed := errors.New("the unit failed to start") |
| 1377 | // B (v2) is answered as the case says; the good build (v1) starts. |
| 1378 | bAnswers := func(inv string, err error) func(instancectl.StagedExe) (string, error) { |
| 1379 | return func(x instancectl.StagedExe) (string, error) { |
| 1380 | if x.Version == "v2" { |
| 1381 | return inv, err |
| 1382 | } |
| 1383 | return "inv-7", nil |
| 1384 | } |
| 1385 | } |
| 1386 | unproven := errors.New("stop timed out") |
| 1387 | for _, tc := range []struct { |
| 1388 | name string |
| 1389 | mode SwapMode |
| 1390 | answer func(instancectl.StagedExe) (string, error) |
| 1391 | stopErr error |
| 1392 | want GenEventKind |
| 1393 | executed bool // GenCancelled only |
| 1394 | goodAsked bool // root was asked to start the good build, which serves |
| 1395 | }{ |
| 1396 | {"hold, the unit exited before it was observed", ModeHold, bAnswers("", nil), nil, GenFailed, false, false}, |
| 1397 | {"hold, root's start failed", ModeHold, bAnswers("", unitFailed), nil, GenFailed, false, false}, |
| 1398 | {"overlap, the unit exited before it was observed, kill proven", ModeOverlap, bAnswers("", nil), nil, GenFellBack, false, true}, |
| 1399 | {"overlap, the unit exited before it was observed, kill unproven", ModeOverlap, bAnswers("", nil), unproven, GenFailed, false, false}, |
| 1400 | {"overlap, root's start failed, kill proven", ModeOverlap, bAnswers("", unitFailed), nil, GenCancelled, true, false}, |
| 1401 | } { |
| 1402 | t.Run(tc.name, func(t *testing.T) { |
| 1403 | b := newLaunchBackend(t, "nested", &fakeSwapBackend{fakeBackend: &fakeBackend{}}) |
| 1404 | b.ctl.exeAnswer, b.ctl.stopGenErr = tc.answer, tc.stopErr |
| 1405 | dir := t.TempDir() |
| 1406 | next := markerScript(t, dir, "v2", filepath.Join(t.TempDir(), "v2.ran")) |
| 1407 | good := markerScript(t, dir, "v1", filepath.Join(t.TempDir(), "v1.ran")) |
| 1408 | plan := WakePlan{Boot: next, Mode: tc.mode, FirstWake: true, Token: 7} |
| 1409 | if tc.mode == ModeOverlap { |
| 1410 | plan.Fallback = &good |
| 1411 | } |
| 1412 | a, evs := launchActivator(t, b, plan) |
| 1413 | done, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| 1414 | if tc.goodAsked { |
| 1415 | if err != nil { |
| 1416 | t.Fatalf("wake = %v, want the good build serving", err) |
| 1417 | } |
| 1418 | done() |
| 1419 | } else if err == nil { |
| 1420 | done() |
| 1421 | t.Fatal("the wake served, want it failed with nothing serving") |
| 1422 | } |
| 1423 | ev := evs.wait(t, tc.want) |
| 1424 | if ev.Token != 7 { |
| 1425 | t.Fatalf("%s event = %+v, want the plan's token", tc.want, ev) |
| 1426 | } |
| 1427 | switch tc.want { |
| 1428 | case GenFailed: |
| 1429 | if ev.Failed.Version != "v2" { |
| 1430 | t.Fatalf("failure event = %+v, want v2 failed", ev) |
| 1431 | } |
| 1432 | case GenFellBack: |
| 1433 | if ev.Failed.Version != "v2" || ev.Gen.Build != "v1" { |
| 1434 | t.Fatalf("fallback event = %+v, want v2 failed and v1 serving", ev) |
| 1435 | } |
| 1436 | case GenCancelled: |
| 1437 | if ev.Executed != tc.executed { |
| 1438 | t.Fatalf("cancellation = %+v, want Executed %v", ev, tc.executed) |
| 1439 | } |
| 1440 | } |
| 1441 | time.Sleep(50 * time.Millisecond) |
| 1442 | if k := evs.kinds(); len(k) != 1 { |
| 1443 | t.Fatalf("events %v, want exactly one terminal event for the token", k) |
| 1444 | } |
| 1445 | b.ctl.mu.Lock() |
| 1446 | var asked []string |
| 1447 | for _, x := range b.ctl.got { |
| 1448 | asked = append(asked, x.Version) |
| 1449 | } |
| 1450 | b.ctl.mu.Unlock() |
| 1451 | if want := map[bool]int{true: 2, false: 1}[tc.goodAsked]; len(asked) != want || asked[0] != "v2" { |
| 1452 | t.Fatalf("root was asked to start %v, want v2 first and %d start(s) in all", asked, want) |
| 1453 | } |
| 1454 | }) |
| 1455 | } |
| 1456 | } |
| 1457 | |
amadan