| 1 | package activator |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "fmt" |
| 7 | "strings" |
| 8 | "testing" |
| 9 | "time" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/cgroup" |
| 12 | "github.com/carlosframework/platform/internal/hostguard" |
| 13 | "github.com/carlosframework/platform/internal/hostwrite" |
| 14 | "github.com/carlosframework/platform/internal/instancectl" |
| 15 | "github.com/carlosframework/platform/internal/registry" |
| 16 | ) |
| 17 | |
| 18 | // The happy path, in spec order: raise, exec the staged path, health |
| 19 | // through the handle, switch, then SIGTERM the retiring generation, then |
| 20 | // restore the parent. The live link names v3 (root moved it) and makes no |
| 21 | // difference: the successor execs the attempt's staged v2 (review focus 2). |
| 22 | // The retiring generation is signalled only after AppSocket has moved |
| 23 | // (review focus 1). |
| 24 | func TestSwapOverlapSwitchesAfterHealthAndRetiresTheIncumbent(t *testing.T) { |
| 25 | f := newSwapFixture(t) |
| 26 | f.b.execVerFn = func() string { return "v3" } |
| 27 | incumbent := f.b.lastProc() |
| 28 | res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) |
| 29 | if res.Outcome != SwapDone || res.Gen.Build != "v2" || res.Gen.StagedPath != f.staged { |
| 30 | t.Fatalf("Swap = %+v", res) |
| 31 | } |
| 32 | if got := f.appSocket(t); got != res.Gen.Socket { |
| 33 | t.Fatalf("AppSocket = %q, want the successor's %q", got, res.Gen.Socket) |
| 34 | } |
| 35 | ev := f.b.events() |
| 36 | order := []int{ |
| 37 | indexPrefix(ev, "raise"), |
| 38 | indexPrefix(ev, "startGen:"+f.staged), |
| 39 | indexPrefix(ev, fmt.Sprintf("switch:%d", res.Gen.N)), |
| 40 | indexPrefix(ev, "stop:instance"), |
| 41 | // The parent restore. A bare "restore" is the wake's data restore |
| 42 | // (fakeBackend.Restore), logged before any of this. |
| 43 | indexPrefix(ev, "restore:"), |
| 44 | } |
| 45 | for i := range order { |
| 46 | if order[i] < 0 || (i > 0 && order[i] < order[i-1]) { |
| 47 | t.Fatalf("event order %v (indexes %v): want raise, startGen, switch, stop:instance, restore", ev, order) |
| 48 | } |
| 49 | } |
| 50 | if !incumbent.isStopped() { |
| 51 | t.Fatal("the retiring generation was never stopped") |
| 52 | } |
| 53 | cur, members := f.a.genStateForTest(f.host) |
| 54 | if cur.N != res.Gen.N || len(members) != 0 { |
| 55 | t.Fatalf("after the swap: current %d, members %v", cur.N, members) |
| 56 | } |
| 57 | if _, raises, restores := f.b.calls(); raises != 1 || restores != 1 { |
| 58 | t.Fatalf("raises %d restores %d, want 1 and 1", raises, restores) |
| 59 | } |
| 60 | if got := f.b.startCount(); got != 1 { |
| 61 | t.Fatalf("Start (live link) calls = %d, want only the original wake", got) |
| 62 | } |
| 63 | } |
| 64 | |
| 65 | // During the overlap the parent holds the incumbent's tier plus the |
| 66 | // successor's, each captured from its own artifact (spec §7.3), and the |
| 67 | // restore is to whichever generation is current afterwards. Here the |
| 68 | // incumbent (v1) declares children (1 GiB) and the successor (v2) does |
| 69 | // not (512 MiB). Summing one tier twice gives a 1 GiB parent (or 2 GiB), |
| 70 | // and restoring to the live tier leaves the wrong limit after a failure. |
| 71 | func TestSwapOverlapUsesEachArtifactsTier(t *testing.T) { |
| 72 | t.Parallel() |
| 73 | const gib, half = int64(1 << 30), int64(1 << 29) |
| 74 | t.Run("switched: restored to the successor's tier", func(t *testing.T) { |
| 75 | f := newSwapFixture(t) |
| 76 | f.b.tiers = map[string]cgroup.Limit{"v1": cgroup.TierChildren, "v2": cgroup.TierDefault} |
| 77 | res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) |
| 78 | if res.Outcome != SwapDone || res.Gen.Tier != cgroup.TierDefault { |
| 79 | t.Fatalf("Swap = %+v, want done with the successor's own tier", res) |
| 80 | } |
| 81 | ev := f.b.events() |
| 82 | if indexPrefix(ev, fmt.Sprintf("raise:%d+%d", gib, half)) < 0 || indexPrefix(ev, fmt.Sprintf("restore:%d", half)) < 0 { |
| 83 | t.Fatalf("events %v: want raise:%d+%d and restore:%d", ev, gib, half, half) |
| 84 | } |
| 85 | if g, _ := f.b.lastGen(); g.Tier != cgroup.TierDefault { |
| 86 | t.Fatalf("the successor spawned at %v, want its own 512 MiB tier", g.Tier) |
| 87 | } |
| 88 | }) |
| 89 | t.Run("failed: restored to the incumbent's tier", func(t *testing.T) { |
| 90 | f := newSwapFixture(t) |
| 91 | f.b.tiers = map[string]cgroup.Limit{"v1": cgroup.TierChildren, "v2": cgroup.TierDefault} |
| 92 | f.a.genHealth = func(context.Context, Gen) error { return errors.New("unhealthy: HTTP 503") } |
| 93 | if res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest); res.Outcome != SwapFailed { |
| 94 | t.Fatalf("Swap = %+v", res) |
| 95 | } |
| 96 | if ev := f.b.events(); indexPrefix(ev, fmt.Sprintf("restore:%d", gib)) < 0 { |
| 97 | t.Fatalf("events %v: want restore:%d (the incumbent is still current)", ev, gib) |
| 98 | } |
| 99 | }) |
| 100 | } |
| 101 | |
| 102 | // The "swapped" event carries the retiring generation's drain time (spec |
| 103 | // §9.4), so the result measures it: from SIGTERM to the proven exit. |
| 104 | func TestSwapOverlapReportsTheDrainTime(t *testing.T) { |
| 105 | t.Parallel() |
| 106 | f := newSwapFixture(t) |
| 107 | gate := make(chan struct{}) |
| 108 | f.b.lastProc().setStopGate(gate) |
| 109 | time.AfterFunc(80*time.Millisecond, func() { close(gate) }) |
| 110 | res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) |
| 111 | if res.Outcome != SwapDone { |
| 112 | t.Fatalf("Swap = %+v", res) |
| 113 | } |
| 114 | if res.Drain < 50*time.Millisecond || res.Drain > 5*time.Second { |
| 115 | t.Fatalf("Drain = %v, want the retiring generation's stop (about 80ms)", res.Drain) |
| 116 | } |
| 117 | } |
| 118 | |
| 119 | // A successor that fails its health check, fails to spawn, or exits before |
| 120 | // the switch leaves the incumbent current and untouched (spec §8.1). |
| 121 | func TestSwapOverlapFailureKeepsTheIncumbent(t *testing.T) { |
| 122 | t.Parallel() |
| 123 | for _, tc := range []struct { |
| 124 | name string |
| 125 | arrange func(f *swapFixture) |
| 126 | reason string |
| 127 | }{ |
| 128 | {"unhealthy", func(f *swapFixture) { |
| 129 | f.a.genHealth = func(context.Context, Gen) error { return errors.New("unhealthy: HTTP 503") } |
| 130 | }, "health"}, |
| 131 | {"spawn fails", func(f *swapFixture) { f.b.genErr = errors.New("exec format error") }, "spawn"}, |
| 132 | {"exits before the switch", func(f *swapFixture) { |
| 133 | f.a.genHealth = func(context.Context, Gen) error { |
| 134 | _, p := f.b.lastGen() |
| 135 | p.exit() |
| 136 | return nil |
| 137 | } |
| 138 | }, errSuccessorExited.Error()}, |
| 139 | } { |
| 140 | t.Run(tc.name, func(t *testing.T) { |
| 141 | f := newSwapFixture(t) |
| 142 | tc.arrange(f) |
| 143 | incumbent := f.b.lastProc() |
| 144 | res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) |
| 145 | if res.Outcome != SwapFailed || !strings.Contains(res.Reason, tc.reason) { |
| 146 | t.Fatalf("Swap = %+v, want failed with %q", res, tc.reason) |
| 147 | } |
| 148 | if got := f.appSocket(t); got != f.incumbent.Socket { |
| 149 | t.Fatalf("AppSocket moved to %q after a failure", got) |
| 150 | } |
| 151 | if incumbent.isStopped() { |
| 152 | t.Fatal("the incumbent was stopped by a failed swap") |
| 153 | } |
| 154 | if _, p := f.b.lastGen(); p != nil && !p.isStopped() { |
| 155 | t.Fatal("the failed successor was left running") |
| 156 | } |
| 157 | if _, raises, restores := f.b.calls(); raises != restores { |
| 158 | t.Fatalf("raises %d restores %d: the parent limit was not restored", raises, restores) |
| 159 | } |
| 160 | }) |
| 161 | } |
| 162 | } |
| 163 | |
| 164 | // Every refusal spawns nothing. Swaps off is the flag-off test. |
| 165 | func TestSwapRefusals(t *testing.T) { |
| 166 | t.Parallel() |
| 167 | for _, tc := range []struct { |
| 168 | name string |
| 169 | arrange func(t *testing.T, f *swapFixture, at *Attempt) |
| 170 | want error |
| 171 | }{ |
| 172 | {"swap off", func(_ *testing.T, f *swapFixture, _ *Attempt) { f.a.gens = false }, errSwapOff}, |
| 173 | {"asleep", func(_ *testing.T, _ *swapFixture, at *Attempt) { at.Host = "asleep.example.com" }, errSwapNotAwake}, |
| 174 | {"worker", func(_ *testing.T, f *swapFixture, _ *Attempt) { |
| 175 | f.a.mu.Lock() |
| 176 | f.a.entries[f.host].worker = true |
| 177 | f.a.mu.Unlock() |
| 178 | }, errSwapKind}, |
| 179 | {"busy", func(t *testing.T, f *swapFixture, _ *Attempt) { |
| 180 | plantMember(t, f.a, f.host, roleSuccessor, &fakeProc{}, 77) |
| 181 | }, errSwapBusy}, |
| 182 | {"unknown mode", func(_ *testing.T, _ *swapFixture, at *Attempt) { at.Mode = "" }, errSwapMode}, |
| 183 | {"superseded", func(_ *testing.T, f *swapFixture, _ *Attempt) { f.rev.Add(1) }, errSwapSuperseded}, |
| 184 | {"fenced", func(_ *testing.T, f *swapFixture, _ *Attempt) { |
| 185 | f.a.mu.Lock() |
| 186 | f.a.fences = map[string]uint64{f.host: 3} |
| 187 | f.a.mu.Unlock() |
| 188 | }, errSwapFenced}, |
| 189 | {"lease lost", func(_ *testing.T, f *swapFixture, _ *Attempt) { f.b.lastRep().setErr(errors.New("renewal failed")) }, errSwapLeaseLost}, |
| 190 | } { |
| 191 | t.Run(tc.name, func(t *testing.T) { |
| 192 | f := newSwapFixture(t) |
| 193 | at := f.attempt(ModeOverlap) |
| 194 | tc.arrange(t, f, &at) |
| 195 | res := f.a.Swap(context.Background(), at, f.newest) |
| 196 | if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, tc.want.Error()) { |
| 197 | t.Fatalf("Swap = %+v, want abandoned: %v", res, tc.want) |
| 198 | } |
| 199 | if gens, raises, _ := f.b.calls(); gens != 0 || raises != 0 { |
| 200 | t.Fatalf("a refused attempt spawned %d generation(s), raised %d time(s)", gens, raises) |
| 201 | } |
| 202 | }) |
| 203 | } |
| 204 | t.Run("digest mismatch", func(t *testing.T) { |
| 205 | f := newSwapFixture(t) |
| 206 | at := f.attempt(ModeOverlap) |
| 207 | at.Digest = strings.Repeat("0", 64) |
| 208 | res := f.a.Swap(context.Background(), at, f.newest) |
| 209 | if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, "staging") { |
| 210 | t.Fatalf("Swap = %+v, want abandoned as a staging error", res) |
| 211 | } |
| 212 | if gens, _, _ := f.b.calls(); gens != 0 { |
| 213 | t.Fatal("a mismatched artifact reached StartGen") |
| 214 | } |
| 215 | }) |
| 216 | t.Run("backend without generations", func(t *testing.T) { |
| 217 | a, _, _, _ := testActivator(t, time.Hour) |
| 218 | a.gens = true |
| 219 | if res := a.Swap(context.Background(), Attempt{Host: "x.example.com", Mode: ModeOverlap}, func(string) uint64 { return 0 }); res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, errSwapNoGens.Error()) { |
| 220 | t.Fatalf("Swap = %+v", res) |
| 221 | } |
| 222 | }) |
| 223 | } |
| 224 | |
| 225 | // A stopping entry admits no successor (spec §7.4: "the entry is still |
| 226 | // awake"). This is the witness for mutation M5. |
| 227 | func TestSwapWitnessNoAdmissionOntoAStoppingEntry(t *testing.T) { |
| 228 | f := newSwapFixture(t) |
| 229 | gate := make(chan struct{}) |
| 230 | f.b.lastProc().setStopGate(gate) |
| 231 | f.a.Bounce(f.host) |
| 232 | waitFor(t, "the entry to stop", func() bool { |
| 233 | f.a.mu.Lock() |
| 234 | defer f.a.mu.Unlock() |
| 235 | e := f.a.entries[f.host] |
| 236 | return e != nil && e.state == actStopping |
| 237 | }) |
| 238 | // The refusal is immediate. A Swap that admits a successor instead |
| 239 | // switches and then drains the incumbent, whose Stop the gate holds, so |
| 240 | // it cannot return until the gate opens: bound the wait, so the |
| 241 | // mutation fails here by name rather than hanging the run. |
| 242 | results := make(chan SwapResult, 1) |
| 243 | go func() { results <- f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) }() |
| 244 | var res SwapResult |
| 245 | select { |
| 246 | case res = <-results: |
| 247 | case <-time.After(5 * time.Second): |
| 248 | close(gate) |
| 249 | select { |
| 250 | case res = <-results: |
| 251 | t.Fatalf("Swap waited on the stopping incumbent's drain, then returned %+v: it admitted a successor onto a stopping entry", res) |
| 252 | case <-time.After(5 * time.Second): |
| 253 | t.Fatal("Swap onto a stopping entry did not return") |
| 254 | } |
| 255 | } |
| 256 | close(gate) |
| 257 | if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, errSwapNotAwake.Error()) { |
| 258 | t.Fatalf("Swap = %+v, want abandoned: not awake", res) |
| 259 | } |
| 260 | if gens, _, _ := f.b.calls(); gens != 0 { |
| 261 | t.Fatalf("StartGen ran %d time(s) on a stopping entry", gens) |
| 262 | } |
| 263 | } |
| 264 | |
| 265 | // The one outcome classifier (spec §6.4, §7.4): only a failure of the build |
| 266 | // is stepFailed. A cancelled lifecycle, an intentional stop and a refusal |
| 267 | // by the attempt gate record nothing; a health deadline that expires is the |
| 268 | // build's. |
| 269 | func TestClassifyStep(t *testing.T) { |
| 270 | live, ended := context.Background(), func() context.Context { |
| 271 | ctx, cancel := context.WithCancel(context.Background()) |
| 272 | cancel() |
| 273 | return ctx |
| 274 | }() |
| 275 | for _, tc := range []struct { |
| 276 | name string |
| 277 | ctx context.Context |
| 278 | err error |
| 279 | want stepOutcome |
| 280 | }{ |
| 281 | {"ok", live, nil, stepOK}, |
| 282 | {"the lifecycle ended", ended, errors.New("dial: connection refused"), stepCancelled}, |
| 283 | {"cancelled", live, fmt.Errorf("health: %w", context.Canceled), stepCancelled}, |
| 284 | {"fenced", live, fmt.Errorf("wake: %w", ErrFenced), stepCancelled}, |
| 285 | {"the barrier", live, errSwapStopped, stepCancelled}, |
| 286 | {"superseded", live, fmt.Errorf("pin: %w", errSwapSuperseded), stepStale}, |
| 287 | {"lease lost", live, errSwapLeaseLost, stepStale}, |
| 288 | // Claims validation at the exec (spawnClaimedGen's Work.With): |
| 289 | // nothing ran, so no failure of the build and no halted cohort. |
| 290 | {"a changed stamp at the exec", live, fmt.Errorf("with: %w", hostguard.ErrStampMismatch), stepStale}, |
| 291 | {"a durable fence at the exec", live, fmt.Errorf("with: %w", hostguard.ErrFenced), stepStale}, |
| 292 | // The strict-host gate refused a placed spawn (plan 2), and every |
| 293 | // claims-on launch is closed on this base: nothing of the build ran. |
| 294 | {"the strict gate", live, fmt.Errorf("x: %w: %w", ErrStrictRequired, ErrStrictGroupInUse), stepStale}, |
| 295 | {"a closed claims launch", live, fmt.Errorf("launch: %w", ErrClaimIsolationUnavailable), stepStale}, |
| 296 | // A row that cannot run a staged generation at all. |
| 297 | {"a unit row", live, errUnitNoGens, stepStale}, |
| 298 | // Amadan #272: the generation start refused before any exec because |
| 299 | // the host's parent holds a process directly. |
| 300 | {"a populated legacy parent", live, fmt.Errorf("h: %w: holds 1 process(es)", cgroup.ErrLegacyParentPopulated), stepStale}, |
| 301 | // Task 19: the start's killed cgroup could not be made fresh, so the |
| 302 | // manager refused before any exec. |
| 303 | {"a killed cgroup that would not go", live, fmt.Errorf("%w: carlos-exec@h/g2: device or resource busy", cgroup.ErrKilledCgroupKept), stepStale}, |
| 304 | // Task 7 (readiness round 2): a generation whose limits could not be |
| 305 | // applied is refused before any exec: the box's condition. |
| 306 | {"a generation's limits not applied", live, fmt.Errorf("h: %w: g2: memory.max: is a directory", cgroup.ErrLimitNotApplied), stepStale}, |
| 307 | // Root refused the nested staged executable before it mounted or |
| 308 | // started anything (Task 9): an abandonment, not a failed build. |
| 309 | {"root refused the staged exe", live, fmt.Errorf("start unit: %w: not blog's staged v2", instancectl.ErrStagedExe), stepStale}, |
| 310 | {"a backend without generations", live, errSwapNoGens, stepStale}, |
| 311 | {"a health deadline", live, fmt.Errorf("health: %w", context.DeadlineExceeded), stepFailed}, |
| 312 | {"HTTP 500", live, errors.New("GET / answered 500"), stepFailed}, |
| 313 | } { |
| 314 | if got := classifyStep(tc.ctx, tc.err); got != tc.want { |
| 315 | t.Errorf("%s: %d, want %d", tc.name, got, tc.want) |
| 316 | } |
| 317 | } |
| 318 | } |
| 319 | |
| 320 | // The gate's verdict is typed (spec §6.3, §7.4): a supersession may restore |
| 321 | // behind a fresh ownership check; a lost lease, a fence, shutdown or a |
| 322 | // claims refusal stops every spawn. |
| 323 | func TestVerdictOf(t *testing.T) { |
| 324 | for _, tc := range []struct { |
| 325 | err error |
| 326 | want gateVerdict |
| 327 | }{ |
| 328 | {nil, gateProceed}, |
| 329 | {errors.New("GET / answered 500"), gateProceed}, |
| 330 | {fmt.Errorf("gate: %w", errSwapSuperseded), gateSuperseded}, |
| 331 | {errSwapLeaseLost, gateNoOwnership}, |
| 332 | {errSwapFenced, gateNoOwnership}, |
| 333 | {errSwapShutdown, gateNoOwnership}, |
| 334 | {errSwapNotAwake, gateNoOwnership}, |
| 335 | // The barrier's cancel (memberCurrentLocked): the entry is stopping, |
| 336 | // so nothing may spawn for it, a restore included. |
| 337 | {errSwapStopped, gateNoOwnership}, |
| 338 | {fmt.Errorf("with: %w", hostguard.ErrStampMismatch), gateNoOwnership}, |
| 339 | {fmt.Errorf("with: %w", hostguard.ErrFenced), gateNoOwnership}, |
| 340 | // P5 review partial new #3: an absent claim, and a claim read that |
| 341 | // failed before the exec, are no failure of the build either. |
| 342 | {fmt.Errorf("with: %w", hostguard.ErrNoClaim), gateNoOwnership}, |
| 343 | {claimAdmission(errors.New("s3: GetObject: request timeout"), false), gateNoOwnership}, |
| 344 | {claimAdmission(errors.New("GET / answered 500"), true), gateProceed}, |
| 345 | // Plan 2's strict-host gate on placed spawns (index P2-8), as the |
| 346 | // exec backend wraps them. |
| 347 | {fmt.Errorf("h: %w", ErrStrictRequired), gateNoOwnership}, |
| 348 | {fmt.Errorf("h: %w: %w", ErrStrictRequired, ErrStrictGroupInUse), gateNoOwnership}, |
| 349 | {fmt.Errorf("h: %w", ErrStrictGroupInUse), gateNoOwnership}, |
| 350 | {fmt.Errorf("h: %w", ErrStrictCleanupPending), gateNoOwnership}, |
| 351 | // Every claims-on launch is closed on this base (index P2-9). It |
| 352 | // wraps hostbox.ErrUnsupported, and is still NoOwnership, raw or |
| 353 | // as a claims admission. |
| 354 | {ErrClaimIsolationUnavailable, gateNoOwnership}, |
| 355 | {fmt.Errorf("start: %w", ErrClaimIsolationUnavailable), gateNoOwnership}, |
| 356 | {claimAdmission(ErrClaimIsolationUnavailable, false), gateNoOwnership}, |
| 357 | // Amadan #272: a populated legacy parent refuses the start before any |
| 358 | // exec, and no build is at fault. |
| 359 | {fmt.Errorf("h: %w: holds 1 process(es)", cgroup.ErrLegacyParentPopulated), gateNoOwnership}, |
| 360 | // Task 19: a start whose killed cgroup could not be made fresh is |
| 361 | // refused before the command is built; no build is at fault. |
| 362 | {cgroup.ErrKilledCgroupKept, gateNoOwnership}, |
| 363 | {fmt.Errorf("%w: carlos-exec@h/role-sidecar: device or resource busy", cgroup.ErrKilledCgroupKept), gateNoOwnership}, |
| 364 | // Task 7: a generation start refused because its limits could not |
| 365 | // all be applied (or it could only be started then moved) is refused |
| 366 | // before the exec; no build is at fault. |
| 367 | {cgroup.ErrLimitNotApplied, gateNoOwnership}, |
| 368 | {fmt.Errorf("h: %w: the placement shim is not available", cgroup.ErrLimitNotApplied), gateNoOwnership}, |
| 369 | // A row that cannot run a staged generation: nothing staged spawns. |
| 370 | {errUnitNoGens, gateNoOwnership}, |
| 371 | {instancectl.ErrStagedExe, gateNoOwnership}, |
| 372 | {fmt.Errorf("start unit: %w: x", instancectl.ErrStagedExe), gateNoOwnership}, |
| 373 | {errSwapNoGens, gateNoOwnership}, |
| 374 | } { |
| 375 | if got := verdictOf(tc.err); got != tc.want { |
| 376 | t.Errorf("verdictOf(%v) = %d, want %d", tc.err, got, tc.want) |
| 377 | } |
| 378 | } |
| 379 | } |
| 380 | |
| 381 | // Spec §6.3, §7.4: the checks run again at the exec, not only at the |
| 382 | // admission. A newer attempt that lands while the parent is being raised |
| 383 | // (or, with plan 5, while the pins are being published) means nothing |
| 384 | // execs, and the raise is undone. Checked only at the admission, the |
| 385 | // superseded build execs and is abandoned only at the switch. |
| 386 | func TestSwapOverlapReChecksAtTheExec(t *testing.T) { |
| 387 | f := newSwapFixture(t) |
| 388 | f.b.onRaise = func() { f.rev.Add(1) } |
| 389 | res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) |
| 390 | if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, errSwapSuperseded.Error()) { |
| 391 | t.Fatalf("Swap = %+v, want abandoned: superseded", res) |
| 392 | } |
| 393 | if gens, raises, restores := f.b.calls(); gens != 0 || raises != restores { |
| 394 | t.Fatalf("spawned %d, raises %d, restores %d: want nothing exec'd and the raise undone", gens, raises, restores) |
| 395 | } |
| 396 | } |
| 397 | |
| 398 | // A newer attempt that lands during the health check is seen at the switch. |
| 399 | // The successor is abandoned and stopped, and the incumbent keeps serving. |
| 400 | func TestSwapOverlapSupersededAtTheSwitch(t *testing.T) { |
| 401 | f := newSwapFixture(t) |
| 402 | f.a.genHealth = func(context.Context, Gen) error { f.rev.Add(1); return nil } |
| 403 | res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) |
| 404 | if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, errSwapSuperseded.Error()) { |
| 405 | t.Fatalf("Swap = %+v", res) |
| 406 | } |
| 407 | if got := f.appSocket(t); got != f.incumbent.Socket { |
| 408 | t.Fatalf("AppSocket = %q, want the incumbent", got) |
| 409 | } |
| 410 | if _, p := f.b.lastGen(); !p.isStopped() { |
| 411 | t.Fatal("the abandoned successor was left running") |
| 412 | } |
| 413 | } |
| 414 | |
| 415 | // useClaimSpawn stands the claimSpawn seam in for the kernel: a claims-on |
| 416 | // spawn needs a delegated cgroup tree and an open isolated launch, and |
| 417 | // every claims-on launch is closed on this base. |
| 418 | func useClaimSpawn(f *swapFixture) { |
| 419 | f.a.claimSpawn = func(ctx context.Context, _ *actEntry, rt registry.Route, _ hostwrite.Work, g Gen) (Gen, Proc, error) { |
| 420 | return f.b.StartStaged(ctx, rt, g) |
| 421 | } |
| 422 | } |
| 423 | |
| 424 | // Claims on: a refused selection or a refused check, before the spawn or |
| 425 | // at the switch, abandons the attempt and is never retried as claims off. |
| 426 | func TestSwapClaimsOn(t *testing.T) { |
| 427 | t.Parallel() |
| 428 | for _, tc := range []struct { |
| 429 | name string |
| 430 | sel func() hostwrite.Selector |
| 431 | wantGens int |
| 432 | }{ |
| 433 | {"selector refuses", func() hostwrite.Selector { return &swapSelector{err: hostwrite.ErrUnavailable} }, 0}, |
| 434 | {"no selector", func() hostwrite.Selector { return nil }, 0}, |
| 435 | {"refused before the spawn", func() hostwrite.Selector { return &swapSelector{work: &swapWork{denyAt: 1}} }, 0}, |
| 436 | {"refused at the exec", func() hostwrite.Selector { return &swapSelector{work: &swapWork{denyAt: 2}} }, 0}, |
| 437 | {"refused at the switch", func() hostwrite.Selector { return &swapSelector{work: &swapWork{denyAt: 3}} }, 1}, |
| 438 | } { |
| 439 | t.Run(tc.name, func(t *testing.T) { |
| 440 | f := newSwapFixture(t) |
| 441 | f.a.cfg.HostWrites = hostwrite.Config{Enabled: true, Selector: tc.sel()} |
| 442 | // A runtime that can start a generation of its own, so each row |
| 443 | // is refused where it names, not by errClaimsNoGenerations. |
| 444 | f.a.cfg.Claims = &ClaimRuntime{Pins: true, Generations: true} |
| 445 | // The seam stands in for an open launch, so each row reaches the |
| 446 | // refusal it names; without it, claimsSwapRefusal refuses every |
| 447 | // row first (the "closed launch, no seam" case below). |
| 448 | useClaimSpawn(f) |
| 449 | res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) |
| 450 | if res.Outcome != SwapAbandoned { |
| 451 | t.Fatalf("Swap = %+v, want abandoned", res) |
| 452 | } |
| 453 | gens, raises, restores := f.b.calls() |
| 454 | if gens != tc.wantGens || raises != restores { |
| 455 | t.Fatalf("gens %d (want %d), raises %d, restores %d", gens, tc.wantGens, raises, restores) |
| 456 | } |
| 457 | if got := f.appSocket(t); got != f.incumbent.Socket { |
| 458 | t.Fatalf("AppSocket = %q, want the incumbent", got) |
| 459 | } |
| 460 | if _, p := f.b.lastGen(); p != nil && !p.isStopped() { |
| 461 | t.Fatal("a refused successor was left running") |
| 462 | } |
| 463 | }) |
| 464 | } |
| 465 | t.Run("allowed", func(t *testing.T) { |
| 466 | f := newSwapFixture(t) |
| 467 | w := &swapWork{} |
| 468 | f.a.cfg.HostWrites = hostwrite.Config{Enabled: true, Selector: &swapSelector{work: w}} |
| 469 | f.a.cfg.Claims = &ClaimRuntime{Pins: true, Generations: true} |
| 470 | useClaimSpawn(f) |
| 471 | if res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest); res.Outcome != SwapDone { |
| 472 | t.Fatalf("Swap = %+v", res) |
| 473 | } |
| 474 | if w.withCalls() != 3 { |
| 475 | t.Fatalf("With calls = %d, want 3 (the admission, the exec and the switch)", w.withCalls()) |
| 476 | } |
| 477 | }) |
| 478 | // Every claims-on launch is closed on this base (index P2-9). Without |
| 479 | // the seam a claims-on swap is refused before its admission and its |
| 480 | // raise (claimsSwapRefusal, final review F9): abandoned (never failed, |
| 481 | // which would halt the cohort for a build that never ran), nothing |
| 482 | // raised or spawned, and never sent down the claims-off StartStaged. |
| 483 | t.Run("closed launch, no seam", func(t *testing.T) { |
| 484 | f := newSwapFixture(t) |
| 485 | w := &swapWork{} |
| 486 | f.a.cfg.HostWrites = hostwrite.Config{Enabled: true, Selector: &swapSelector{work: w}} |
| 487 | f.a.cfg.Claims = &ClaimRuntime{Pins: true, Generations: true} |
| 488 | res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) |
| 489 | if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, ErrClaimIsolationUnavailable.Error()) { |
| 490 | t.Fatalf("Swap = %+v, want abandoned: %v", res, ErrClaimIsolationUnavailable) |
| 491 | } |
| 492 | gens, raises, restores := f.b.calls() |
| 493 | if gens != 0 || raises != 0 || restores != 0 { |
| 494 | t.Fatalf("StartStaged %d, raises %d, restores %d: want nothing raised or spawned", gens, raises, restores) |
| 495 | } |
| 496 | if ev := f.b.events(); indexPrefix(ev, "startGen:") >= 0 { |
| 497 | t.Fatalf("events %v: a claims-on swap went down the claims-off spawn", ev) |
| 498 | } |
| 499 | if w.withCalls() != 0 { |
| 500 | t.Fatalf("With calls = %d, want 0 (refused before the admission)", w.withCalls()) |
| 501 | } |
| 502 | if got := f.appSocket(t); got != f.incumbent.Socket { |
| 503 | t.Fatalf("AppSocket = %q, want the incumbent", got) |
| 504 | } |
| 505 | if _, members := f.a.genStateForTest(f.host); len(members) != 0 { |
| 506 | t.Fatalf("members %v left in the operation set", members) |
| 507 | } |
| 508 | }) |
| 509 | } |
| 510 | |
amadan