amadan
Merge remote-tracking branch 'origin/main' into instance-egress-sizes-328-329
77 files changed,
+1583
−96
cmd/moveprobe/main.go+17 −2cmd/moveprobe/main_test.go+47 −11docs/superpowers/plans/2026-10-21-offline-move-runbook.md+894 −0internal/activator/activator_test.go+116 −14internal/activator/admission_ordering_test.go+1 −0internal/activator/backend_exec.go+9 −4internal/activator/backend_exec_test.go+15 −13internal/activator/backend_legacy_test.go+1 −0internal/activator/backend_unit_test.go+2 −0internal/activator/beforeexec_test.go+6 −0internal/activator/childwait_test.go+135 −0internal/activator/claimgen_test.go+4 −0internal/activator/claimloop_linux_test.go+10 −5internal/activator/claimproducer_test.go+1 −0internal/activator/claimretry_instancekey_linux_test.go+1 −0internal/activator/claimwake.go+6 −1internal/activator/cleanup_test.go+1 −0internal/activator/cmd_swap_test.go+6 −0internal/activator/cmd_test.go+10 −0internal/activator/cmdunit.go+2 −2internal/activator/cmdunit_test.go+12 −0internal/activator/config_gate_test.go+3 −0internal/activator/converge_adopted_test.go+5 −0internal/activator/declinedgate_identity_test.go+1 −0internal/activator/exec_gate_test.go+8 −0internal/activator/fenceowner_test.go+2 −0internal/activator/finalreview1_failedwake_test.go+1 −0internal/activator/generations_backend_test.go+1 −0internal/activator/generations_instances_test.go+13 −6internal/activator/gensocket_rev19_test.go+2 −0internal/activator/identity_test.go+1 −0internal/activator/instancekey_delivery_test.go+1 −1internal/activator/instancekey_test.go+14 −16internal/activator/intent_events_test.go+3 −0internal/activator/logsink_test.go+1 −0internal/activator/maintenance_test.go+16 −0internal/activator/opset_test.go+11 −0internal/activator/outcome_fence_test.go+1 −0internal/activator/plan14_generation_workers_test.go+3 −0internal/activator/probewake_test.go+5 −0internal/activator/recoveryprobe_sweep_test.go+4 −0internal/activator/recoveryprobe_test.go+6 −0internal/activator/replica_instancekey_linux_test.go+1 −1internal/activator/replica_instancekey_test.go+4 −10internal/activator/scheduled_token_test.go+1 −0internal/activator/seed_exclusion_test.go+1 −0internal/activator/seed_rollback_stop_test.go+6 −0internal/activator/settle_test.go+1 −0internal/activator/shutdown_lease_test.go+10 −1internal/activator/startup_test.go+1 −0internal/activator/stop_returned_sidecar_test.go+1 −0internal/activator/swap_executed_test.go+1 −0internal/activator/swap_hold_exec_test.go+2 −0internal/activator/swap_hold_test.go+7 −0internal/activator/swap_overlap_test.go+5 −0internal/activator/swap_parent_repair_test.go+2 −0internal/activator/swap_parent_retry_join_test.go+1 −0internal/activator/swap_review_test.go+1 −0internal/activator/swap_schedule_test.go+2 −0internal/activator/swap_sidecar_test.go+12 −0internal/activator/unitproc_gen_test.go+3 −0internal/activator/unitproc_test.go+10 −4internal/activator/wakecheck.go+2 −2internal/activator/wakecheck_test.go+21 −3internal/activator/wakeidentity_route_test.go+2 −0internal/activator/wakeidentity_test.go+14 −0internal/activator/wakeplan_test.go+17 −0internal/activator/wakerecord_recheck_test.go+1 −0internal/activator/worker_gen_test.go+1 −0internal/activator/worker_generations_test.go+5 −0internal/activator/worker_identity_test.go+13 −0internal/activator/worker_respawn_exec_test.go+1 −0internal/activator/worker_retained_pin_test.go+1 −0internal/activator/worker_staged_test.go+2 −0internal/activator/workerswap_test.go+4 −0internal/provision/plan14_root_reads_test.go+18 −0internal/provision/testdata/plan14-root-reads-ci-speed-activator.tsv+11 −0
diff --git a/cmd/moveprobe/main.go b/cmd/moveprobe/main.go| index 9d07ecea8..fd005ee26 100644 |
| --- a/cmd/moveprobe/main.go |
| +++ b/cmd/moveprobe/main.go |
| @@ -86,6 +86,20 @@ const ( |
| modeAfterWrite = "after-write" |
| ) |
| +// workerGraceOverride, set at link time (-ldflags "-X |
| +// main.workerGraceOverride=8s"), replaces workerGrace. Only the tests set |
| +// it, to watch the worker leave after seconds rather than a minute; a build |
| +// that does not set it keeps workerGrace. |
| +var workerGraceOverride string |
| + |
| +// orphanGrace is workerGrace, or the link-time override when one is set. |
| +func orphanGrace() time.Duration { |
| + if d, err := time.ParseDuration(workerGraceOverride); err == nil && d > 0 { |
| + return d |
| + } |
| + return workerGrace |
| +} |
| + |
| func main() { |
| if len(os.Args) > 1 && os.Args[1] == "worker" { |
| os.Exit(runWorker(os.Args[2:])) |
| @@ -708,7 +722,7 @@ func (a *app) checkWorker() error { |
| } |
| // runWorker writes a worker row every second. Once its parent has gone it |
| -// keeps writing for workerGrace and then exits. |
| +// keeps writing for workerGrace (orphanGrace) and then exits. |
| func runWorker(args []string) int { |
| fl := flag.NewFlagSet("moveprobe worker", flag.ContinueOnError) |
| dbPath := fl.String("db", "", "the instance database") |
| @@ -737,6 +751,7 @@ func runWorker(args []string) int { |
| return 1 |
| } |
| defer db.Close() |
| + grace := orphanGrace() |
| var orphaned time.Time |
| t := time.NewTicker(workerEvery) |
| defer t.Stop() |
| @@ -744,7 +759,7 @@ func runWorker(args []string) int { |
| if orphaned.IsZero() && os.Getppid() != *parent { |
| orphaned = time.Now() |
| } |
| - if !orphaned.IsZero() && time.Since(orphaned) >= workerGrace { |
| + if !orphaned.IsZero() && time.Since(orphaned) >= grace { |
| return 0 |
| } |
| report(insertSeq(db, "worker", here)) |
diff --git a/cmd/moveprobe/main_test.go b/cmd/moveprobe/main_test.go| index 693ecda3c..e64ffb336 100644 |
| --- a/cmd/moveprobe/main_test.go |
| +++ b/cmd/moveprobe/main_test.go |
| @@ -93,6 +93,8 @@ func shortTempDir(t *testing.T) string { |
| // fixture is one instance's world: its data directory (the flat layout, |
| // <dir>/h1.db and <dir>/h1/), a test S3 holding its store, and a control |
| // socket served by the platform's own schedule verbs over a registry. |
| +// Each test makes its own, and shares nothing else but the read-only |
| +// binPath, so every test runs in parallel. |
| type fixture struct { |
| t *testing.T |
| sockDir string |
| @@ -252,10 +254,16 @@ type probe struct { |
| // exec.Command(bin, "--socket", sock, "--db", dbPath) with the composed env. |
| // Its stdout and stderr are pipes, as the backend's tee makes them. |
| func (f *fixture) start(env map[string]string) *probe { |
| + f.t.Helper() |
| + return f.startBin(binPath, env) |
| +} |
| + |
| +// startBin is start with another build of moveprobe. |
| +func (f *fixture) startBin(bin string, env map[string]string) *probe { |
| f.t.Helper() |
| f.starts++ |
| p := &probe{t: f.t, sock: filepath.Join(f.sockDir, fmt.Sprintf("a%d.sock", f.starts)), out: &lockedBuffer{}, done: make(chan struct{})} |
| - p.cmd = exec.Command(binPath, "--socket", p.sock, "--db", f.db) |
| + p.cmd = exec.Command(bin, "--socket", p.sock, "--db", f.db) |
| p.cmd.Env = childEnv(env) |
| p.cmd.Stdout = p.out |
| p.cmd.Stderr = p.out |
| @@ -405,8 +413,9 @@ func (p *probe) report() reportDoc { |
| return r |
| } |
| -// workerPIDs lists the pids whose command line starts with the test binary |
| -// and "worker" and names db. |
| +// workerPIDs lists the pids whose command line starts with a moveprobe |
| +// build the tests made (binPath, or a test's own, all named moveprobe*) and |
| +// "worker", and names db. |
| func workerPIDs(db string) []int { |
| var pids []int |
| ents, _ := os.ReadDir("/proc") |
| @@ -420,7 +429,7 @@ func workerPIDs(db string) []int { |
| continue |
| } |
| args := strings.Split(strings.TrimRight(string(raw), "\x00"), "\x00") |
| - if len(args) < 2 || args[0] != binPath || args[1] != "worker" || !strings.Contains(string(raw), db) { |
| + if len(args) < 2 || !strings.HasPrefix(filepath.Base(args[0]), "moveprobe") || args[1] != "worker" || !strings.Contains(string(raw), db) { |
| continue |
| } |
| if alive(pid) { |
| @@ -540,6 +549,7 @@ func (p *probe) fire(tm registry.Timer, token string) (int, string) { |
| // ---- the launch contract -------------------------------------------------- |
| func TestMoveprobeLaunchContract(t *testing.T) { |
| + t.Parallel() |
| t.Run("the agent's variables and a store endpoint", func(t *testing.T) { |
| f := newFixture(t) |
| p := f.start(f.env) |
| @@ -765,10 +775,23 @@ func (o *oneConn) Addr() net.Addr { return o.c.LocalAddr() } |
| // ---- the forked worker ------------------------------------------------------ |
| +// The worker's grace is a minute in every build but this test's: it links |
| +// its own moveprobe with an 8 s grace (workerGraceOverride) and watches |
| +// that build's worker outlive its main, keep writing, and leave on its own |
| +// at the grace, as the shipped build does at a minute. |
| func TestMoveprobeWorkerOutlivesMain(t *testing.T) { |
| - t.Parallel() // a minute of waiting for the worker's own exit |
| + t.Parallel() |
| + if workerGrace != 60*time.Second || workerGraceOverride != "" || orphanGrace() != workerGrace { |
| + t.Fatalf("a plain build's worker grace is %s (const %s, override %q), want a minute", orphanGrace(), workerGrace, workerGraceOverride) |
| + } |
| + const grace = 8 * time.Second |
| + bin := filepath.Join(t.TempDir(), "moveprobe-grace") |
| + build := exec.Command("go", "build", "-ldflags", "-X main.workerGraceOverride="+grace.String(), "-o", bin, ".") |
| + if out, err := build.CombinedOutput(); err != nil { |
| + t.Fatalf("build moveprobe with an %s grace: %v\n%s", grace, err, out) |
| + } |
| f := newFixture(t) |
| - p := f.start(f.env) |
| + p := f.startBin(bin, f.env) |
| p.wait("/ready", http.StatusOK) |
| r := p.report() |
| if r.WorkerPID <= 0 || !alive(r.WorkerPID) { |
| @@ -798,22 +821,27 @@ func TestMoveprobeWorkerOutlivesMain(t *testing.T) { |
| t.Fatalf("the worker stopped writing after the main died: %d then %d rows", n0, n1) |
| } |
| - // It exits on its own about a minute after its parent has gone. |
| - for alive(r.WorkerPID) && time.Since(killed) < 80*time.Second { |
| - time.Sleep(250 * time.Millisecond) |
| + // It exits on its own about its grace after its parent has gone. |
| + for alive(r.WorkerPID) && time.Since(killed) < grace+20*time.Second { |
| + time.Sleep(50 * time.Millisecond) |
| } |
| gone := time.Since(killed) |
| if alive(r.WorkerPID) { |
| t.Fatalf("the worker is still alive %s after the main died", gone.Round(time.Second)) |
| } |
| - if gone < 59*time.Second || gone > 70*time.Second { |
| - t.Errorf("the worker exited %s after the main died, want about 60s", gone.Round(time.Second)) |
| + // The worker notices its parent has gone on its next 1 s tick and |
| + // leaves on the first tick at or past the grace, so it exits between |
| + // the grace and about 2 s after it. 4 s of slack still refuses a worker |
| + // that waits twice its grace. |
| + if gone < grace-time.Second || gone > grace+4*time.Second { |
| + t.Errorf("the worker exited %s after the main died, want about %s", gone.Round(time.Second), grace) |
| } |
| } |
| // ---- the failure switch --------------------------------------------------- |
| func TestReadyFailsWhereTold(t *testing.T) { |
| + t.Parallel() |
| f := newFixture(t) |
| // The installed state: a normal boot in uk, stopped, its worker killed |
| @@ -900,6 +928,7 @@ func TestReadyFailsWhereTold(t *testing.T) { |
| // ---- the report ----------------------------------------------------------- |
| func TestReportCountsRowsByRegion(t *testing.T) { |
| + t.Parallel() |
| f := newFixture(t) |
| uk := f.start(f.env) |
| uk.wait("/ready", http.StatusOK) |
| @@ -991,6 +1020,7 @@ func TestReportCountsRowsByRegion(t *testing.T) { |
| // ---- the instance key ----------------------------------------------------- |
| func TestReadyChecksBothKeys(t *testing.T) { |
| + t.Parallel() |
| f := newFixture(t) |
| p := f.start(f.env) |
| p.wait("/ready", http.StatusOK) |
| @@ -1061,6 +1091,7 @@ func allowInserts(t *testing.T, db, kind string) { |
| // TestReadyFollowsTheWorkersLastWrite: /ready is 200 only once a worker row |
| // has committed since this start and the worker's latest write succeeded. |
| func TestReadyFollowsTheWorkersLastWrite(t *testing.T) { |
| + t.Parallel() |
| f := newFixture(t) |
| seed := f.start(f.env) |
| seed.wait("/ready", http.StatusOK) |
| @@ -1113,6 +1144,7 @@ func TestReadyFollowsTheWorkersLastWrite(t *testing.T) { |
| } |
| func TestAfterWriteExitsWhenItsRowFails(t *testing.T) { |
| + t.Parallel() |
| f := newFixture(t) |
| seed := f.start(f.env) |
| seed.wait("/ready", http.StatusOK) |
| @@ -1197,6 +1229,7 @@ func (r *failRows) Next(dest []driver.Value) error { |
| func init() { sql.Register("moveprobe-fail", failDriver{}) } |
| func TestReportNeverAnswersPartial(t *testing.T) { |
| + t.Parallel() |
| report := func(t *testing.T, db *sql.DB) (int, string) { |
| t.Helper() |
| a := &app{db: db, filesDir: t.TempDir(), region: "uk", mode: modeNormal, storeErr: errors.New("store: not under test")} |
| @@ -1237,6 +1270,7 @@ func TestReportNeverAnswersPartial(t *testing.T) { |
| } |
| func TestArmOnceReusesOneControlConnection(t *testing.T) { |
| + t.Parallel() |
| f := newFixture(t) |
| p := f.start(f.env) |
| p.wait("/ready", http.StatusOK) |
| @@ -1254,6 +1288,7 @@ func TestArmOnceReusesOneControlConnection(t *testing.T) { |
| // TestReadyFailsWhenTheWorkerStops: a worker that has died, or has sent no |
| // status for 3 s, fails readiness, and the report says so. |
| func TestReadyFailsWhenTheWorkerStops(t *testing.T) { |
| + t.Parallel() |
| t.Run("the worker dies after its first good write", func(t *testing.T) { |
| f := newFixture(t) |
| p := f.start(f.env) |
| @@ -1301,6 +1336,7 @@ func TestReadyFailsWhenTheWorkerStops(t *testing.T) { |
| // store check per 5 s, however many requests arrive, and requests that land |
| // while a check is running share its result. |
| func TestReadyCoalescesTheStoreCheck(t *testing.T) { |
| + t.Parallel() |
| f := newFixture(t) |
| p := f.start(f.env) |
| p.wait("/ready", http.StatusOK) |
diff --git a/docs/superpowers/plans/2026-10-21-offline-move-runbook.md b/docs/superpowers/plans/2026-10-21-offline-move-runbook.md| new file mode 100644 |
| index 000000000..c31df766c |
| --- /dev/null |
| +++ b/docs/superpowers/plans/2026-10-21-offline-move-runbook.md |
| @@ -0,0 +1,894 @@ |
| +# Offline move runbook: moving one instance to another region with `carlos move offline` (v0.28.7 and later) |
| + |
| +This is the operator runbook for moving one instance from one region to another with planned downtime, using the pre-copy (`stage`) that v0.28.7 added. It was rewritten on 2026-10-21 from what the first real moves taught. Its commands are the ones that worked on dev and on the flagship on 2026-10-20 and 2026-10-21, with the instance names replaced by placeholders. |
| + |
| +The file runs nothing and authorizes nothing. Moving a customer's instance needs Paul's go for that instance, and anything that writes to a box as root needs its own approval (AGENTS.md, "Production infrastructure needs scoped approval"). A step that writes says so in a line that starts **Go:**. Everything else is read-only. |
| + |
| +Sources: harness `RUNLOG.md` Runs 9 and 10 with corrections M2 to M15; the dev rehearsals on v0.28.6 and v0.28.7; the flagship moves of 2026-10-21, without and with `stage` (all in the session's `own/after236/` notes). The history at the end says what happened when. |
| + |
| +## The short version |
| + |
| +1. Check the fleet once (section 2) and the instance and the target box before each move (section 3). The target check is the one that matters most: a stale `/opt/carlos/live/<host>` link on the target makes `flip` fail only after the instance is already offline. |
| +2. Run `stage` while the instance serves, and run it again until a round copies almost nothing (section 4). |
| +3. Run `move.sh`. It stages once more, records a baseline, then runs `start`, `seal`, `flip`, a check of the target and `open` back to back, judging each step by its exit code. Before `open`, any surprise aborts the move. Expect about 2.5 to 3 minutes offline for an instance with a 1.6 to 3.7 GB replica (section 5). |
| +4. Measure the offline time yourself, from just before `start` to the first 2xx through the target. The console's figure read 40 to 75 seconds low on the flagship (section 6). |
| +5. Verify on the target with the row fingerprint (section 7), then run `finish` (section 8). |
| +6. About an hour after `open`, the target stops the instance briefly to nest it. Check it (section 9). |
| +7. Once everything checks out, delete the source region's copy of the replica by exact key, keeping `lock.json` (section 10). |
| +8. Before `open`, a problem means `abort`, and nothing is lost. After `open` there is no move back: fix forward on the target (section 11). |
| + |
| +Write every step to the log as it runs, and after any restart, read `carlos move offline status` before anything else (section 1). |
| + |
| +## Contents |
| + |
| +- 0. Before you start |
| +- 1. Logging discipline |
| +- 2. Fleet pre-checks, once per session |
| +- 3. Per-instance pre-checks, including the target box |
| +- 4. Staging |
| +- 5. The offline doors |
| +- 6. Measuring the downtime |
| +- 7. Verification |
| +- 8. finish |
| +- 9. The nesting stop, about an hour after open |
| +- 10. Replica cleanup |
| +- 11. Rollback |
| +- 12. Known gaps |
| +- 13. Rehearsing on dev |
| +- 14. Telling the customer |
| +- Appendix A. Setup and helper scripts |
| +- History |
| + |
| +--- |
| + |
| +## 0. Before you start |
| + |
| +**Placeholders.** `shop.example.com` is the instance's host, `<acct>` its account sqid, `<app>` its app. The source is "the IE box" (the home region, record region empty, console name `ie`). The target is "the UK box" (region `uk`). Swap the values in `env.sh` (Appendix A) for another pair of regions. |
| + |
| +**What has to be true.** |
| + |
| +- Every box in both regions and the console run v0.28.7 or later. `stage` refuses while any box in either region lacks the `offline-stage-1` capability. |
| +- Your login is a system operator on the console and a member of the instance's account. The doors check both and answer a byte-identical 404 otherwise. |
| +- Paul has given his go for this instance, and the customer has been told the window (section 14). |
| +- No other instance in the deployment is staged or moving. There is one offline-move lock per deployment, and a staged move holds it. Stage, move and finish one instance, then start on the next. |
| +- Nobody promotes the app from the first `stage` until `finish`. The landing's `release` item compares the build the seal recorded with what the target holds. |
| +- `carlos move` refuses to run as root. Run it as yourself on DOCS-T1. |
| + |
| +**Getting the fleet onto the release.** That is the release recipe in the README ("Releasing the platform") and each release's operator notes, not this file. Read the notes for every release between the fleet's version and the one you roll: [v0.28.5](https://github.com/carlosframework/releases/releases/tag/v0.28.5), [v0.28.6](https://github.com/carlosframework/releases/releases/tag/v0.28.6), [v0.28.7](https://github.com/carlosframework/releases/releases/tag/v0.28.7). The parts that matter for moves: |
| + |
| +- Roll dev first, then flagship. Boxes before their console. |
| +- From v0.28.5 on, every box takes the next release through its ordinary `carlos system update`, armed or not. The hand re-arm of the v0.28.5 roll is history. |
| +- A guard-armed console box (dev's) upgrades its console with `carlos system claims install-console --upgrade --from <sha256 of the running console>` as root, after the box's router is on the new release (the v0.28.6 notes). The flagship console is unguarded and rolls by ship and promote. |
| +- After the roll, check that every box's edges file lists both `offline-move-1` and `offline-stage-1` (2.2). |
| + |
| +**Set up** the scratch directory, `env.sh` and the helpers in Appendix A before anything else. |
| + |
| +--- |
| + |
| +## 1. Logging discipline |
| + |
| +On 2026-10-21 a session restart killed a move script in the middle of a move. Its log wrapper wrote a step's output only when the step finished, so the log showed nothing, and the resumed session believed the move had not started. The instance had been started and sealed, and stayed offline for 12.7 minutes until someone read `status` and aborted it. Nothing was lost, but the outage was avoidable. |
| + |
| +The rules: |
| + |
| +1. **Write each step to the log as it runs**, not when it ends. `L` in Appendix A pipes output through `tee -a` line by line, and `move.sh` and `stage.sh` print as they go. |
| +2. **Before each door, log a line `ABOUT TO RUN <door> <host> <UTC time>`.** `move.sh` and `stage.sh` do it. If you run a door by hand, write the line first. |
| +3. **After any restart** (your session, the agent, a lost terminal), run `MV status` for every host that might be in play **before** reading the log or doing anything else. "No move is in flight", "is done" and "is aborted" are safe. Anything else is a live move: decide from `status`, not from the log. |
| +4. Write a short note into the log before each block saying what you are about to do and what you expect. |
| +5. Never print a secret. The helpers print names, sha256 prefixes and counts only. |
| + |
| +--- |
| + |
| +## 2. Fleet pre-checks, once per session |
| + |
| +All read-only. Run them through `L` so they land in the log. |
| + |
| +### 2.1 Versions and the doors |
| + |
| +```bash |
| +carlos version; sha256sum "$(command -v carlos)" | cut -c1-16 |
| +curl -sS -o /dev/null -D - "$C/healthz" | grep -i -E '^HTTP|x-carlos-version' |
| +carlos system status --console "$C" |
| +MV status |
| +``` |
| + |
| +Pass: the CLI is v0.28.7 or later; the console answers 200; every box reads `current` on the release; `MV status` answers "No move is in flight for shop.example.com." (or the outcome of an earlier move), not a 404. A 404 means an older console or that you are not an operator and member. |
| + |
| +### 2.2 Edges files: capabilities, buckets, secrets key |
| + |
| +```bash |
| +aws --profile "$PROFILE" s3 ls "s3://$DEPB/control/edges/" |
| +for k in $(aws --profile "$PROFILE" s3 ls "s3://$DEPB/control/edges/" | awk '{print $4}'); do |
| + echo "== $k"; aws --profile "$PROFILE" s3 cp --quiet "s3://$DEPB/control/edges/$k" - | jq -c '{region, placement, caps: .capabilities, secrets_key_fp: (.secrets_key_fp // "" | .[0:20]), data_bucket, data_bucket_region, hibernation_configured, hosts: (.hosts | length)}' |
| +done |
| +``` |
| + |
| +Pass: |
| + |
| +- one file per live box, and no file for a box that no longer exists (a stale file naming the home placement blocks `seal`; removing it needs Paul); |
| +- every box in both regions lists `offline-move-1` **and** `offline-stage-1`; |
| +- the target's `placement` is the region you move to, and it has a `data_bucket` (that is `$TGTDATA`); |
| +- `hibernation_configured` true; |
| +- the target's `secrets_key_fp` equals the source's. The start door checks only the target's fingerprint against the console. A box with no `CARLOS_SECRETS_KEY` publishes no fingerprint at all; on dev, leo-climbing is such a box, and it is not a blocker unless it is the target. |
| + |
| +### 2.3 The boxes and the console |
| + |
| +```bash |
| +src "$(cat "$W/block.box.sh")" |
| +tgt "$(cat "$W/block.box.sh")" |
| +src 'P=$(systemctl show -p MainPID --value console.service 2>/dev/null); [ -n "$P" ] && [ "$P" != 0 ] && { tr "\0" "\n" < /proc/$P/environ | grep -E "^CARLOS_(HOST_CLAIM|MOVES_PHASE1)=" ; echo "console exe sha16 $(sha256sum < /proc/$P/exe | cut -c1-16)"; } || echo "no console on this box"' |
| +``` |
| + |
| +Pass on both boxes: `claim lines: 0`, `phase1 lines: 0`, `edge env claim/phase1 vars: 0`; `User=carlos` (any other value is a STOP: the move reads bucket and tenant-reachable inputs, so it never runs where the agent runs as root); `Delegate=yes`; `ActiveState=active`; `stage.path: active enabled`; the release version; the same `secrets key sha8` on both. An armed guard is allowed. The console, if it runs on the source box, shows no `CARLOS_HOST_CLAIM` value and no `CARLOS_MOVES_PHASE1`. Check every other box in the source region the same way: `start` waits for every box in the source region to confirm it has stopped the host. |
| + |
| +### 2.4 The staging lane |
| + |
| +The source box's role must be able to write into the target data bucket's `moves/` prefix. The boxes have no `aws` CLI, so simulate the policy from DOCS-T1: |
| + |
| +```bash |
| +P=$(aws --profile "$PROFILE" --region "$SRC_REGION" ec2 describe-instances --instance-ids "$SRC_ID" --query 'Reservations[0].Instances[0].IamInstanceProfile.Arn' --output text) |
| +R=$(aws --profile "$PROFILE" iam get-instance-profile --instance-profile-name "${P##*/}" --query 'InstanceProfile.Roles[0].Arn' --output text); echo "role $R" |
| +aws --profile "$PROFILE" iam simulate-principal-policy --policy-source-arn "$R" --action-names s3:PutObject s3:GetObject s3:DeleteObject --resource-arns "arn:aws:s3:::$TGTDATA/moves/x/y" --query 'EvaluationResults[].[EvalActionName,EvalDecision]' --output text |
| +aws --profile "$PROFILE" iam simulate-principal-policy --policy-source-arn "$R" --action-names s3:ListBucket --resource-arns "arn:aws:s3:::$TGTDATA" --context-entries ContextKeyName=s3:prefix,ContextKeyValues=moves/,ContextKeyType=string --query 'EvaluationResults[].[EvalActionName,EvalDecision]' --output text |
| +``` |
| + |
| +Pass: `PutObject allowed`, `DeleteObject allowed`, `ListBucket allowed`, and **`GetObject implicitDeny`**. The deny is the designed grant (Put and Delete on `moves/*`, a `moves/`-conditioned List, never `instances/*` and never Get). `implicitDeny` on Put, Delete or List means the lane is missing: STOP, it is an infra change. |
| + |
| +### 2.5 Leftovers in the control plane |
| + |
| +```bash |
| +aws --profile "$PROFILE" s3api head-object --bucket "$DEPB" --key control/hosts-backfill.json >/dev/null 2>&1 && echo "hosts-backfill PRESENT" || echo "control/hosts-backfill.json absent" |
| +carlos status --layout --console "$C" | grep -E "$H|WARNING" |
| +``` |
| + |
| +Pass: `absent`. The layout row for the host is `nested` on the source. A warning that the source box "runs nested instances beside flat, foreign or external ones" is about the box, not this move, and does not block it. |
| + |
| +--- |
| + |
| +## 3. Per-instance pre-checks |
| + |
| +### 3.1 The record and the doors |
| + |
| +```bash |
| +MV status |
| +aws --profile "$PROFILE" s3 cp --quiet "s3://$DEPB/apps/$ACCT/$APP/instances/$H.json" - | jq -c . |
| +aws --profile "$PROFILE" s3api head-object --bucket "$DEPB" --key "control/hosts/$H.json" >/dev/null 2>&1 && echo "control/hosts/$H.json PRESENT" || echo "control/hosts/$H.json absent" |
| +aws --profile "$PROFILE" s3 ls "s3://$DEPB/control/certs/" | grep -c "$H" |
| +carlos schedule ls --app "$APP" --account "$ACCT" --console "$C" |
| +``` |
| + |
| +Pass: no move in flight; the record has no seed, replica or replicas, steering, unit, fleet, pool or open run (the door refuses those classes, so a surprise is a STOP, not a workaround); `control/hosts/<host>.json` absent. For certificates, either an object under `control/certs/` for the host or a wildcard that covers it; confirm TLS through the target before the move with `curl -sS -o /dev/null -w '%{http_code} ssl_verify=%{ssl_verify_result}\n' --resolve $H:443:$TGT_IP https://$H$PROBE_PATH` (any HTTP code is fine, `ssl_verify=0` is the pass). `schedule ls` names the timers that must arrive on the target. A schedule that "fires more often than the idle window" means the instance never sleeps, so plan for the nesting stop in section 9. |
| + |
| +### 3.2 Sizes on the source |
| + |
| +```bash |
| +src "set -- $H; $(cat "$W/sizes.box.sh")" |
| +aws --profile "$PROFILE" s3 ls --recursive --summarize "s3://$SRCDATA/instances/$H/" | tail -2 |
| +``` |
| + |
| +Pass: the files tree is under 268435456 bytes (256 MiB; over that the seal refuses safely); no sidecar file. Record the database size and the replica's object count and total size: staging time and the size of the target's copy scale with the replica, not the database. A legacy flat `/data/<host>/` directory holding old key copies needs nothing: the seal passes and the retire parks it with the instance. |
| + |
| +### 3.3 The target box |
| + |
| +This check exists because of the first staged move on the flagship. An earlier attempt had left a root-owned link `/opt/carlos/live/<host>` on the UK box, naming a build from weeks before. Nothing reports it before `flip`, and `flip` fails the landing's `release` item only after `start` has taken the instance offline: `flip` exited 4 with "uk could not take shop.example.com: the target box still holds an older build of this instance", the move was aborted, and the instance was offline for about two minutes for nothing (amadan #339). |
| + |
| +```bash |
| +tgt "set -- $H; $(cat "$W/target.box.sh")" |
| +src "readlink /opt/carlos/live/$H" |
| +curl -sS -m 30 -D - -o /dev/null --resolve "$H:443:$SRC_IP" "https://$H$PROBE_PATH" | grep -i -E '^HTTP|x-carlos-version' |
| +for p in instances moves; do echo "target $p/$H/ current objects: $(aws --profile "$PROFILE" s3 ls --recursive "s3://$TGTDATA/$p/$H/" | wc -l)"; done |
| +echo "target lock: $(aws --profile "$PROFILE" s3 cp --quiet "s3://$TGTDATA/instances/$H/lock.json" - 2>&1 | head -c 200)" |
| +tgt "df -B1 --output=avail,pcent / | tail -1" |
| +``` |
| + |
| +Pass, every item: |
| + |
| +- **Live link:** `live link: absent` on the target. If it is present and names a different build from the one the source's `readlink` shows (and the `x-carlos-version` header confirms), STOP before staging. Clearing it is a root action on the target box and needs Paul's approval; the recipe is below. |
| +- **`/data` leftovers:** every path `absent`: `/data/instances/<host>`, `/data/.instances/<host>`, `/data/<host>`, `/data/programs/<host>.json`, `/data/<host>.env`, `/data/<host>.config.env`, `/data/<host>.db`. |
| +- **The route:** `registry routes [] timers 0`. |
| +- **A parked directory:** `/data/.offline-moves/<host>` absent, or holding only `<move-id>/target/` from an earlier aborted move. That one does not block: the retry on 2026-10-21 passed with it in place. Anything else under it is a STOP. |
| +- **The UK bucket prefixes:** 0 current objects under `instances/<host>/` and `moves/<host>/` in the target data bucket, and no `lock.json`. Versions and delete markers left by an earlier aborted move are fine. |
| +- **Disk:** at least twice the replica's total size free on the target. It holds the staged copy and then the live replica. |
| +- **Capabilities and switches, again for this move:** run 2.2 and 2.3 for the source and the target before each instance, not only once per session. Both boxes list `offline-move-1` and `offline-stage-1`, and neither has a claim or Phase 1 switch (`claim lines: 0`, `phase1 lines: 0`, `edge env claim/phase1 vars: 0`). |
| + |
| +**Go (Paul, root on the target box): clearing a stale live link.** Only when the link names a build the instance does not run. Confirm that no other link names the same version directory and no process runs from it, then remove the link and that version directory by exact path, never a glob: |
| + |
| +```bash |
| +tgt "ls -la /opt/carlos/live/ | grep -F '/opt/carlos/versions/<old-version>/' ; grep -l '/opt/carlos/versions/<old-version>/' /proc/[0-9]*/maps 2>/dev/null | head -3; du -sh /opt/carlos/versions/<old-version>" |
| +tgt "rm -- /opt/carlos/live/$H && rm -r -- /opt/carlos/versions/<old-version> && echo removed" |
| +``` |
| + |
| +Then run 3.3 again. |
| + |
| +### 3.4 Baseline and the row fingerprint |
| + |
| +```bash |
| +src "set -- $H; $(cat "$W/facts.box.sh")" |
| +tgt "set -- $H; $(cat "$W/facts.box.sh")" |
| +echo "source lock: $(aws --profile "$PROFILE" s3 cp --quiet "s3://$SRCDATA/instances/$H/lock.json" - | jq -c .)" |
| +src "set -- $H; $(cat "$W/schema.box.sh")" |
| +``` |
| + |
| +`facts` on the source prints the env file's sha, the instance key's digest, `config.env`'s sha and names, the files tree's count and digest, the database, the row, the timers and anything parked. On the target it must print `env file: absent`, `row: []` and `timers: []`. |
| + |
| +The row fingerprint compares the database before and after. `schema.box.sh` lists each table with its row count and time columns. Pick two or three tables the app only appends to (an events or audit log, for example), with plenty of rows. A table whose old rows the app updates will show a different digest with nothing wrong, so leave those out. Then read their highest rowids: |
| + |
| +```bash |
| +src "set -- $H <table1> <table2>; $(cat "$W/rows.box.sh")" |
| +``` |
| + |
| +and set `ROWS_ARGS` in `env.sh` to `"<table1>:<max_rowid1> <table2>:<max_rowid2>"`. With a limit, `rows.box.sh` also prints a digest of every row up to that rowid, which must be identical on the target. It never prints row content. |
| + |
| +--- |
| + |
| +## 4. Staging |
| + |
| +`stage` copies the instance's replica into the target data bucket's `moves/<host>/` prefix while the instance keeps serving. Each run copies only what changed since the last. On the flagship, the first round copied 1.6 GB in 76 to 79 seconds and 3.7 GB in 142 seconds, with no failed requests through the source while it ran. Later rounds took 15 to 18 seconds. |
| + |
| +**Run it until it has caught up.** |
| + |
| +```bash |
| +L "stage 1" "bash $W/stage.sh" |
| +L "stage 2" "bash $W/stage.sh" |
| +``` |
| + |
| +`stage.sh` logs `ABOUT TO RUN stage`, runs the door (it waits for the round to finish), then prints `status`, the record's `move` and the stager's report. Caught up means `status` says "Staged n of n objects" with the two numbers equal, and the stager report's `round_bytes` is small (a few KB). The first round of a large replica can end at "Staged 237 of 240" because Litestream compacts while it copies; the next round closes the gap. |
| + |
| +Where to read what a round did: the stager report `control/offline-moves/<host>/<move-id>/stager/<box>.json` (`round_objects`, `round_bytes`, `staged_objects`, `replica_objects`) and the source journal line "staged round: N object(s) changed, B bytes copied; n of m objects staged". `status` shows only the totals. `round_objects` counts removals too: a round that copies 2 KB can report 124 objects changed when stale compacted objects are dropped from the staged prefix. |
| + |
| +**Run it once more right before `start`.** Litestream writes a new object about every 10 seconds while the instance takes writes, and whatever is new at `start` is copied by the seal while the instance is offline. `move.sh` does this final stage itself, about 25 seconds before `start`. On the flagship the seal then copied 0 to 2 objects. |
| + |
| +While a move is staged: |
| + |
| +- the instance serves as normal; |
| +- it refuses `carlos run` and deletion; |
| +- no other instance in the deployment can stage or move offline; |
| +- `abort` removes the staged copy and ends the move with the instance never offline. `abort` and the `finish` after it both exit 3, the normal "aborted" code, with "The staged copy of shop.example.com is removed." and "shop.example.com is back in ie and serving. Nothing was lost." Afterwards `status` prints "Moving shop.example.com from ie to uk. Moving shop.example.com back to ie." and a list of items "waiting". Ignore that; the abort's own lines are the record (#339). |
| + |
| +--- |
| + |
| +## 5. The offline doors |
| + |
| +**Go (Paul): this instance is offline from `start` until it first answers on the target.** |
| + |
| +```bash |
| +L "move" "bash $W/move.sh 1200" |
| +``` |
| + |
| +What `move.sh` does, in order (Appendix A has the script): |
| + |
| +1. **Final stage**, as in section 4. |
| +2. **Baseline** on the source: `facts`, kept for the gate, and the row fingerprint with `ROWS_ARGS`. If the baseline is incomplete it stops with nothing offline. |
| +3. **T0**, then the doors back to back, each preceded by an `ABOUT TO RUN` line: |
| + - `start --to uk` takes the instance offline and waits until every box in the source region confirms it stopped it; |
| + - `seal` copies whatever is newer than the last stage and checks the copy; |
| + - the source slice's process count (it should be 0); |
| + - `flip` stops the instance in the source region for good and lands it on the target; |
| + - **the gate**: `facts` on the target must show the row `landing` in `uk` at `/data/<host>.db` (flat), and the same env file sha and instance key digest, the same `config.env` sha, the same files count and digest and the same timer names as the baseline; |
| + - `open` serves it from the target. |
| +4. **The probe**: requests `$PROBE_PATH` through the target every 2 seconds until the first 2xx, prints the offline seconds since T0, then checks the host through the source box and through DNS, and prints `status`. |
| + |
| +**Judge each door by its exit code, never by its text.** From `docs/site/cli.md`: `0` done, `3` rolled back or aborted, `4` needs an operator (stuck, waiting for a box, or the landing failed an item), `5` refused, `1` the console could not be reached. Several successful doors print lines that read like trouble: |
| + |
| +- `start` prints "Waiting for <box> in ie to confirm it has stopped shop.example.com." before it succeeds; |
| +- `open` starts with "this move is offline-open. run carlos move offline status ... to see what it waits for", then "Waiting for <box> to clean up", and exits 0; |
| +- after `open`, `status` may say "Its report is not current" for every box. From `open` on, no door reads those reports. |
| + |
| +The one wait the script retries is "the report from <box> is not current" (exit 5, MV-O178): it tries again every 15 seconds, up to 8 times. It is never a process. |
| + |
| +**Before `open`, any other failure aborts.** If `seal`, `flip` or the gate fails, `move.sh` logs `STOP`, runs `abort` (expect exit 3) and `finish` (expect exit 3, "Nothing was lost"), prints `status` and stops. If `start` itself fails, it stops and leaves the decision to you: read `status`, and if a move is in flight, abort it the same way. If `open` fails, it stops without aborting; read `status` (section 11). |
| + |
| +**What it looks like** on the flagship, v0.28.7, staged (UTC seconds, all exit 0): |
| + |
| +| Step | 1.6 GB replica, 305 objects | 3.7 GB replica, 270 objects | |
| +|---|---|---| |
| +| final stage | 15.8 | 18.5 | |
| +| start | 23.0 | 23.3 | |
| +| seal | 28.4 (copied 2 objects, 487 bytes) | 30.6 (copied 0) | |
| +| flip | 44.2 | 57.7 | |
| +| gate (two SSM reads) | about 2 | about 3 | |
| +| open | 7.3 | 15.9 | |
| +| open returned to first 2xx through the target | 40.2 | 57.0 | |
| +| of which: promoting the staged objects into the live replica and restoring | 29 | 52 | |
| +| **offline, client-measured** | **147.6** | **190.4** | |
| +| console's `offline_s` | 99.9 | 117.2 | |
| + |
| +So: **about 2.5 to 3 minutes offline at 1.6 to 3.7 GB.** Without `stage` (v0.28.6), the seal of the same 1.6 GB replica took about 6 minutes on its own, and a 27 MB instance with 288 replica objects was offline 164 seconds. On dev, an instance of under a megabyte is offline 30 to 40 seconds. |
| + |
| +**After `open`, expect the target to be slow to answer, and do not abort** (abort is refused after `open` anyway). The first request wakes the instance, and the wake first copies every staged object from `moves/<host>/` into `instances/<host>/` in the target bucket (the journal says "promoting 270 staged object(s) from moves/shop.example.com/ (residence move landing)") and then restores the database from it. That is 30 to 50 seconds at 1.6 to 3.7 GB, and it sits inside the offline window: `stage` does not shorten it. Meanwhile the target answers 503 at the 30 second wake bound, and the probe's 10 second requests time out (`000`). For the first 10 to 15 seconds after `open` the target may also answer 404, because it does not route the host until the row is ready. Requests that reach a source-region edge can 404 for up to about 20 seconds after the target first serves, until the target's published host list reaches that edge. |
| + |
| +--- |
| + |
| +## 6. Measuring the downtime |
| + |
| +Report the offline time measured from the client: T0 (logged just before `start`) to the first 2xx through the target. `move.sh` prints it as "FIRST 2xx through the target at <time>: offline <n> s since start". |
| + |
| +Do not report the console's figure. `last_move.offline_s` on the record, and the "It was offline for N seconds" that `finish` prints, both end at `open`, so they leave out the first wake. On 2026-10-21 they read 42 to 73 seconds low (99.9 against 147.6, 117.2 against 190.4, 122.5 against 164.1). |
| + |
| +Two more numbers worth recording for each move: |
| + |
| +- **What each stage round copied**: from the stager report and journal (section 4). |
| +- **What the seal copied**, which no output reports: list the object versions written into the target's `moves/<host>/` during the seal window. Take the window from the `seal` line in the log (seconds are enough): |
| + |
| + ```bash |
| + bash "$W/sealcount.sh" 2026-10-21T15:34:38 2026-10-21T15:35:06 |
| + ``` |
| + |
| + The versions survive the promotion, so this works any time after the move. |
| + |
| +--- |
| + |
| +## 7. Verification |
| + |
| +Run these after the probe has had its first 2xx, before `finish`. Every item must pass. |
| + |
| +```bash |
| +tgt "set -- $H; $(cat "$W/facts.box.sh")" |
| +tgt "set -- $H $ROWS_ARGS; $(cat "$W/rows.box.sh")" |
| +tgt "python3 -c \"import sqlite3; c=sqlite3.connect('file:/data/$H.db?mode=ro',uri=True); print('quick_check', c.execute('pragma quick_check').fetchone()[0])\"" |
| +tgt "journalctl -u carlos-edge --since '-15 min' --no-pager | grep -F $H | grep -v 'replica sync' | grep -i -E 'landed|opened|promot|restor|layout|woke|config|refused|extracted' | tail -15" |
| +for ip in "$TGT_IP" "$SRC_IP"; do curl -sS -m 30 -o /dev/null -w "$ip %{http_code} %{time_total}s\n" --resolve "$H:443:$ip" "https://$H$PROBE_PATH"; done |
| +curl -sS -m 30 -o /dev/null -w "DNS %{http_code}\n" "https://$H$PROBE_PATH" |
| +``` |
| + |
| +| Check | Pass | |
| +|---|---| |
| +| The row on the target | `ready`, region `uk`, `/data/<host>.db` (flat until the nesting stop) | |
| +| Env file and instance key | the same sha and key digest as the baseline | |
| +| `config.env` | the same sha and names; it was written at landing, before the first request (the journal's "config updated" comes before "opened") | |
| +| Files | the same file count and digest | |
| +| Rows | the per-table digests up to the recorded rowids are identical to the source's just before `start`. Table count, total rows and the counts digest also match if the app has written nothing since it woke; if it has, they grow, and only the digests have to match | |
| +| Database | `quick_check ok` | |
| +| Timers | the same timer names on the target; a schedule tick in its journal or the app's own tick rows after `open`; `carlos schedule ls` shows the instance on the target box. Ticks due while the instance was offline were refused ("maintenance fence active") and do not run later; a once-timer that came due during the move fires once at the first wake | |
| +| Serving | 2xx through the target, through the source box (it backhauls to the target) and through DNS, with the expected `x-carlos-version`. If the app has a sign-in, Paul or the customer's contact tries it | |
| + |
| +A failed item after `open` is fixed forward on the target (section 11). |
| + |
| +--- |
| + |
| +## 8. finish |
| + |
| +```bash |
| +L "finish" 'echo "ABOUT TO RUN finish $H $(date -u +%FT%T.%3NZ)"; MV finish; echo "finish exit $?"; MV status' |
| +``` |
| + |
| +`finish` waits for the source box to clean up, then prints "Moved. shop.example.com is served from uk. It was offline for N seconds." and exits 0. It took 1.5 to 4 minutes on 2026-10-21; the instance serves throughout. If it answers that a report "is not current", run it again after 15 seconds. `status` then says "Move <id> of shop.example.com is done." |
| + |
| +After `finish`: |
| + |
| +```bash |
| +src "set -- $H; $(cat "$W/stray.box.sh")" |
| +src "set -- $H; $(cat "$W/facts.box.sh")" |
| +echo "source lock: $(aws --profile "$PROFILE" s3 cp --quiet "s3://$SRCDATA/instances/$H/lock.json" - | jq -c .)" |
| +echo "target lock: $(aws --profile "$PROFILE" s3 cp --quiet "s3://$TGTDATA/instances/$H/lock.json" - | jq -c .)" |
| +aws --profile "$PROFILE" s3 cp --quiet "s3://$DEPB/apps/$ACCT/$APP/instances/$H.json" - | jq -c '{region, move, last_move}' |
| +echo "target moves/$H/ current: $(aws --profile "$PROFILE" s3 ls --recursive "s3://$TGTDATA/moves/$H/" | wc -l)" |
| +carlos status --layout --console "$C" | grep -F "$H" |
| +``` |
| + |
| +Pass: |
| + |
| +- the source's `/data/.instances/<host>/` is empty (no stray env file); |
| +- the source has no row and no timers for the host, and its journal says "removed retired instance route" and then "tombstone lifted"; |
| +- parked on the source, under `/data/.offline-moves/<host>/<move-id>/`: `sealer/`, `source/instance/` (the old nested directory, database included) and, if there was one, `source/<host>/` (the legacy flat directory). The tool never deletes these. Add them to the record for the claim census; |
| +- left in place on the source by design: `/data/instances/<host>` and `/data/.instances/<host>` (empty), `/opt/carlos/live/<host>` (root-owned link) and `/data/programs/<host>.json` (root-owned, see section 12). `status` lists each with "left ... in place"; |
| +- the source `lock.json` reads `owner moved:<move-id>` with `expires_at 1970-01-01T00:00:00Z`. That expired object is the normal end state, not an abandoned move. Before `finish` it carries a 7-day expiry, renewed on each moves pass; |
| +- the target lock is the target box's own lease; |
| +- the record: `region uk`, `move null`, `last_move` outcome `done`; |
| +- 0 current objects under the target's `moves/<host>/` (the promotion moved them; versions and delete markers remain); |
| +- the layout row is on the target box, `flat`. |
| + |
| +--- |
| + |
| +## 9. The nesting stop, about an hour after open |
| + |
| +After `open` the instance runs flat on the target. The layout pass nests it when it is asleep, or, if it stays awake, stops it once the 1-hour grace is up and nests it. The target journal says so right after `open`: "layout: deferred until asleep -> shop.example.com (awake (stopped for the move in 1h0m0s unless it sleeps first))". An instance with a schedule shorter than its idle window never sleeps, so expect the stop. Tell the customer's contact there may be a stop of a few seconds about an hour after the move. |
| + |
| +About 65 minutes after `open`: |
| + |
| +```bash |
| +tgt "journalctl -u carlos-edge --since '-80 min' --no-pager | grep -F $H | grep -E 'layout' | tail -8" |
| +carlos status --layout --console "$C" | grep -E "$H|WARNING" |
| +carlos logs --app "$APP" --account "$ACCT" --console "$C" --since 90m | grep -E 'instance (stopping|exited|woke)' | tail -5 |
| +tgt "set -- $H $ROWS_ARGS; $(cat "$W/rows.box.sh")" |
| +curl -sS -m 30 -o /dev/null -w "target after nesting: %{http_code}\n" --resolve "$H:443:$TGT_IP" "https://$H$PROBE_PATH" |
| +``` |
| + |
| +Pass: the journal's "awake past its 1h0m0s grace -> stopping it for the move" and "migrated"; `status --layout` shows the host `nested` with "migrated at <time>: moved with it"; the digests unchanged; a 2xx. Measured stops: about 2 seconds on dev (the wake took 0.9 s), 24 seconds in the harness. The stop for a large flagship instance was not timed. |
| + |
| +Then check the instance's custom domains and aliases, if it has any: amadan #338 found an alias row that kept the flat socket after nesting and answered 502. |
| + |
| +Do not move the next instance until this stop is done, so two instances are never nested at once on the target. |
| + |
| +--- |
| + |
| +## 10. Replica cleanup |
| + |
| +A finished move leaves the source region's whole replica in the source data bucket, and the parked local Litestream directory on the source box. For a large instance that is several GB in the source region until someone removes it. Remove both, per instance, once it has passed section 7 and `finish`, and only then. |
| + |
| +**Go (Paul): deletes the source region's replica of one moved instance.** |
| + |
| +```bash |
| +L "cleanup" "bash $W/cleanup.sh /data/.offline-moves/$H/<move-id>/source/instance <open time, e.g. 2026-10-21T15:36:02+00:00>" |
| +``` |
| + |
| +`cleanup.sh` (Appendix A): |
| + |
| +1. refuses unless the source `lock.json` is the lifted tombstone (`moved:<id>`, expiring in 1970), so a move still in flight is never touched; |
| +2. lists the current objects under `instances/<host>/` in the source data bucket and refuses if any of them, other than `lock.json`, was written after the open time you give it. A write after `open` would mean something still writes there; |
| +3. removes the parked local replica `.../source/instance/.activator-<host>.db-litestream/ltx` on the source box, by exact path (a few hundred KB); |
| +4. deletes every object under `instances/<host>/` **by exact key, keeping `lock.json`** (the lifted tombstone stays as the record), and logs each key with the version id it removed and its size. |
| + |
| +The data buckets are **versioned with no lifecycle rule**. A delete only adds a delete marker; the bytes stay as noncurrent versions, are still billed, and can be brought back from the logged version id. Removing the noncurrent versions is a separate decision. The parked database, env and `config.env` under `source/instance/` stay too. |
| + |
| +Never run this for an instance whose move was aborted: its source replica is the live one. |
| + |
| +On 2026-10-21 this removed 290 objects (72 MB) for the 27 MB instance, 305 objects (1.58 GB) for the 1.6 GB instance and 270 objects (3.66 GB) for the 3.7 GB instance, about 4 minutes each. |
| + |
| +--- |
| + |
| +## 11. Rollback |
| + |
| +**Before `open`: abort.** At any doubt, a failed item, a target missing `config.env`, or a door that refuses for a real reason. `move.sh` does it on its own for `seal`, `flip` and the gate. By hand: |
| + |
| +```bash |
| +echo "ABOUT TO RUN abort $H $(date -u +%FT%T.%3NZ)"; MV abort; echo "abort exit $?" |
| +echo "ABOUT TO RUN finish $H $(date -u +%FT%T.%3NZ)"; MV finish; echo "finish exit $?" |
| +``` |
| + |
| +Both exit 3. `abort` prints "Moving shop.example.com back to ie." then "Every box has put shop.example.com back in ie."; `finish` prints "shop.example.com is back in ie and serving. Nothing was lost." The source's database was never removed, so nothing is lost. It took 25 seconds for the abort and under a second for the finish on the flagship, and the instance answered 2xx on the source 2 seconds later. |
| + |
| +Check after an abort: |
| + |
| +- `facts` and the row fingerprint on the source equal the baseline, and `quick_check ok`; |
| +- the source row is `ready` and nested; the source lease is the source box's own, at `generation 2`; |
| +- the source journal shows "wake fence lifted" and "tombstone lifted on abort"; |
| +- the target has no row and no timers, and parked what it had extracted under `/data/.offline-moves/<host>/<move-id>/target/`. Leave it: it does not block a retry; |
| +- the target bucket has 0 current objects under `instances/<host>/` and `moves/<host>/`; |
| +- the record has no region, `move null`, `last_move` outcome `aborted`; |
| +- 2xx through the source box, through the target box (backhaul) and through DNS. |
| + |
| +A retry is a new move, staged from scratch, after section 3 passes again. |
| + |
| +**After `open`: there is no move back.** `abort` is refused once the move is open (MV-O27), the target holds the only live copy and may already have taken writes. Fix forward on the target. Most problems after `open` (an app that cannot boot there, a certificate, reaching its store from the new region) show up in the first minute of section 7. A move back to the source region would be a new offline move in the other direction. It has never been run, it needs the target region's staging grant on the source data bucket, and the source box would need the section 3.3 checks, including its own stale `/opt/carlos/live/<host>` link, which the move leaves in place. Treat it as a new piece of work with its own approval, not as a rollback. |
| + |
| +--- |
| + |
| +## 12. Known gaps |
| + |
| +- **amadan #339: nothing checks the target before the instance goes offline.** A stale root-owned `/opt/carlos/live/<host>` on the target fails `flip` (exit 4, the `release` item, MV-O177) only after `start`. Section 3.3 is the workaround. The same discussion covers two smaller things: after an abort from staging, `status` reads as if the instance moved and came back; and no output says how many objects the seal copied (section 6 has the count). |
| +- **amadan #338: an alias row can keep the flat socket after nesting** and answer 502. Check an instance's aliases after the nesting stop (section 9). |
| +- **The source keeps root-owned files.** `/data/programs/<host>.json` (root, 0644, in a root-owned directory) cannot be parked by the agent, which runs as `carlos`; `status` says "left /data/programs/<host>.json in place: cannot be renamed: ... permission denied". The root-owned `/opt/carlos/live/<host>` link stays too, by design. Both are listed for the claim census, and both are the kind of leftover that section 3.3 catches if that box is ever a move target. |
| +- **The console's offline figure ends at `open`** and leaves out the first wake (section 6). |
| +- **One offline-move lock per deployment.** A staged or moving instance blocks every other one. |
| +- **The source region keeps the replica** until the cleanup in section 10, and the parked database and directories indefinitely. |
| +- **A new nested instance can start a few seconds before its `config.env`** (seen on dev on v0.28.6: about 12 seconds, then a config-change restart). That is the instance-create path, not the move: the move writes `config.env` at landing, before the first request. |
| + |
| +--- |
| + |
| +## 13. Rehearsing on dev |
| + |
| +Rehearse each new release's move path on dev before using it on the flagship. Dev's pair is sophie-knitting (home, `ie`, the console box, guard armed) to dennis-stenciling (`uk`); leo-climbing is a second home-region box and takes part as a source box. The values are in Appendix A. |
| + |
| +The rehearsals of 2026-10-21 used one app, `moverehearsal` in account `bab`, built from wakecheck's bytes, with four settings including store-shaped ones, one secret and a schedule `every-minute --every 1m --path /tick`. Each scenario uses a fresh instance of that app (`moverehearsal-a.dev.carlosframework.com` and so on), created with `carlos instances create` and woken through DNS so that it lands on sophie, its nested home. The probe path is `/`. |
| + |
| +Scenarios worth running: |
| + |
| +1. **A move with stage**: two stage rounds with writes between them, then `move.sh`. Expect about 40 seconds offline. |
| +2. **Abort from staging**: stage, then `abort` and `finish` (both exit 3); the instance never goes offline and the same process keeps serving. |
| +3. **Abort after flip**, before open: about 30 seconds offline, nothing lost. |
| +4. **The nesting stop** an hour after the first move's `open`. |
| + |
| +Afterwards: `carlos instances delete` each rehearsal host (the route retires on the next pass; parked data stays), then the replica cleanup of section 10 for the moved ones. Record what is left on each box and in each bucket. |
| + |
| +--- |
| + |
| +## 14. Telling the customer |
| + |
| +A draft for Paul to edit and send, one per instance, at least a day ahead: |
| + |
| +> We will move `<host>` to our UK region on `<date>` at `<time>` UTC. It will be offline for about 3 minutes, and we have booked 15 minutes for it. Your data moves with it, and if anything looks wrong before we reopen it, we put everything back as it was. Scheduled jobs that fall due while it is offline will not run. About an hour later it may pause again for a few seconds while we finish settling it in. |
| + |
| +Tell the customer's contact when each move starts and when it is done. |
| + |
| +--- |
| + |
| +## Appendix A. Setup and helper scripts |
| + |
| +### A.1 Scratch and values |
| + |
| +Make a scratch directory under `$TMPDIR` (never `/tmp` directly) and keep its exact path; it is removed by that path at the end (A.4). Keep the log somewhere that outlives the scratch. |
| + |
| +```bash |
| +W=$(mktemp -d "$TMPDIR/offline-move.XXXXXX"); echo "$W" |
| +``` |
| + |
| +Write `$W/env.sh`. These are the flagship's values for IE to UK; dev's are in the comments. |
| + |
| +```bash |
| +cat > "$W/env.sh" <<'EOF' |
| +export W=<the scratch path printed above> |
| +export LOG=<the move log, e.g. $HOME/.cache/<task>/moves-<date>.md> |
| +# Deployment. Dev: C=https://platform.dev.carlosframework.com PROFILE=kass DEPB=carlos-dev-640012953448 |
| +export C=https://console.carloku.com PROFILE=keymail DEPB=carlos-flagship-271376211898 |
| +# Source box (flagship IE, annie-welding). Dev: sophie-knitting i-05498eada6f0afd23, eu-west-1 |
| +export SRC_REGION=eu-west-1 SRC_ID=i-092c0c1eea75723cb |
| +# Target box (flagship UK, iris-clockmaking). Dev: dennis-stenciling i-009d593f451cfd562, eu-west-2 |
| +export TGT_REGION=eu-west-2 TGT_ID=i-03c310ceaf50a9bab |
| +# Data buckets: each box's CARLOS_HIBERNATE_BUCKET, also data_bucket in its edges file. Dev UK: carlos-dev-uk-640012953448 |
| +export SRCDATA=carlos-data-ie-271376211898 TGTDATA=carlos-data-uk-271376211898 |
| +export TO=uk |
| +# The instance |
| +export H=shop.example.com APP=<app> ACCT=<account sqid> |
| +export PROBE_PATH=/signin # a path that answers 2xx without signing in |
| +export ROWS_ARGS="" # "<table>:<max_rowid> ...", from 3.4 |
| +export SRC_IP=<source public IP> TGT_IP=<target public IP> |
| +ts() { date -u +%T.%3NZ; } |
| +MV() { carlos move offline "$1" --host "$H" --app "$APP" --account "$ACCT" --console "$C" ${2:-}; } |
| +src() { "$W/ssm.sh" "$PROFILE" "$SRC_REGION" "$SRC_ID" "$1"; } |
| +tgt() { "$W/ssm.sh" "$PROFILE" "$TGT_REGION" "$TGT_ID" "$1"; } |
| +# L <label> <command>: logs the label and command, then the output line by line as it runs, then the exit |
| +L() { local label=$1; shift |
| + printf '\n#### %s (started %s)\n```\n$ %s\n' "$label" "$(date -u +%FT%T.%3NZ)" "$*" >> "$LOG" |
| + bash -c "source $W/env.sh; $*" 2>&1 | tee -a "$LOG"; local rc=${PIPESTATUS[0]} |
| + printf '```\n(ended %s, exit %s)\n' "$(date -u +%FT%T.%3NZ)" "$rc" >> "$LOG"; return $rc; } |
| +# note <text>: a timestamped line in the log, e.g. what you are about to run and what you expect |
| +note() { printf -- '- %s %s\n' "$(date -u +%FT%T.%3NZ)" "$*" | tee -a "$LOG"; } |
| +EOF |
| +source "$W/env.sh" |
| +``` |
| + |
| +The two addresses, for `--resolve`: |
| + |
| +```bash |
| +aws --profile "$PROFILE" --region "$SRC_REGION" ec2 describe-instances --instance-ids "$SRC_ID" --query 'Reservations[0].Instances[0].PublicIpAddress' --output text |
| +aws --profile "$PROFILE" --region "$TGT_REGION" ec2 describe-instances --instance-ids "$TGT_ID" --query 'Reservations[0].Instances[0].PublicIpAddress' --output text |
| +``` |
| + |
| +Put them in `env.sh` and source it again. Do the same whenever you change `H` or `ROWS_ARGS`. |
| + |
| +### A.2 ssm.sh |
| + |
| +Runs one command as root on a box over SSM and prints its stdout, with stderr kept apart and a non-zero exit unless the command succeeded. The command goes into a temporary file and runs with stdin closed. SSM returns at most 24,000 characters. |
| + |
| +```bash |
| +cat > "$W/ssm.sh" <<'EOF' |
| +#!/bin/bash |
| +# usage: ssm.sh <profile> <region> <instance-id> '<shell command>' |
| +set -euo pipefail |
| +profile=$1 region=$2 inst=$3 cmd=$4 |
| +b64=$(printf '%s' "$cmd" | base64 -w0) |
| +params=$(python3 -I -c 'import json,sys;print(json.dumps({"commands":["f=$(mktemp) && echo %s | base64 -d > $f && bash $f </dev/null; rc=$?; rm -f $f; exit $rc" % sys.argv[1]]}))' "$b64") |
| +id=$(aws --profile "$profile" --region "$region" ssm send-command --instance-ids "$inst" \ |
| + --document-name AWS-RunShellScript --parameters "$params" --query Command.CommandId --output text) |
| +until s=$(aws --profile "$profile" --region "$region" ssm get-command-invocation --command-id "$id" \ |
| + --instance-id "$inst" --query Status --output text 2>/dev/null) && [ "$s" != InProgress ] && [ "$s" != Pending ]; do sleep 2; done |
| +aws --profile "$profile" --region "$region" ssm get-command-invocation --command-id "$id" --instance-id "$inst" \ |
| + --query StandardOutputContent --output text |
| +aws --profile "$profile" --region "$region" ssm get-command-invocation --command-id "$id" --instance-id "$inst" \ |
| + --query StandardErrorContent --output text >&2 |
| +[ "$s" = Success ] |
| +EOF |
| +chmod 700 "$W/ssm.sh" |
| +``` |
| + |
| +### A.3 Box scripts |
| + |
| +Each is sent with `src` or `tgt`, taking the host as `$1` where it needs one: `src "set -- $H; $(cat "$W/facts.box.sh")"`. All are read-only. |
| + |
| +```bash |
| +# block.box.sh: claim and Phase 1 switches, the edge unit, the stage path, version, guard, secrets key, region, disk |
| +cat > "$W/block.box.sh" <<'EOF' |
| +echo "== $(hostname)" |
| +echo "claim lines: $(grep -c "^CARLOS_HOST_CLAIM=." /etc/carlos/host.env || true)" |
| +echo "phase1 lines: $(grep -c "^CARLOS_MOVES_PHASE1=" /etc/carlos/host.env || true)" |
| +P=$(systemctl show -p MainPID --value carlos-edge) |
| +echo "edge env claim/phase1 vars: $(tr "\0" "\n" < /proc/$P/environ | grep -cE "^CARLOS_(HOST_CLAIM=.|MOVES_PHASE1=)" || true)" |
| +systemctl show carlos-edge -p User -p Delegate -p ActiveState |
| +echo "stage.path: $(systemctl is-active carlos-stage.path) $(systemctl is-enabled carlos-stage.path)" |
| +/opt/carlos/carlos version 2>&1 | tail -1 |
| +if [ -f /opt/carlos/system/guard/policy.json ]; then echo "guard policy present"; else echo "no guard policy"; fi |
| +echo "secrets key lines: $(grep -c "^CARLOS_SECRETS_KEY=" /etc/carlos/host.env || true)" |
| +echo "secrets key sha8: $(grep "^CARLOS_SECRETS_KEY=" /etc/carlos/host.env | cut -d= -f2- | tr -d "\"\n" | sha256sum | cut -c1-8)" |
| +echo "region/pools: $(grep -E "^CARLOS_(REGION|POOLS)=" /etc/carlos/host.env | tr "\n" " ")" |
| +echo "disk avail bytes, use: $(df -B1 --output=avail,pcent / | tail -1)" |
| +EOF |
| + |
| +# target.box.sh <host>: what the target box already holds for the host (3.3) |
| +cat > "$W/target.box.sh" <<'EOF' |
| +H=$1 |
| +L=/opt/carlos/live/$H |
| +if [ -L "$L" ] || [ -e "$L" ]; then echo "live link: PRESENT -> $(readlink "$L") (dated $(stat -c %y "$L" | cut -c1-19))"; else echo "live link: absent"; fi |
| +for p in /data/instances/$H /data/.instances/$H /data/$H /data/programs/$H.json /data/.offline-moves/$H /data/$H.env /data/$H.config.env /data/$H.db; do |
| + if [ -e "$p" ]; then echo "PRESENT $p"; else echo "absent $p"; fi |
| +done |
| +[ -d /data/.offline-moves/$H ] && find /data/.offline-moves/$H -maxdepth 2 | sed 's/^/ parked: /' |
| +python3 - "$H" <<'PY' |
| +import sqlite3,sys |
| +h=sys.argv[1]; c=sqlite3.connect('file:/data/registry.db?mode=ro',uri=True) |
| +print('registry routes', c.execute('select host,status from routes where host=?',(h,)).fetchall(), |
| + 'timers', c.execute('select count(*) from timers where host=?',(h,)).fetchone()[0]) |
| +PY |
| +EOF |
| + |
| +# facts.box.sh <host>: env and key digests, config.env, files, db, row, timers, parked (never values) |
| +cat > "$W/facts.box.sh" <<'EOF' |
| +H=$1 |
| +E=""; for d in /data/instances/$H /data; do [ -f $d/$H.env ] && { E=$d/$H.env; break; }; done |
| +if [ -n "$E" ]; then |
| + echo " env file: $E $(sha256sum < $E | cut -c1-16) key-sha $(grep ^CARLOS_INSTANCE_KEY= $E | cut -d= -f2- | tr -d '"\n' | sha256sum | cut -c1-16) names: $(cut -d= -f1 $E | sort | tr '\n' ' ')" |
| +else echo " env file: absent"; fi |
| +for f in /data/instances/$H/$H.config.env /data/$H.config.env; do [ -f $f ] && echo " config.env $f ($(stat -c %y $f)) sha $(sha256sum < $f | cut -c1-16): $(cut -d= -f1 $f | sort | tr '\n' ' ')"; done |
| +FD=""; for d in /data/instances/$H/$H /data/$H; do [ -d $d ] && { FD=$d; break; }; done |
| +if [ -n "$FD" ]; then |
| + echo " files: $FD $(cd $FD && find . -type f | wc -l) files, digest $(cd $FD && find . -type f -print0 | sort -z | xargs -0 -r sha256sum | sha256sum | cut -c1-16)" |
| +else echo " files: none 0 files, digest none"; fi |
| +for f in /data/instances/$H/$H.db /data/$H.db; do [ -f $f ] && echo " db: $f $(stat -c %s $f) bytes"; done |
| +ls /data/instances/$H 2>/dev/null | tr '\n' ' ' | sed 's/^/ instance dir: /'; echo |
| +python3 - $H <<'PY' |
| +import sqlite3,sys |
| +h=sys.argv[1]; c=sqlite3.connect('file:/data/registry.db?mode=ro',uri=True) |
| +print(' row:', c.execute('select status,last_error,region,db_path,next_due,replication from routes where host=?',(h,)).fetchall()) |
| +try: |
| + t=c.execute('select * from timers where host=?',(h,)).fetchall() |
| + print(' timers:', [(x[1],x[2],x[3],x[4],x[7]) for x in t]) |
| + print(' timer names:', ' '.join(sorted(x[1] for x in t))) |
| +except Exception as e: print(' timers: ?', e) |
| +PY |
| +ls -d /data/.offline-moves/$H/*/* 2>/dev/null | sed 's/^/ parked: /' |
| +EOF |
| + |
| +# sizes.box.sh <host>: files tree (limit 256 MiB), nested dir, db, local Litestream dir, sidecars, legacy flat dir |
| +cat > "$W/sizes.box.sh" <<'EOF' |
| +H=$1; D=/data/.instances/$H |
| +echo "== $H" |
| +echo "files tree bytes: $(du -sb $D/$H 2>/dev/null | cut -f1)" |
| +echo "nested dir: $(du -sh $D | cut -f1)" |
| +ls -l $D/$H.db 2>&1 | awk '{print "db bytes:", $5}' |
| +echo "local ltx: $(du -sh $D/.activator-$H.db-litestream 2>/dev/null | cut -f1)" |
| +ls $D/*sidecar* /data/$H-sidecar.env 2>&1 | head -2 |
| +ls -la /data/$H 2>&1 | head -6 |
| +EOF |
| + |
| +# procs.box.sh <host>: processes in the host's cgroup slice (0 before flip) |
| +cat > "$W/procs.box.sh" <<'EOF' |
| +H=$1 |
| +f=$(find /sys/fs/cgroup/carlos.slice/carlos-instances.slice -maxdepth 1 -name "carlos-instances-$H.slice" 2>/dev/null | head -1) |
| +echo "slice: ${f:-none}" |
| +if [ -n "$f" ]; then echo "processes: $(cat $f/cgroup.procs | wc -l)"; else echo "processes: 0 (no slice)"; fi |
| +d=$(find /sys/fs/cgroup/carlos.slice/carlos-edge.service -maxdepth 1 -name "carlos-exec@$H" 2>/dev/null | head -1) |
| +if [ -n "$d" ]; then echo "exec cgroup $d processes: $(cat $d/cgroup.procs | wc -l)"; fi |
| +EOF |
| + |
| +# stray.box.sh <host>: what the source keeps after finish |
| +cat > "$W/stray.box.sh" <<'EOF' |
| +H=$1 |
| +echo "== /data/.instances/$H"; ls -la /data/.instances/$H/ 2>&1 |
| +echo "== parked"; ls -d /data/.offline-moves/$H/*/* /data/.offline-moves/$H/*/source/* 2>&1 |
| +echo "== legacy /data/$H"; ls -la /data/$H 2>&1 | head -5 |
| +echo "== programs"; ls -la /data/programs/$H.json 2>&1 |
| +echo "== live link"; ls -la /opt/carlos/live/$H 2>&1 |
| +EOF |
| + |
| +# schema.box.sh <host>: tables, row counts and time columns, to choose the fingerprint tables |
| +cat > "$W/schema.box.sh" <<'EOF' |
| +H=$1 |
| +DB=""; for f in /data/instances/$H/$H.db /data/$H.db; do [ -f $f ] && { DB=$f; break; }; done |
| +echo "db: ${DB:-none}"; [ -z "$DB" ] && exit 0 |
| +python3 - "$DB" <<'PY' |
| +import sqlite3,sys |
| +c=sqlite3.connect('file:%s?mode=ro'%sys.argv[1],uri=True) |
| +for (n,) in c.execute("select name from sqlite_master where type='table' order by name"): |
| + cols=[r[1] for r in c.execute('pragma table_info("%s")'%n)] |
| + k=c.execute('select count(*) from "%s"'%n).fetchone()[0] |
| + tcols=[x for x in cols if x in ('created_at','updated_at','at','ts','timestamp','occurred_at','sent_at','inserted_at')] |
| + print(n, k, 'timecols=%s'%tcols) |
| +PY |
| +EOF |
| + |
| +# rows.box.sh <host> [table[:max_rowid] ...]: read-only row fingerprint (counts, max rowids, digests; never row content) |
| +cat > "$W/rows.box.sh" <<'EOF' |
| +H=$1; shift |
| +DB=""; for f in /data/instances/$H/$H.db /data/$H.db; do [ -f $f ] && { DB=$f; break; }; done |
| +echo "db: ${DB:-none}"; [ -z "$DB" ] && exit 0 |
| +python3 - "$DB" "$@" <<'PY' |
| +import sqlite3,sys,hashlib |
| +c=sqlite3.connect('file:%s?mode=ro'%sys.argv[1],uri=True) |
| +c.execute('begin') |
| +tabs=[n for (n,) in c.execute("select name from sqlite_master where type='table' and name not like 'sqlite_%' and name not like '_litestream%' order by name")] |
| +counts={n:c.execute('select count(*) from "%s"'%n).fetchone()[0] for n in tabs} |
| +print('tables %d, total rows %d, nonempty %d' % (len(tabs), sum(counts.values()), sum(1 for v in counts.values() if v))) |
| +print('counts digest', hashlib.sha256(''.join('%s:%d\n'%(n,counts[n]) for n in tabs).encode()).hexdigest()[:16]) |
| +for arg in sys.argv[2:]: |
| + t,_,lim=arg.partition(':') |
| + r=c.execute('select count(*),max(rowid) from "%s"'%t).fetchone() |
| + print('%s: count %d max_rowid %s' % (t,r[0],r[1])) |
| + if lim: |
| + h=hashlib.sha256(); n=0 |
| + for row in c.execute('select rowid,* from "%s" where rowid<=? order by rowid'%t,(int(lim),)): |
| + h.update(repr(row).encode()); n+=1 |
| + print('%s rowid<=%s: %d rows digest %s' % (t,lim,n,h.hexdigest()[:16])) |
| +c.execute('rollback') |
| +PY |
| +EOF |
| +``` |
| + |
| +### A.4 Operator scripts |
| + |
| +```bash |
| +# stage.sh: one staging round for $H, then status, the record's move and the stager report |
| +cat > "$W/stage.sh" <<'EOF' |
| +#!/bin/bash |
| +source "$(dirname "$0")/env.sh" |
| +now() { date -u +%FT%T.%3NZ; } |
| +echo "=== before $(now): source instances/$H/ $(aws --profile $PROFILE s3 ls --recursive s3://$SRCDATA/instances/$H/ | wc -l) objects; target moves/$H/ $(aws --profile $PROFILE s3 ls --recursive s3://$TGTDATA/moves/$H/ | wc -l) objects" |
| +echo "ABOUT TO RUN stage $H $(now)" |
| +s=$(ts); out=$(MV stage "--to $TO" 2>&1); rc=$? |
| +echo "--- stage $s -> $(ts) exit=$rc"; echo "$out" | sed 's/^/ /' |
| +echo "--- status $(ts)"; MV status | sed 's/^/ /' |
| +M=$(aws --profile $PROFILE s3 cp --quiet s3://$DEPB/apps/$ACCT/$APP/instances/$H.json - | jq -c '.move') |
| +echo "record move: $M" |
| +ID=$(echo "$M" | jq -r '.id // empty') |
| +if [ -n "$ID" ]; then |
| + for k in $(aws --profile $PROFILE s3 ls --recursive "s3://$DEPB/control/offline-moves/$H/$ID/stager/" | awk '{print $4}'); do |
| + echo "stager report $k" |
| + aws --profile $PROFILE s3 cp --quiet "s3://$DEPB/$k" - | jq -c '{box, phase, at, replica_objects, replica_bytes, staged_objects, staged_bytes, round_objects, round_bytes, stage_error}' |
| + done |
| +fi |
| +echo "END stage.sh rc=$rc $(now)" |
| +exit $rc |
| +EOF |
| + |
| +# move.sh [probe-seconds]: final stage, baseline, then start, seal, flip, the gate and open back to back. |
| +# Before open, a failed seal, flip or gate aborts the move and runs finish. After open it only reports. |
| +cat > "$W/move.sh" <<'EOF' |
| +#!/bin/bash |
| +source "$(dirname "$0")/env.sh" |
| +PROBE_FOR=${1:-1200} |
| +now() { date -u +%FT%T.%3NZ; } |
| +door() { # door <verb> [args]: retries only a report that is not current (MV-O178) |
| + local i out rc s |
| + for i in 1 2 3 4 5 6 7 8; do |
| + echo "ABOUT TO RUN $1 $H $(now) (try $i)" |
| + s=$(ts); out=$(MV "$1" "${2:-}" 2>&1); rc=$? |
| + echo "--- $1 try $i $s -> $(ts) exit=$rc"; echo "$out" | sed 's/^/ /' |
| + if [ $rc -ne 0 ] && echo "$out" | grep -q "is not current"; then sleep 15; continue; fi |
| + return $rc |
| + done; return $rc; } |
| +back_out() { # back_out <reason>: abort and finish, both expected to exit 3 |
| + local s out rc |
| + echo "STOP: $1 $(now). Aborting (before open)." |
| + echo "ABOUT TO RUN abort $H $(now)"; s=$(ts); out=$(MV abort 2>&1); rc=$? |
| + echo "--- abort $s -> $(ts) exit=$rc"; echo "$out" | sed 's/^/ /' |
| + echo "ABOUT TO RUN finish $H $(now)"; s=$(ts); out=$(MV finish 2>&1); rc=$? |
| + echo "--- finish $s -> $(ts) exit=$rc"; echo "$out" | sed 's/^/ /' |
| + for i in $(seq 1 20); do c=$(curl -s -m 20 -o /dev/null -w '%{http_code}' --resolve $H:443:$SRC_IP https://$H$PROBE_PATH); echo "source $c $(ts)"; case "$c" in 2*) break;; esac; sleep 3; done |
| + MV status; echo "END move.sh STOPPED $(now)"; exit 12; } |
| + |
| +echo "=== final stage right before start $(now)" |
| +door stage "--to $TO" || { echo "STOP: final stage exit $? $(now). Nothing is offline."; MV status; exit 10; } |
| +echo "=== baseline on the source $(now)" |
| +A=$(src "set -- $H; $(cat $W/facts.box.sh)"); echo "$A" |
| +ENVSHA=$(echo "$A" | sed -n -E 's/^ env file: \S+ ([0-9a-f]{16}) key-sha .*/\1/p') |
| +KEYSHA=$(echo "$A" | sed -n -E 's/^ env file: .* key-sha ([0-9a-f]{16}) .*/\1/p') |
| +CFGSHA=$(echo "$A" | sed -n -E 's/^ config.env .* sha ([0-9a-f]{16}): .*/\1/p') |
| +FILES=$(echo "$A" | sed -n -E 's/^ files: \S+ (.*)$/\1/p') |
| +TIMERS=$(echo "$A" | sed -n -E 's/^ timer names: (.*)$/\1/p') |
| +echo "expect env $ENVSHA key $KEYSHA cfg ${CFGSHA:-none} files $FILES timers $TIMERS" |
| +for v in "$ENVSHA" "$KEYSHA" "$FILES"; do [ -n "$v" ] || { echo "STOP: baseline incomplete. Nothing is offline."; exit 9; }; done |
| +echo "=== rows fingerprint on the source, just before start $(now)" |
| +src "set -- $H $ROWS_ARGS; $(cat $W/rows.box.sh)" |
| + |
| +T0=$(date +%s.%N); echo "=== T0 $(now)" |
| +door start "--to $TO" || { echo "STOP: start exit $? $(now). Read status: if a move is in flight, abort it and run finish."; MV status; exit 10; } |
| +door seal || back_out "seal exit $?" |
| +echo "--- source slice $(now)"; src "set -- $H; $(cat $W/procs.box.sh)" |
| +door flip || back_out "flip exit $?" |
| + |
| +echo "--- target facts (gate before open) $(now)" |
| +B=$(tgt "set -- $H; $(cat $W/facts.box.sh)"); echo "$B" |
| +fail="" |
| +echo "$B" | grep -q -F "row: [('landing', '', '$TO', '/data/$H.db'" || fail="$fail row" |
| +echo "$B" | grep -q -E "^ env file: /data/$H.env $ENVSHA key-sha $KEYSHA " || fail="$fail env/key" |
| +[ "$(echo "$B" | sed -n -E 's/^ config.env .* sha ([0-9a-f]{16}): .*/\1/p')" = "$CFGSHA" ] || fail="$fail config" |
| +[ "$(echo "$B" | sed -n -E 's/^ files: \S+ (.*)$/\1/p')" = "$FILES" ] || fail="$fail files" |
| +[ "$(echo "$B" | sed -n -E 's/^ timer names: (.*)$/\1/p')" = "$TIMERS" ] || fail="$fail timers" |
| +[ -z "$fail" ] || back_out "gate failed:$fail" |
| +echo "gate before open: ok $(now)" |
| + |
| +door open || { echo "STOP: open exit $? $(now). Do not abort blindly: read status (runbook section 11)."; MV status; exit 14; } |
| +echo "--- probe $PROBE_PATH through the target until a 2xx (up to $PROBE_FOR s) $(now)" |
| +prev=""; first=""; end=$(( $(date +%s) + PROBE_FOR )) |
| +while [ $(date +%s) -lt $end ]; do |
| + c=$(curl -s -m 10 -o /dev/null -w '%{http_code}' --resolve $H:443:$TGT_IP https://$H$PROBE_PATH) |
| + [ "$c" != "$prev" ] && { echo "target $c $(ts)"; prev=$c; } |
| + case "$c" in 2*) first=$(date +%s.%N); break;; esac; sleep 2; done |
| +[ -n "$first" ] || { echo "NO 2xx through the target within $PROBE_FOR s $(now)"; exit 15; } |
| +echo "FIRST 2xx through the target at $(ts): offline $(python3 -c "print(round($first-$T0,1))") s since start" |
| +curl -sS -m 30 -D - -o /dev/null --resolve $H:443:$TGT_IP https://$H$PROBE_PATH | grep -i -E "^HTTP|x-carlos-version" |
| +for i in $(seq 1 25); do a=$(curl -s -m 20 -o /dev/null -w '%{http_code}' --resolve $H:443:$SRC_IP https://$H$PROBE_PATH); d=$(curl -s -m 20 -o /dev/null -w '%{http_code}' https://$H$PROBE_PATH); echo "$(ts) via source $a | via DNS $d"; [ "${a:0:1}" = 2 ] && [ "${d:0:1}" = 2 ] && break; sleep 3; done |
| +echo "--- status after open $(now)"; MV status |
| +echo "END move.sh OK $(now)" |
| +EOF |
| + |
| +# sealcount.sh <seal-start> <seal-end> (UTC, to the second, e.g. 2026-10-21T15:34:38): versions the seal wrote |
| +cat > "$W/sealcount.sh" <<'EOF' |
| +#!/bin/bash |
| +source "$(dirname "$0")/env.sh" |
| +aws --profile $PROFILE s3api list-object-versions --bucket $TGTDATA --prefix moves/$H/ --query 'Versions[].[LastModified,Size]' --output text | |
| + awk -v a="$1" -v b="$2" '{t=substr($1,1,19)} t<a {pre++} t>=a && t<=b {n++; s+=$2} t>b {post++} END {print "before the seal (stage rounds):", pre+0; print "during the seal window:", n+0, "objects,", s+0, "bytes"; print "after the seal window:", post+0}' |
| +EOF |
| + |
| +# cleanup.sh <parked source/instance dir> <open time, e.g. 2026-10-21T15:36:02+00:00>: section 10 |
| +cat > "$W/cleanup.sh" <<'EOF' |
| +#!/bin/bash |
| +source "$(dirname "$0")/env.sh" |
| +P=$1 OPENED=$2 PFX=instances/$H/ |
| +[ -n "$P" ] && [ -n "$OPENED" ] || { echo "usage: cleanup.sh <parked source/instance dir> <open time>"; exit 2; } |
| +LOCK=$(aws --profile $PROFILE s3 cp --quiet "s3://$SRCDATA/${PFX}lock.json" -); echo "lock.json: $LOCK" |
| +echo "$LOCK" | jq -e '(.owner|startswith("moved:")) and .expires_at=="1970-01-01T00:00:00Z"' >/dev/null || { echo "STOP: the tombstone is not lifted, so the move is not finished"; exit 2; } |
| +aws --profile $PROFILE s3api list-objects-v2 --bucket $SRCDATA --prefix $PFX --query 'Contents[].[Key,LastModified,Size]' --output text > "$W/cleanup-$H.keys" |
| +LATEST=$(grep -v -F "${PFX}lock.json" "$W/cleanup-$H.keys" | cut -f2 | sort | tail -1) |
| +echo "current objects: $(wc -l < "$W/cleanup-$H.keys"); latest write excluding lock.json: $LATEST; opened: $OPENED" |
| +[[ "$LATEST" < "$OPENED" ]] || { echo "STOP: something wrote to the source replica after open"; exit 2; } |
| +L2=$P/.activator-$H.db-litestream/ltx |
| +echo "== parked local replica: $L2" |
| +src "test -d $L2 && echo \"size: \$(du -sh $L2 | cut -f1), files: \$(find $L2 -type f | wc -l)\" && rm -r -- $L2 && echo removed $L2; ls -la $(dirname $L2)" |
| +echo "== source replica objects by exact key (key, removed version, size), keeping lock.json" |
| +n=0; bytes=0 |
| +while IFS=$'\t' read -r k lm size; do |
| + case "$k" in |
| + "${PFX}lock.json") echo "keep $k"; continue;; |
| + "$PFX"*) ;; |
| + *) echo "SKIP unexpected $k"; continue;; |
| + esac |
| + v=$(aws --profile $PROFILE s3api head-object --bucket $SRCDATA --key "$k" --query VersionId --output text) |
| + aws --profile $PROFILE s3api delete-object --bucket $SRCDATA --key "$k" >/dev/null && echo "deleted ${k#$PFX} version $v size $size" && n=$((n+1)) && bytes=$((bytes+size)) |
| +done < "$W/cleanup-$H.keys" |
| +echo "deleted $n source replica objects of $H ($bytes bytes)" |
| +echo "== after"; aws --profile $PROFILE s3 ls --recursive "s3://$SRCDATA/$PFX" |
| +aws --profile $PROFILE s3 cp --quiet "s3://$SRCDATA/${PFX}lock.json" - | jq -c . |
| +EOF |
| +chmod 700 "$W"/*.sh |
| +``` |
| + |
| +At the end of the session, check nothing you started is still running, then remove the scratch by its exact path (never a glob): `rm -r -- <the path printed in A.1>`. Keep the log. |
| + |
| +--- |
| + |
| +## History |
| + |
| +### v0.28.5 roll, 2026-10-20 |
| + |
| +The first release with the offline-move doors. Dev's guard-armed boxes could not take it through their own updater (the v0.28.4 updater read a candidate's stderr as part of its answer), so leo-climbing and sophie-knitting took it by a ruled hand re-arm, and dennis-stenciling by its normal `system update`. Sophie's guarded console could not be replaced by `install-console` (it refused an occupied slot), so it was upgraded by an attended recovery, "E1": both slots moved aside, then `install-console`. All of it was rehearsed first in the harness (Run 9), which also ran the first offline move of a 4 MB instance: 366 seconds offline, most of it waiting for stale box reports. The full roll runbook with its signing session, re-arm and E1 is this file as it stood at commit b1464900 on branch v0285-roll-runbook. |
| + |
| +Run 9 found four defects, all fixed in v0.28.6: |
| + |
| +| Defect | What it was | |
| +|---|---| |
| +| 5 | `install-console` could not replace an installed protected console. Fixed by `install-console --upgrade --from <sha256>` | |
| +| 6 | a new nested instance could run without its `config.env` for up to about 70 minutes | |
| +| 7 | `flip` and `open` trusted a box report for 2 minutes while boxes rewrote theirs every 5 to 6, so the doors refused with "still runs a process" for most of each cycle. Fixed by a 15-second report refresh during a move and a "not current" refusal | |
| +| 8 | the source's scheduler re-created the parked instance's env file after `open` | |
| + |
| +### v0.28.6 roll and the first moves, 2026-10-20 and 21 |
| + |
| +The harness (Run 10) rolled the v0.28.6 candidate by plain `system update` on every box and upgraded the guarded console with `--upgrade`, and moved a small instance in about 20 seconds and a 636 MB instance with a 3.5 GB replica history in 296 seconds. It found that the console's offline figure ends at `open`, that a large replica's first wake on the target takes minutes, and that a finished move leaves the whole replica in the source region. |
| + |
| +v0.28.6 went to dev, then flagship (boxes 08:39 to 08:43Z, console 08:45Z on 2026-10-21). |
| + |
| +- **Dev rehearsal on v0.28.6** (08:02 to 09:22Z): a move of a small instance from sophie to dennis, 30.6 seconds offline (the console said 23.5); an abort after flip, 32.8 seconds; the nesting stop at the hour, about 2 seconds; the cleanup. Its corrections: the staging lane's `GetObject implicitDeny` is the designed grant; only the target's secrets key fingerprint matters; `open` prints a refusal-like first line while succeeding; the source cannot park the root-owned `/data/programs/<host>.json`. |
| +- **The first flagship move**, a 27 MB instance without `stage`: seal 82 seconds for 288 replica objects on real S3, 164 seconds offline (the console said 123). Verified by the row fingerprint, nested at the hour. |
| +- **The aborted move**, a 1.6 GB instance without `stage`: the seal alone took about 6 minutes. A session restart killed the move script after the seal; its log wrapper had written nothing, so the move was believed not to have started. Found by `status` 12 minutes later, aborted, back on the source with nothing lost after 12.7 minutes offline. That is where section 1 comes from. |
| + |
| +Paul then approved building the pre-copy and parallel object copies. |
| + |
| +### v0.28.7 and the moves with stage, 2026-10-21 |
| + |
| +v0.28.7 added `carlos move offline stage` and eight object copies at once. It went to dev (14:30Z), then flagship (14:41Z). |
| + |
| +- **Dev pre-copy rehearsal** (14:31 to 14:54Z): a move from staging, 40.6 seconds offline (the console said 22.3), the seal copying the 4 objects written since the last stage; an abort from staging with the instance never offline. Its corrections are sections 4 and 6: stage once more right before `start`; read round counts from the stager report; `stage` does not shorten the first wake on the target; the target can 404 for about 8 seconds after `open`; report client-measured time; abort from staging exits 3; `status` after it reads oddly; one lock per deployment. |
| +- **The 1.6 GB instance, first attempt with stage**: staged 1.58 GB in 79 seconds while serving with no failed requests, seal copied nothing, then `flip` exited 4 on the target's stale `/opt/carlos/live/<host>` link from an earlier attempt. Aborted automatically, about 2 minutes offline, nothing lost. Paul approved clearing the link and its old version directory on the target. Section 3.3 is the check that would have caught it before staging, and amadan #339 the fix. |
| +- **The 3.7 GB instance**, with the new target pre-checks: staged in 142 seconds, 190 seconds offline (the console said 117), 52 of them promoting the staged copy into the live replica on the first wake. |
| +- **The 1.6 GB instance, retry**: 148 seconds offline (the console said 100). |
| +- **Cleanup**: the source region's replicas of all three moved instances deleted by exact key with their version ids logged, `lock.json` kept, the parked Litestream directories removed. |
diff --git a/internal/activator/activator_test.go b/internal/activator/activator_test.go| index fc1b86555..84c04e9df 100644 |
| --- a/internal/activator/activator_test.go |
| +++ b/internal/activator/activator_test.go |
| @@ -1092,11 +1092,15 @@ func requireUnixSockets(t *testing.T) { |
| l.Close() |
| } |
| +// tmpSockSeq numbers tmpSock's paths, so tests running in parallel never |
| +// share one. |
| +var tmpSockSeq atomic.Uint64 |
| + |
| // tmpSock returns a short unix socket path (the sun_path limit is ~104 bytes, |
| // so t.TempDir() can be too long) and schedules its removal. |
| func tmpSock(t *testing.T) string { |
| t.Helper() |
| - s := filepath.Join(os.TempDir(), fmt.Sprintf("carlos-act-%d-%d.sock", os.Getpid(), time.Now().UnixNano()%1_000_000)) |
| + s := filepath.Join(os.TempDir(), fmt.Sprintf("carlos-act-%d-%d.sock", os.Getpid(), tmpSockSeq.Add(1))) |
| os.Remove(s) |
| t.Cleanup(func() { os.Remove(s) }) |
| return s |
| @@ -1107,6 +1111,7 @@ func tmpSock(t *testing.T) string { |
| // Many concurrent requests during a wake coalesce into exactly one wake: one |
| // lease claim, one restore, one spawn. |
| func TestActivatorWakeCoalesces(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Minute) |
| b.startGate = make(chan struct{}) |
| @@ -1478,6 +1483,7 @@ func TestActivatorWakeRefusesFencedRowBeforeAcquire(t *testing.T) { |
| // The other two doWake bail-outs release the lease exactly once too — the |
| // property the seed step slots into the middle of. |
| func TestActivatorWakeFailuresReleaseLeaseOnce(t *testing.T) { |
| + t.Parallel() |
| for _, tc := range []struct { |
| name string |
| inject func(*fakeBackend) |
| @@ -1517,6 +1523,7 @@ func TestActivatorWakeFailuresReleaseLeaseOnce(t *testing.T) { |
| // wakes under -race and contending goroutines and asserts the invariant |
| // acquires == requests exactly. |
| func TestActivatorFailedWakeIsSingleflight(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Minute) |
| b.restoreErr = errors.New("s3 down") |
| rt := testRoute("singleflight.example.com") |
| @@ -1583,6 +1590,7 @@ func TestActivatorRestoreFailureSkipsSeed(t *testing.T) { |
| // instance: process stopped, replication closed (final checkpoint + lease |
| // release), entry forgotten; the next request wakes it afresh. |
| func TestActivatorIdleSleep(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _ := testActivator(t, 10*time.Second) |
| release, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| @@ -1622,6 +1630,7 @@ func TestActivatorIdleSleep(t *testing.T) { |
| // An in-flight request (an open WebSocket, in production) holds the instance |
| // awake past the idle window; only after it finishes does the idle clock run. |
| func TestActivatorInflightKeepsAwake(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _ := testActivator(t, 10*time.Second) |
| release, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| @@ -1652,6 +1661,7 @@ func TestActivatorInflightKeepsAwake(t *testing.T) { |
| // rate-limited to one per tick, gracefully cycles the child so its next wake |
| // execs the new binary. |
| func TestActivatorConvergesOnSymlinkRepoint(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _ := testActivator(t, time.Hour) // idle far away — isolate the convergence path |
| var ver atomic.Value // the live/<host> symlink target; swapped to model a ship |
| ver.Store("v1") |
| @@ -1712,6 +1722,7 @@ func TestActivatorConvergesOnSymlinkRepoint(t *testing.T) { |
| // Two awake children go stale at once (a fleet-wide ship). One sweep cycles at |
| // most one — a staggered convergence, never a simultaneous blip of every space. |
| func TestActivatorConvergenceRateLimitsPerSweep(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _ := testActivator(t, time.Hour) |
| var ver atomic.Value |
| ver.Store("v1") |
| @@ -1745,6 +1756,7 @@ func TestActivatorConvergenceRateLimitsPerSweep(t *testing.T) { |
| // idle-hibernate can never touch. Unlike idle-hibernate, convergence fires with |
| // a request in flight — the graceful stop drains it exactly as a restart would. |
| func TestActivatorConvergesWithRequestInFlight(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _ := testActivator(t, time.Hour) |
| var ver atomic.Value |
| ver.Store("v1") |
| @@ -1815,6 +1827,7 @@ func TestConvergeDecision(t *testing.T) { |
| // A lost lease (failed renewal) stands the instance down immediately — no |
| // waiting for idle: someone else may own the S3 prefix now. |
| func TestActivatorLeaseLostStandsDown(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| release, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| @@ -1862,6 +1875,7 @@ func TestActivatorDeadChildRewakesOnRequest(t *testing.T) { |
| // A request that arrives mid-stop waits for the stop, then wakes fresh. |
| func TestActivatorRequestDuringStopRewakes(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _ := testActivator(t, 10*time.Second) |
| release, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| @@ -2200,6 +2214,7 @@ func TestSweepKeepsHostAwakeWhenATickIsImminent(t *testing.T) { |
| // Beyond the window, the sleep is worth taking. |
| func TestSweepHibernatesWhenTheNextTickIsFarOff(t *testing.T) { |
| + t.Parallel() |
| const host = "faroff.bab.oncarlos.com" |
| a, _, clk, reg := testActivator(t, 15*time.Minute) |
| rt := testRoute(host) |
| @@ -2221,6 +2236,7 @@ func TestSweepHibernatesWhenTheNextTickIsFarOff(t *testing.T) { |
| // A host with no timer rows is untouched by the guard — the overwhelming |
| // majority of instances, and the case that must not regress. |
| func TestSweepHibernatesAHostWithNoTimers(t *testing.T) { |
| + t.Parallel() |
| const host = "untimed.bab.oncarlos.com" |
| a, _, clk, _ := testActivator(t, 15*time.Minute) // no timer rows at all |
| rt := testRoute(host) |
| @@ -2263,6 +2279,7 @@ func TestSweepKeepsHostAwakeForAnOverdueTick(t *testing.T) { |
| // must fall through to the convergence check, not skip it, or a `carlos |
| // ship` behind a tick-held host is served stale indefinitely. |
| func TestSweepHeldForTickStillConverges(t *testing.T) { |
| + t.Parallel() |
| const host = "tickheld.bab.oncarlos.com" |
| a, b, clk, reg := testActivator(t, 15*time.Minute) |
| var ver atomic.Value // the live/<host> symlink target; swapped to model a ship |
| @@ -2313,6 +2330,7 @@ func TestSweepHeldForTickStillConverges(t *testing.T) { |
| // hibernating hosts, so a systemd-managed instance can never be spawned |
| // against its own unit. |
| func TestActivatorReloadRetiresUnflaggedHost(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, reg := testActivator(t, time.Hour) |
| route := testRoute("flip.example.com") |
| if err := reg.Upsert(route); err != nil { |
| @@ -2357,6 +2375,7 @@ func TestActivatorReloadRetiresUnflaggedHost(t *testing.T) { |
| // once the wake resolves, so a flip-off during the small wake window still |
| // releases cleanly. |
| func TestActivatorRetireDuringWake(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, reg := testActivator(t, time.Hour) |
| route := testRoute("mid.example.com") |
| if err := reg.Upsert(route); err != nil { |
| @@ -2396,6 +2415,7 @@ func TestActivatorRetireDuringWake(t *testing.T) { |
| // With hibernation on, sibling services (kind "service") are untouched: the |
| // edge proxies them directly, no wake, no activity tracking. |
| func TestActivatorServiceRoutesBypass(t *testing.T) { |
| + t.Parallel() |
| requireUnixSockets(t) |
| sock := tmpSock(t) |
| ln, err := net.Listen("unix", sock) |
| @@ -2498,6 +2518,7 @@ func TestBounceStopsAwakeInstanceOnly(t *testing.T) { |
| // child, so a host draining its SIGTERM must still read as owned — otherwise |
| // the advisory calls the activator's own children strays. |
| func TestOwnsCoversEntryLifetimeAnyState(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _ := testActivator(t, 10*time.Second) |
| host := "own.example.com" |
| if a.Owns(host) { |
| @@ -2574,6 +2595,7 @@ func (a *Service) entryAwake(host string) bool { |
| // in the registry; the sweeper wakes the host (no request) when it arrives; the |
| // following sleep re-records — and once nothing is scheduled, no more due wakes. |
| func TestActivatorDueTimeWakeCycle(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, reg := testActivator(t, 10*time.Second) |
| route := testRoute("due.example.com") |
| if err := reg.Upsert(route); err != nil { |
| @@ -2632,6 +2654,7 @@ func TestActivatorDueTimeWakeCycle(t *testing.T) { |
| // there is no ping-pong, because a woken instance can't sleep again until a |
| // full idle window passes. |
| func TestActivatorDuePastWakesOnceNoPingPong(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, reg := testActivator(t, 10*time.Second) |
| route := testRoute("late.example.com") |
| if err := reg.Upsert(route); err != nil { |
| @@ -2669,6 +2692,7 @@ func TestActivatorDuePastWakesOnceNoPingPong(t *testing.T) { |
| // the sleep proceeds — the instance degrades to "sleeps until a request", |
| // never a crash and never a stuck stop. |
| func TestActivatorDueProbeFailureRecordsZero(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, reg := testActivator(t, 10*time.Second) |
| route := testRoute("deaf.example.com") |
| if err := reg.Upsert(route); err != nil { |
| @@ -2699,6 +2723,7 @@ func TestActivatorDueProbeFailureRecordsZero(t *testing.T) { |
| // restart still fires: a brand-new activator over the same registry — no |
| // in-memory state carried over — wakes the host when its time arrives. |
| func TestActivatorDueSurvivesRestart(t *testing.T) { |
| + t.Parallel() |
| a1, b1, clk, reg := testActivator(t, 10*time.Second) |
| route := testRoute("reborn.example.com") |
| if err := reg.Upsert(route); err != nil { |
| @@ -2768,6 +2793,7 @@ func TestActivatorCoSpawnsSidecarAfterInstance(t *testing.T) { |
| // host serves without its sidecar, and the sweeper keeps retrying with backoff |
| // until a spawn sticks. |
| func TestActivatorSidecarFailureDoesNotFailWake(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _ := testActivator(t, time.Hour) |
| b.sidecarStaged = true |
| b.setSidecarErr(errors.New("exec: live symlink gone")) |
| @@ -2815,6 +2841,7 @@ func TestActivatorSidecarFailureDoesNotFailWake(t *testing.T) { |
| // next-due probe (so bots can't shift the schedule being recorded) and before |
| // the instance's own stop (so the sidecar never outlives its instance). |
| func TestActivatorCoStopOrdering(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _ := testActivator(t, 10*time.Second) |
| b.sidecarStaged = true |
| @@ -2843,6 +2870,7 @@ func TestActivatorCoStopOrdering(t *testing.T) { |
| // sidecar with it. An orphaned sidecar would reconnect-loop against the edge |
| // forever (503s at best; a wake source if the marker rule ever broke). |
| func TestActivatorInstanceDeathStopsSidecar(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| b.sidecarStaged = true |
| @@ -2867,6 +2895,7 @@ func TestActivatorInstanceDeathStopsSidecar(t *testing.T) { |
| // capped backoff — Restart=always semantics, sweep-driven — and a run that |
| // held for sidecarHealthyRun earns a fresh backoff. |
| func TestActivatorSidecarRestartBackoff(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _ := testActivator(t, time.Hour) |
| b.sidecarStaged = true |
| @@ -2953,6 +2982,7 @@ func TestActivatorSidecarRestartBackoff(t *testing.T) { |
| // stamps (stamps are per-request), so the marker is not a DoS lever in either |
| // direction. |
| func TestActivatorSidecarRequestsInvisibleToActivity(t *testing.T) { |
| + t.Parallel() |
| requireUnixSockets(t) |
| a, b, clk, reg := testActivator(t, 10*time.Second) |
| @@ -3051,6 +3081,7 @@ func TestActivatorSidecarRequestsInvisibleToActivity(t *testing.T) { |
| // DB parking, singleflight). |
| func TestWorkerWakeUsesStartWorkerAndNoProbe(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, 10*time.Second) |
| rt := testWorkerRoute("bot.x.internal") |
| if err := a.WakeWorker(rt); err != nil { |
| @@ -3069,6 +3100,7 @@ func TestWorkerWakeUsesStartWorkerAndNoProbe(t *testing.T) { |
| // authorize it. WorkerToken must agree with what the child was told, or every |
| // call the sidecar makes is a 401. |
| func TestWorkerWakeThreadsTokenAndSocketEnv(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| if err := a.WakeWorker(testWorkerRoute("bot.x.internal")); err != nil { |
| t.Fatalf("WakeWorker: %v", err) |
| @@ -3094,6 +3126,7 @@ func TestWorkerWakeThreadsTokenAndSocketEnv(t *testing.T) { |
| } |
| func TestWorkerSleepsWhenIdleAndRecordsNextDue(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, reg := testActivator(t, 10*time.Second) |
| rt := testWorkerRoute("bot.x.internal") |
| if err := reg.Upsert(rt); err != nil { |
| @@ -3129,6 +3162,7 @@ func TestWorkerSleepsWhenIdleAndRecordsNextDue(t *testing.T) { |
| } |
| func TestWorkerContactDefersSleep(t *testing.T) { |
| + t.Parallel() |
| a, _, clk, _ := testActivator(t, 10*time.Second) |
| if err := a.WakeWorker(testWorkerRoute("bot.x.internal")); err != nil { |
| t.Fatalf("WakeWorker: %v", err) |
| @@ -3148,6 +3182,7 @@ func TestWorkerContactDefersSleep(t *testing.T) { |
| // on the route (Task 1) precisely so the sweeper never needs a bucket read to |
| // decide a worker has sat idle long enough. |
| func TestWorkerRouteIdleWindowOverridesPlatformDefault(t *testing.T) { |
| + t.Parallel() |
| a, _, clk, _ := testActivator(t, time.Hour) // the platform default would never fire |
| rt := testWorkerRoute("bot.x.internal") |
| rt.IdleSeconds = 30 |
| @@ -3165,6 +3200,7 @@ func TestWorkerRouteIdleWindowOverridesPlatformDefault(t *testing.T) { |
| } |
| func TestWorkerMaxRunStopsALoopingWorker(t *testing.T) { |
| + t.Parallel() |
| a, _, clk, _ := testActivator(t, time.Hour) // idle window never fires |
| rt := testWorkerRoute("bot.x.internal") |
| rt.MaxRunSeconds = 60 |
| @@ -3182,6 +3218,7 @@ func TestWorkerMaxRunStopsALoopingWorker(t *testing.T) { |
| // No ceiling declared (0) means no ceiling — an always-on worker must not be |
| // stopped by a default nobody asked for. |
| func TestWorkerWithoutMaxRunKeepsRunning(t *testing.T) { |
| + t.Parallel() |
| a, _, clk, _ := testActivator(t, time.Hour) |
| if err := a.WakeWorker(testWorkerRoute("bot.x.internal")); err != nil { |
| t.Fatalf("WakeWorker: %v", err) |
| @@ -3197,6 +3234,7 @@ func TestWorkerWithoutMaxRunKeepsRunning(t *testing.T) { |
| } |
| func TestSleepWorkerIsImmediate(t *testing.T) { |
| + t.Parallel() |
| a, _, _, _ := testActivator(t, time.Hour) |
| if err := a.WakeWorker(testWorkerRoute("bot.x.internal")); err != nil { |
| t.Fatalf("WakeWorker: %v", err) |
| @@ -3206,6 +3244,7 @@ func TestSleepWorkerIsImmediate(t *testing.T) { |
| } |
| func TestWakeWorkerIsIdempotent(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| rt := testWorkerRoute("bot.x.internal") |
| for i := 0; i < 3; i++ { |
| @@ -3223,6 +3262,7 @@ func TestWakeWorkerIsIdempotent(t *testing.T) { |
| // an operator for a worker that cannot write is the one answer that makes a |
| // dead sidecar look healthy. |
| func TestWakeWorkerStandsDownALostLeaseAndRewakes(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| rt := testWorkerRoute("bot.x.internal") |
| if err := a.WakeWorker(rt); err != nil { |
| @@ -3247,6 +3287,7 @@ func TestWakeWorkerStandsDownALostLeaseAndRewakes(t *testing.T) { |
| // Concurrent wakes coalesce on one entry, the same singleflight the request |
| // path gets: three goroutines, one child, one lease. |
| func TestWakeWorkerCoalescesConcurrentCallers(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| rt := testWorkerRoute("bot.x.internal") |
| @@ -3311,6 +3352,7 @@ func TestWorkerWakeFailureReleasesLeaseAndClearsEntry(t *testing.T) { |
| // A worker is nobody's parent: the legacy co-spawn step must not run for one, |
| // even on a box where the staging file happens to exist. |
| func TestWorkerNeverCoSpawnsASidecar(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| b.sidecarStaged = true |
| @@ -3328,6 +3370,7 @@ func TestWorkerNeverCoSpawnsASidecar(t *testing.T) { |
| // be in `keep`. Without an explicit exemption every edge reload (any route |
| // change at all) would stop every running standalone sidecar. |
| func TestRetireExceptLeavesWorkersAlone(t *testing.T) { |
| + t.Parallel() |
| a, _, _, _ := testActivator(t, time.Hour) |
| if err := a.WakeWorker(testWorkerRoute("bot.x.internal")); err != nil { |
| t.Fatalf("WakeWorker: %v", err) |
| @@ -3350,6 +3393,7 @@ func TestRetireExceptLeavesWorkersAlone(t *testing.T) { |
| // one), so a due sidecar row must come up through the worker path — StartWorker |
| // and no health probe — not the instance one. |
| func TestDueSweepWakesASidecarAsAWorker(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, reg := testActivator(t, time.Hour) |
| rt := testWorkerRoute("bot.x.internal") |
| if err := reg.Upsert(rt); err != nil { |
| @@ -3373,6 +3417,7 @@ func TestDueSweepWakesASidecarAsAWorker(t *testing.T) { |
| // acquire and one S3 restore/park per cycle, and nothing in the logs that |
| // reads as a failure. |
| func TestDueFiredWorkerWakeHappensExactlyOnce(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, reg := testActivator(t, 10*time.Second) |
| rt := testWorkerRoute("bot.x.internal") |
| if err := reg.Upsert(rt); err != nil { |
| @@ -3414,6 +3459,7 @@ func TestDueFiredWorkerWakeHappensExactlyOnce(t *testing.T) { |
| // the wake the timer caused) would delete the feature — a sleeping sidecar's |
| // next_due is its only way back up, since nothing can dial one. |
| func TestWorkerSleepingBeforeItsDueTimeStillWakesForIt(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, reg := testActivator(t, 10*time.Second) |
| rt := testWorkerRoute("bot.x.internal") |
| if err := reg.Upsert(rt); err != nil { |
| @@ -3449,6 +3495,7 @@ func TestWorkerSleepingBeforeItsDueTimeStillWakesForIt(t *testing.T) { |
| // whose only way back up has been erased, silently, by a wake that never |
| // happened. |
| func TestFailedDueFiredWakeConsumesNothing(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, reg := testActivator(t, 10*time.Second) |
| rt := testWorkerRoute("bot.x.internal") |
| if err := reg.Upsert(rt); err != nil { |
| @@ -3492,6 +3539,7 @@ func TestFailedDueFiredWakeConsumesNothing(t *testing.T) { |
| // the agent socket during the settle window, and an unconditional clear would |
| // eat it — a bot that ran once and never again. |
| func TestDueFiredWakeKeepsANewNextDuePostedByTheChild(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, reg := testActivator(t, 10*time.Second) |
| rt := testWorkerRoute("bot.x.internal") |
| if err := reg.Upsert(rt); err != nil { |
| @@ -3665,6 +3713,7 @@ func alwaysOnWorkerRoute(host string) registry.Route { |
| // in place, keeping its lease and replication — and the backoff GROWS, so a |
| // crash-looping bot paces itself instead of hot-looping the box. |
| func TestAlwaysOnWorkerCrashRestartsWithGrowingBackoff(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _ := testActivator(t, time.Hour) |
| if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil { |
| t.Fatalf("WakeWorker: %v", err) |
| @@ -3721,6 +3770,7 @@ func waitForRestart(t *testing.T, a *Service, b *fakeBackend, host string, want |
| // A run that lasted sidecarHealthyRun earns a fresh backoff — one crash a day |
| // must never escalate to the cap (the co-spawn supervisor's rule). |
| func TestAlwaysOnWorkerHealthyRunResetsBackoff(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _ := testActivator(t, time.Hour) |
| if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil { |
| t.Fatalf("WakeWorker: %v", err) |
| @@ -3740,6 +3790,7 @@ func TestAlwaysOnWorkerHealthyRunResetsBackoff(t *testing.T) { |
| // A restart is a new incarnation: a fresh token, so the dead child's |
| // credential can never speak for the live one. |
| func TestAlwaysOnWorkerRestartMintsANewToken(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil { |
| t.Fatalf("WakeWorker: %v", err) |
| @@ -3761,6 +3812,7 @@ func TestAlwaysOnWorkerRestartMintsANewToken(t *testing.T) { |
| // A HIBERNATING worker's exit means "I finished my work" — the existing |
| // dead-child stand-down is exactly right, and must not become a restart loop. |
| func TestHibernatingWorkerExitSleepsRatherThanRestarting(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| if err := a.WakeWorker(testWorkerRoute("bot.x.internal")); err != nil { // Hibernate: true |
| t.Fatalf("WakeWorker: %v", err) |
| @@ -3781,6 +3833,7 @@ func TestHibernatingWorkerExitSleepsRatherThanRestarting(t *testing.T) { |
| // something whose record says it must be running, and a stopped one has no |
| // wake source at all (no request can reach it, and it has no schedule). |
| func TestAlwaysOnWorkerIgnoresTheIdleClock(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _ := testActivator(t, 10*time.Second) |
| rt := alwaysOnWorkerRoute("bot.x.internal") |
| rt.IdleSeconds = 5 |
| @@ -3805,6 +3858,7 @@ func TestAlwaysOnWorkerIgnoresTheIdleClock(t *testing.T) { |
| // a crash restarts, unqualified. Read as a finish, it would sleep forever: |
| // it fell over before it could set a next-due, and nothing can dial one. |
| func TestHibernatingWorkerCrashRestartsInsteadOfSleeping(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| if err := a.WakeWorker(testWorkerRoute("bot.x.internal")); err != nil { // Hibernate: true |
| t.Fatalf("WakeWorker: %v", err) |
| @@ -3825,6 +3879,7 @@ func TestHibernatingWorkerCrashRestartsInsteadOfSleeping(t *testing.T) { |
| // exactly the shape that gets wedged. It is spent as a RECYCLE there, not a |
| // stop: stopping one leaves nothing to start it again. |
| func TestAlwaysOnWorkerMaxRunRecyclesItsChild(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _ := testActivator(t, time.Hour) |
| rt := alwaysOnWorkerRoute("bot.x.internal") |
| rt.MaxRunSeconds = 30 |
| @@ -3859,6 +3914,7 @@ func TestAlwaysOnWorkerMaxRunRecyclesItsChild(t *testing.T) { |
| // sidecar created while the agent was running had nothing at all that would |
| // start it. |
| func TestSweepStartsAnAlwaysOnWorkerThatAppearedMidFlight(t *testing.T) { |
| + t.Parallel() |
| a, b, _, reg := testActivator(t, time.Hour) |
| rt := alwaysOnWorkerRoute("bot.x.internal") |
| if err := reg.Upsert(rt); err != nil { // the reconcile pass mints it, mid-flight |
| @@ -3904,6 +3960,7 @@ func TestSweepNeverStartsAHibernatingWorker(t *testing.T) { |
| // A stranded always-on worker is a bounded batch, not a stampede: each wake |
| // claims an S3 lease and restores a database. |
| func TestSweepBoundsAlwaysOnStartsPerTick(t *testing.T) { |
| + t.Parallel() |
| a, b, _, reg := testActivator(t, time.Hour) |
| for i := 0; i < maxWorkerStartsPerSweep+3; i++ { |
| rt := alwaysOnWorkerRoute(fmt.Sprintf("bot%02d.x.internal", i)) |
| @@ -3958,6 +4015,7 @@ func TestWorkerCrashRestartSpawnsOffTheActivatorLock(t *testing.T) { |
| // starting a process that would write the tenant's DB with no single-writer |
| // claim behind it — and possibly outlive the agent as an orphan. |
| func TestShutdownStopsARespawnFromOutlivingTheLease(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, ver := shippingActivator(t, time.Hour) |
| if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil { |
| t.Fatalf("WakeWorker: %v", err) |
| @@ -4005,6 +4063,7 @@ func TestShutdownStopsARespawnFromOutlivingTheLease(t *testing.T) { |
| // pointed at the same database. Narrowing that window is not closing it: the |
| // shed has to complete first. |
| func TestRespawnShedsARacedChildBeforeTheLeaseIsReleased(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, ver := shippingActivator(t, time.Hour) |
| if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil { |
| t.Fatalf("WakeWorker: %v", err) |
| @@ -4068,6 +4127,7 @@ func TestRespawnShedsARacedChildBeforeTheLeaseIsReleased(t *testing.T) { |
| // goroutine reads the entry second. No invariant is bent to produce it; only |
| // the schedule is pinned. |
| func TestStopScheduledAfterTheInterlockClearStillWaitsForTheShed(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, ver := shippingActivator(t, time.Hour) |
| if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil { |
| t.Fatalf("WakeWorker: %v", err) |
| @@ -4196,6 +4256,7 @@ func TestShutdownCancelsAnInFlightRespawnInsteadOfWaiting(t *testing.T) { |
| // Shutdown's announcement does not cover — and, unlike a shutdown, it has no |
| // deadline, so here the stop WAITS the spawn out rather than cancelling it. |
| func TestStopWaitsOutAnInFlightRespawn(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, ver := shippingActivator(t, time.Hour) |
| if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil { |
| t.Fatalf("WakeWorker: %v", err) |
| @@ -4270,6 +4331,7 @@ func shippingActivatorWithRegistry(t *testing.T, idle time.Duration) (*Service, |
| // same entry, same lease — because the ordinary cycle's second half ("a |
| // request wakes it onto the new build") does not exist for a worker. |
| func TestAlwaysOnWorkerRespawnsOntoANewBuild(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, ver := shippingActivator(t, time.Hour) |
| if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil { |
| t.Fatalf("WakeWorker: %v", err) |
| @@ -4316,6 +4378,7 @@ func TestAlwaysOnWorkerRespawnsOntoANewBuild(t *testing.T) { |
| // The one-per-sweep budget still applies: a bad ship must not respawn every |
| // sidecar on the box at once. |
| func TestWorkerConvergeRespawnIsRateLimitedPerSweep(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, ver := shippingActivator(t, time.Hour) |
| for _, host := range []string{"bot-a.x.internal", "bot-b.x.internal"} { |
| if err := a.WakeWorker(alwaysOnWorkerRoute(host)); err != nil { |
| @@ -4341,6 +4404,7 @@ func TestWorkerConvergeRespawnIsRateLimitedPerSweep(t *testing.T) { |
| // A HIBERNATING worker's convergence is unchanged: it is CYCLED (stopped, DB |
| // parked, lease released) and comes back on its schedule, exactly as before. |
| func TestHibernatingWorkerStillConvergenceCycles(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, ver := shippingActivator(t, time.Hour) |
| if err := a.WakeWorker(testWorkerRoute("bot.x.internal")); err != nil { // Hibernate: true |
| t.Fatalf("WakeWorker: %v", err) |
| @@ -4364,6 +4428,7 @@ func TestHibernatingWorkerStillConvergenceCycles(t *testing.T) { |
| // NOTHING: read as a crash it would start a second child, and the box would |
| // end up running two of a bot that must run once. |
| func TestWorkerConvergeRespawnIsNotMistakenForACrash(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, ver := shippingActivator(t, time.Hour) |
| if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil { |
| t.Fatalf("WakeWorker: %v", err) |
| @@ -4393,6 +4458,7 @@ func TestWorkerConvergeRespawnIsNotMistakenForACrash(t *testing.T) { |
| // place, and the ordinary crash-restart path picks it up on a later tick with |
| // its backoff. Convergence degrades into supervision. |
| func TestFailedConvergeRespawnFallsBackToCrashRestart(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, ver := shippingActivator(t, time.Hour) |
| if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil { |
| t.Fatalf("WakeWorker: %v", err) |
| @@ -4425,6 +4491,7 @@ func TestFailedConvergeRespawnFallsBackToCrashRestart(t *testing.T) { |
| // the stop would delete its entry, leaving nothing — not even the crash |
| // restart — to bring it back before the agent restarts. |
| func TestSleepWorkerIgnoresAnAlwaysOnWorker(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil { |
| t.Fatalf("WakeWorker: %v", err) |
| @@ -4893,6 +4960,7 @@ func TestExecBackendStartWorkerEnvWinsOverTenantFile(t *testing.T) { |
| // them) — a crash with no event is a status word with no explanation. A |
| // nonzero exit must land a warn-level event naming the worker's host. |
| func TestExecBackendStartWorkerEmitsExitEvent(t *testing.T) { |
| + t.Parallel() |
| b := testExecBackend(t) |
| sink := &fakeSink{} |
| b.Logs = sink |
| @@ -5055,6 +5123,9 @@ func TestExecBackendStartPassesControlSocketInEnvAndStartsPurgeListener(t *testi |
| gotArgv := strings.Fields(waitForFile(t, argvPath, time.Second)) |
| gotEnv := strings.TrimSpace(waitForFile(t, envPath, time.Second)) |
| + // Everything above was seen while the health wait ran; the fake never |
| + // binds, so end the wait rather than sit out its timeout. |
| + cancel() |
| <-done |
| if proc != nil { |
| defer proc.Stop(context.Background()) |
| @@ -5128,6 +5199,9 @@ func TestExecBackendStartWithNilCacheStillBindsControlSocket(t *testing.T) { |
| } |
| } |
| + // Everything above was seen while the health wait ran; the fake never |
| + // binds, so end the wait rather than sit out its timeout. |
| + cancel() |
| <-done |
| if proc != nil { |
| defer proc.Stop(context.Background()) |
| @@ -5315,7 +5389,7 @@ func TestExecBackendStartMergesConfigEnvUnderOwnEnv(t *testing.T) { |
| Socket: filepath.Join(t.TempDir(), "instance.sock"), |
| } |
| - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) |
| + ctx, cancel := recordedCtx(2*time.Second, envPath) |
| defer cancel() |
| proc, _, _ := b.Start(ctx, rt) // health-wait error from the fake binary is expected and irrelevant here |
| if proc != nil { |
| @@ -5362,7 +5436,7 @@ func TestExecBackendStartAdvertisesHeaderPurgeChannel(t *testing.T) { |
| Socket: tmpSock(t), |
| } |
| - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) |
| + ctx, cancel := recordedCtx(2*time.Second, envPath) |
| defer cancel() |
| proc, _, _ := b.Start(ctx, rt) // health-wait error from the fake binary is expected and irrelevant here |
| if proc != nil { |
| @@ -5401,7 +5475,7 @@ func TestExecBackendStartWithNilCacheSetsNoPurgeVar(t *testing.T) { |
| Socket: filepath.Join(t.TempDir(), "instance.sock"), |
| } |
| - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) |
| + ctx, cancel := recordedCtx(2*time.Second, envPath) |
| defer cancel() |
| proc, _, _ := b.Start(ctx, rt) |
| if proc != nil { |
| @@ -5446,7 +5520,7 @@ func TestExecBackendStartTenantCannotClaimAbsentPurgeChannel(t *testing.T) { |
| Socket: tmpSock(t), |
| } |
| - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) |
| + ctx, cancel := recordedCtx(2*time.Second, envPath) |
| defer cancel() |
| proc, _, _ := b.Start(ctx, rt) |
| if proc != nil { |
| @@ -5494,7 +5568,7 @@ func TestExecBackendStartOverridesTenantSuppliedPurgeVars(t *testing.T) { |
| Socket: tmpSock(t), |
| } |
| - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) |
| + ctx, cancel := recordedCtx(2*time.Second, envPath) |
| defer cancel() |
| proc, _, _ := b.Start(ctx, rt) |
| if proc != nil { |
| @@ -5535,7 +5609,16 @@ func TestExecBackendStartPurgesHostCacheOnSpawn(t *testing.T) { |
| }) |
| } |
| - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) |
| + // GetFor is the cache's request-time lookup (the by-VaryValue one is |
| + // package-internal); both entries were stored from responses with no Vary, |
| + // so any request for the URL reaches them. |
| + cached := func(k edge.CacheKey) bool { |
| + _, ok := cache.GetFor(k, httptest.NewRequest("GET", "http://"+k.Host+k.Path, nil)) |
| + return ok |
| + } |
| + // The purge comes before the health wait, which the fake never answers: |
| + // once it has happened that wait has nothing left to show. |
| + ctx, cancel := ctxUntil(2*time.Second, func() bool { return !cached(stale) }) |
| defer cancel() |
| rt := registry.Route{ |
| Host: "deploy.example.com", |
| @@ -5547,13 +5630,6 @@ func TestExecBackendStartPurgesHostCacheOnSpawn(t *testing.T) { |
| defer proc.Stop(context.Background()) |
| } |
| - // GetFor is the cache's request-time lookup (the by-VaryValue one is |
| - // package-internal); both entries were stored from responses with no Vary, |
| - // so any request for the URL reaches them. |
| - cached := func(k edge.CacheKey) bool { |
| - _, ok := cache.GetFor(k, httptest.NewRequest("GET", "http://"+k.Host+k.Path, nil)) |
| - return ok |
| - } |
| if cached(stale) { |
| t.Fatal("spawning a fresh process for a host must drop that host's cached responses") |
| } |
| @@ -6029,6 +6105,22 @@ func TestMain(m *testing.M) { |
| os.Exit(m.Run()) |
| } |
| +// Which tests call t.Parallel(). A parallel test runs beside the other |
| +// parallel tests once every sequential one has finished, so it may not |
| +// write package state (a shrunk timeout, a hook, the generations switch, |
| +// the log's output), set the environment or the working directory, depend |
| +// on time it measures itself (time.Since), or use the /proc child waits in |
| +// childwait_test.go, which count every child of this binary. Nor may it |
| +// bind a socket through edge.SecureUnixListen (a control, log or cmd-stream |
| +// listener: every ExecBackend spawn and nested start binds one), which sets |
| +// the process's umask to 0177 around the bind, so a directory a parallel |
| +// test made in that window comes out 0600 and unusable. A test that does |
| +// any of these stays sequential. The tests marked parallel were checked |
| +// for all of it, through every test helper they call, and run once each |
| +// to see that none reaches SecureUnixListen; a test that later starts |
| +// shrinking a package var or spawning through the backend must drop its |
| +// t.Parallel(). |
| + |
| // fakeAppMain is the app half: bind the --socket, tighten it to 0600 — the |
| // exact titogo habit that stranded the sibling router in issue #103 — and |
| // serve 200s until killed (500s under ACTIVATOR_FAKE_APP_UNHEALTHY, the |
| @@ -6363,6 +6455,7 @@ func TestChildEnvKeepsOneEntryPerKeyLastWins(t *testing.T) { |
| // released: stop() keeps the entry in stopping and the sweeper finishes it |
| // once the proc reports exited (design §6.3, codex R2-5). |
| func TestStopDoesNotReleaseTheLeaseOverAnUnprovenStop(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _ := testActivator(t, time.Minute) |
| b.procStuck = true // Start returns a fakeProc whose Stop reports ErrStillRunning until exit() |
| rt := testRoute("h.example.com") |
| @@ -6396,6 +6489,7 @@ func TestStopDoesNotReleaseTheLeaseOverAnUnprovenStop(t *testing.T) { |
| // retries proc.Stop on its cadence until done closes"). Without the retry, a |
| // stop whose first Control.Stop call itself failed would never ask again. |
| func TestFinishPendingStopsRetriesTheStopWhileStillRunning(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _ := testActivator(t, time.Minute) |
| b.procStuck = true // Stop reports ErrStillRunning until exit(), and this proc never exits here |
| rt := testRoute("stuck.example.com") |
| @@ -6431,6 +6525,7 @@ func TestFinishPendingStopsRetriesTheStopWhileStillRunning(t *testing.T) { |
| // release before Shutdown returns, via the same finishPendingStops pass |
| // Shutdown's own drain loop re-runs on its poll ticker. |
| func TestShutdownDrainsAPendingStopThatExitsMidShutdown(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _ := testActivator(t, time.Minute) |
| b.procStuck = true |
| rt := testRoute("shutdown-pending.example.com") |
| @@ -6627,6 +6722,7 @@ func (b *seedCountingBackend) SeedPlaced(_ context.Context, rt registry.Route, r |
| } |
| func TestDoWakeRecordsSeedPlacementAfterReplication(t *testing.T) { |
| + t.Parallel() |
| for _, recErr := range []error{nil, errors.New("record unreachable")} { |
| b := &seedCountingBackend{err: recErr} |
| reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| @@ -6751,6 +6847,7 @@ func TestAFlagOffWakeProvesTheHostCgroupBeforeItRestores(t *testing.T) { |
| // in carlos-exec@<host> (nested, unit-backed) asks no proof, and neither |
| // does a wake with generations on, whose proofs are its own. |
| func TestAWakeAsksTheHostProofOnlyFlagOffForAnExecChild(t *testing.T) { |
| + t.Parallel() |
| t.Run("no delegation", func(t *testing.T) { |
| a, b, _, _ := testActivator(t, time.Minute) |
| h := &hostEmptyFake{fakeBackend: b} |
| @@ -6800,6 +6897,7 @@ func (b *fakeBackend) lateSidecarLabel(label string) { |
| // lease held; the sweep retries, and the lease goes only after a proof |
| // that holds. |
| func TestAFlagOffStopKeepsTheLeaseUntilTheHostCgroupIsProvenEmpty(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _ := testActivator(t, time.Minute) |
| h := &hostEmptyFake{fakeBackend: b} |
| a.backend = h |
| @@ -6834,6 +6932,7 @@ func TestAFlagOffStopKeepsTheLeaseUntilTheHostCgroupIsProvenEmpty(t *testing.T) |
| // A passing proof releases the lease in the stop itself, after the proof. |
| func TestAFlagOffStopProvesTheHostCgroupBeforeTheLeaseGoes(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _ := testActivator(t, time.Minute) |
| h := &hostEmptyFake{fakeBackend: b} |
| a.backend = h |
| @@ -6855,6 +6954,7 @@ func TestAFlagOffStopProvesTheHostCgroupBeforeTheLeaseGoes(t *testing.T) { |
| // No delegated cgroup (cgroup.ErrNoDelegation): the leader's exit is the |
| // only proof there is (ledger 127), and the lease goes at once. |
| func TestAFlagOffStopWithNoDelegationReleasesAtOnce(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _ := testActivator(t, time.Minute) |
| h := &hostEmptyFake{fakeBackend: b} |
| h.set(cgroup.ErrNoDelegation) |
| @@ -6874,6 +6974,7 @@ func TestAFlagOffStopWithNoDelegationReleasesAtOnce(t *testing.T) { |
| // carlos-exec@<host>: its stop asks no host proof, decided from the layout |
| // the wake launched on (actEntry.launchLayout, set at install). |
| func TestAFlagOffStopOfANestedRowAsksNoHostProof(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _ := testActivator(t, time.Minute) |
| b.layoutFn = func(registry.Route) (Layout, string) { return LayoutNested, "" } |
| h := &hostEmptyFake{fakeBackend: b} |
| @@ -6930,6 +7031,7 @@ func TestEmptyHostWithNoCgroupKillIsTheLeaderOnlyProof(t *testing.T) { |
| // The pending-stop path (a leader that was still running when stop() asked) |
| // is held to the same proof once the leader exits. |
| func TestAPendingFlagOffStopProvesTheHostCgroupBeforeTheLeaseGoes(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _ := testActivator(t, time.Minute) |
| h := &hostEmptyFake{fakeBackend: b} |
| a.backend = h |
diff --git a/internal/activator/admission_ordering_test.go b/internal/activator/admission_ordering_test.go| index 5a4cf7da5..dcd150bf0 100644 |
| --- a/internal/activator/admission_ordering_test.go |
| +++ b/internal/activator/admission_ordering_test.go |
| @@ -8,6 +8,7 @@ import ( |
| ) |
| func TestFenceInitialAdmissionTimeoutHasNoFence(t *testing.T) { |
| + t.Parallel() |
| a := &Service{} |
| a.mu.Lock() |
| ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) |
diff --git a/internal/activator/backend_exec.go b/internal/activator/backend_exec.go| index ee0d2ec6c..738ff0ccc 100644 |
| --- a/internal/activator/backend_exec.go |
| +++ b/internal/activator/backend_exec.go |
| @@ -1629,7 +1629,7 @@ func (b *ExecBackend) seedUnitFailed(ctx context.Context, rt registry.Route, dbP |
| } |
| // waitSeedCgroupEmpty polls the seed unit's cgroup.events until it reports |
| -// empty or actStopTimeout elapses, honouring ctx and b.ObservePoll like the |
| +// empty or roleEmptyBound elapses, honouring ctx and b.ObservePoll like the |
| // exit observer (unitproc.go). A timeout or a read error is returned to the |
| // caller as a log line, not a failure: it is the deciding result's |
| // confidence that is at stake here, not the wake itself. |
| @@ -1637,8 +1637,12 @@ func (b *ExecBackend) waitSeedCgroupEmpty(ctx context.Context, rt registry.Route |
| return b.waitRoleCgroupEmpty(ctx, rt, instancectl.RoleSeed) |
| } |
| +// roleEmptyBound is how long waitRoleCgroupEmpty waits for a role's cgroup |
| +// to empty: actStopTimeout. A var so tests can shrink it. |
| +var roleEmptyBound = actStopTimeout |
| + |
| // waitRoleCgroupEmpty polls role's unit's cgroup.events until it reports |
| -// empty or actStopTimeout elapses, honouring ctx and b.ObservePoll. |
| +// empty or roleEmptyBound elapses, honouring ctx and b.ObservePoll. |
| func (b *ExecBackend) waitRoleCgroupEmpty(ctx context.Context, rt registry.Route, role instancectl.Role) error { |
| events := b.CgroupEvents |
| if events == nil { |
| @@ -1650,7 +1654,8 @@ func (b *ExecBackend) waitRoleCgroupEmpty(ctx context.Context, rt registry.Route |
| } |
| slice := instancectl.SliceName(rt.Host) |
| unit := instancectl.UnitName(role, rt.Host) |
| - deadline := time.Now().Add(actStopTimeout) |
| + bound := roleEmptyBound |
| + deadline := time.Now().Add(bound) |
| for { |
| exists, populated, err := events(slice, unit) |
| if err != nil { |
| @@ -1660,7 +1665,7 @@ func (b *ExecBackend) waitRoleCgroupEmpty(ctx context.Context, rt registry.Route |
| return nil |
| } |
| if time.Now().After(deadline) { |
| - return fmt.Errorf("processes remained after %s", actStopTimeout) |
| + return fmt.Errorf("processes remained after %s", bound) |
| } |
| select { |
| case <-ctx.Done(): |
diff --git a/internal/activator/backend_exec_test.go b/internal/activator/backend_exec_test.go| index 60eec7b27..5c8de257f 100644 |
| --- a/internal/activator/backend_exec_test.go |
| +++ b/internal/activator/backend_exec_test.go |
| @@ -94,7 +94,7 @@ func TestExecBackendStartChildRegion(t *testing.T) { |
| t.Fatal(err) |
| } |
| rt := registry.Route{Host: host, Kind: "instance", Socket: filepath.Join(t.TempDir(), "instance.sock")} |
| - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) |
| + ctx, cancel := recordedCtx(2*time.Second, envPath) |
| defer cancel() |
| proc, _, _ := b.Start(ctx, rt) // fake binary never binds the socket; health-wait error expected |
| if proc != nil { |
| @@ -126,7 +126,7 @@ func TestExecBackendStartChildHandsTheHost(t *testing.T) { |
| t.Fatal(err) |
| } |
| rt := registry.Route{Host: host, Kind: "instance", Socket: filepath.Join(t.TempDir(), "instance.sock")} |
| - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) |
| + ctx, cancel := recordedCtx(2*time.Second, envPath) |
| defer cancel() |
| proc, _, _ := b.Start(ctx, rt) // fake binary never binds the socket; health-wait error expected |
| if proc != nil { |
| @@ -513,10 +513,12 @@ func TestSpawnUnderAMoveIsStrictOrRefused(t *testing.T) { |
| spawnHealthTimeout = 300 * time.Millisecond |
| t.Cleanup(func() { spawnHealthTimeout = old }) |
| host := "strict.example.com" |
| - start := func(t *testing.T, b *ExecBackend) error { |
| + // argvPath is the recorder's first file: once it exists and the child |
| + // has exited, the health wait the recorder never answers is over. |
| + start := func(t *testing.T, b *ExecBackend, argvPath string) error { |
| t.Helper() |
| rt := registry.Route{Host: host, Kind: "instance", Socket: filepath.Join(t.TempDir(), "i.sock")} |
| - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) |
| + ctx, cancel := recordedCtx(2*time.Second, argvPath) |
| defer cancel() |
| proc, _, err := b.Start(ctx, rt) // the fake binary never binds: a health error is expected |
| if proc != nil { |
| @@ -526,7 +528,7 @@ func TestSpawnUnderAMoveIsStrictOrRefused(t *testing.T) { |
| } |
| noSpawn := func(t *testing.T, argvPath string) { |
| t.Helper() |
| - time.Sleep(200 * time.Millisecond) |
| + awaitChildFile(argvPath, 200*time.Millisecond) // until no child is left to write it |
| if _, err := os.Stat(argvPath); !os.IsNotExist(err) { |
| t.Fatalf("the child ran: %v", err) |
| } |
| @@ -537,7 +539,7 @@ func TestSpawnUnderAMoveIsStrictOrRefused(t *testing.T) { |
| argvPath, _ := stageRecordingLive(t, b, host, "v1") |
| b.Strict = func(h string) bool { return h == host } |
| b.prepareStrict = func(string, cgroup.Limit) (strictProc, error) { return nil, cgroup.ErrStrictRefused } |
| - if err := start(t, b); !errors.Is(err, ErrStrictRequired) { |
| + if err := start(t, b, argvPath); !errors.Is(err, ErrStrictRequired) { |
| t.Fatalf("Start: %v, want ErrStrictRequired", err) |
| } |
| noSpawn(t, argvPath) |
| @@ -547,7 +549,7 @@ func TestSpawnUnderAMoveIsStrictOrRefused(t *testing.T) { |
| b := testExecBackend(t) // Cgroups nil |
| argvPath, _ := stageRecordingLive(t, b, host, "v1") |
| b.Strict = func(string) bool { return true } |
| - if err := start(t, b); !errors.Is(err, ErrStrictRequired) { |
| + if err := start(t, b, argvPath); !errors.Is(err, ErrStrictRequired) { |
| t.Fatalf("Start: %v, want ErrStrictRequired", err) |
| } |
| noSpawn(t, argvPath) |
| @@ -560,7 +562,7 @@ func TestSpawnUnderAMoveIsStrictOrRefused(t *testing.T) { |
| rec := newRecordingStrict() |
| failing := &failingStrict{recordingStrict: rec} |
| b.prepareStrict = func(string, cgroup.Limit) (strictProc, error) { return failing, nil } |
| - if err := start(t, b); !errors.Is(err, ErrStrictRequired) { |
| + if err := start(t, b, argvPath); !errors.Is(err, ErrStrictRequired) { |
| t.Fatalf("Start: %v, want ErrStrictRequired", err) |
| } |
| noSpawn(t, argvPath) |
| @@ -578,7 +580,7 @@ func TestSpawnUnderAMoveIsStrictOrRefused(t *testing.T) { |
| var gotLimit cgroup.Limit |
| b.Tier = func(registry.Route) cgroup.Limit { return cgroup.TierChildren } |
| b.prepareStrict = func(h string, l cgroup.Limit) (strictProc, error) { gotHost, gotLimit = h, l; return rec, nil } |
| - _ = start(t, b) |
| + _ = start(t, b, argvPath) |
| waitForFile(t, argvPath, time.Second) |
| select { |
| case <-rec.closed: |
| @@ -609,7 +611,7 @@ func TestSpawnUnderAMoveIsStrictOrRefused(t *testing.T) { |
| asked = true |
| return nil, errors.New("must not be asked") |
| } |
| - _ = start(t, b) |
| + _ = start(t, b, argvPath) |
| waitForFile(t, argvPath, time.Second) |
| if asked { |
| t.Fatal("the strict seam was asked for a host not in a move") |
| @@ -761,7 +763,7 @@ func TestStrictCleanupIsRetriedAndRefusesUntilItSucceeds(t *testing.T) { |
| prepared := func() int { mu.Lock(); defer mu.Unlock(); return len(handles) } |
| startOnce := func() error { |
| rt := registry.Route{Host: host, Kind: "instance", Socket: filepath.Join(t.TempDir(), "i.sock")} |
| - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) |
| + ctx, cancel := recordedCtx(2*time.Second, argvPath) // the recorder ran and exited |
| defer cancel() |
| proc, _, err := b.Start(ctx, rt) |
| if proc != nil { |
| @@ -965,7 +967,7 @@ func TestPendingStrictCleanupOwnsTheGroup(t *testing.T) { |
| } |
| startOnce := func() error { |
| rt := registry.Route{Host: host, Kind: "instance", Socket: filepath.Join(t.TempDir(), "i.sock")} |
| - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) |
| + ctx, cancel := recordedCtx(2*time.Second, argvPath) // the recorder ran and exited |
| defer cancel() |
| proc, _, err := b.Start(ctx, rt) |
| if proc != nil { |
| @@ -998,7 +1000,7 @@ func TestPendingStrictCleanupOwnsTheGroup(t *testing.T) { |
| if err := startOnce(); !errors.Is(err, ErrStrictCleanupPending) { |
| t.Fatalf("ordinary spawn during pending cleanup: %v, want ErrStrictCleanupPending", err) |
| } |
| - time.Sleep(100 * time.Millisecond) |
| + awaitChildFile(argvPath, 100*time.Millisecond) // until no child is left to write it |
| if _, err := os.Stat(argvPath); !os.IsNotExist(err) { |
| t.Fatalf("an ordinary child ran in a group awaiting cleanup: %v", err) |
| } |
diff --git a/internal/activator/backend_legacy_test.go b/internal/activator/backend_legacy_test.go| index 20a98f426..93a1e1448 100644 |
| --- a/internal/activator/backend_legacy_test.go |
| +++ b/internal/activator/backend_legacy_test.go |
| @@ -85,6 +85,7 @@ func TestACancelledLegacyStartIsMarkedAndItsStopClearsTheQueuedJob(t *testing.T) |
| // The proof fails, and so keeps the lease, while the unit is not stopped |
| // or its cgroup still holds a process. |
| func TestLegacyRunGoneRefusesWhileTheUnitRuns(t *testing.T) { |
| + t.Parallel() |
| for _, tc := range []struct { |
| name string |
| active string |
diff --git a/internal/activator/backend_unit_test.go b/internal/activator/backend_unit_test.go| index 5d87bc3f1..bb32f82b9 100644 |
| --- a/internal/activator/backend_unit_test.go |
| +++ b/internal/activator/backend_unit_test.go |
| @@ -93,6 +93,7 @@ func testUnitActivator(t *testing.T, idle time.Duration) (*Service, *fakeBackend |
| // start, health-gated); the idle window stops it (systemctl stop); the next |
| // request starts it again. The exec side is never touched. |
| func TestUnitTenantWakeIdleSleepCycle(t *testing.T) { |
| + t.Parallel() |
| a, eb, sc, clk, _ := testUnitActivator(t, 10*time.Second) |
| route := testUnitRoute("kass.example.com") |
| unit := UnitName(route.Host) |
| @@ -140,6 +141,7 @@ func TestUnitTenantWakeIdleSleepCycle(t *testing.T) { |
| // SplitBackend dispatch, the other direction: an exec-backed route wakes |
| // through the fake exec backend and never touches systemctl. |
| func TestSplitBackendExecRouteBypassesUnit(t *testing.T) { |
| + t.Parallel() |
| a, eb, sc, _, _ := testUnitActivator(t, time.Minute) |
| release, err := a.beginRequest(context.Background(), testRoute("plain.example.com")) |
diff --git a/internal/activator/beforeexec_test.go b/internal/activator/beforeexec_test.go| index 6c90b47a6..3dd182181 100644 |
| --- a/internal/activator/beforeexec_test.go |
| +++ b/internal/activator/beforeexec_test.go |
| @@ -190,6 +190,7 @@ func TestAnUnwrittenBuildStateIsARefusal(t *testing.T) { |
| // #11: a hold swap's pinned wake records the attempt's build before its |
| // exec; a refused record abandons the attempt with nothing exec'd. |
| func TestBeforeExecRunsBeforeAHoldSwap(t *testing.T) { |
| + t.Parallel() |
| f := newSwapFixture(t) |
| l := &execLog{spawns: func() int { n, _, _ := f.b.calls(); return n }} |
| f.a.BeforeExec = l.hook |
| @@ -223,6 +224,7 @@ func TestBeforeExecRunsBeforeAHoldSwap(t *testing.T) { |
| // record is SwapAbandoned, never SwapFailed (which the scheduler writes as |
| // the build's failure), and the incumbent keeps serving. |
| func TestBeforeExecRunsBeforeAnOverlapSuccessor(t *testing.T) { |
| + t.Parallel() |
| f := newSwapFixture(t) |
| l := &execLog{spawns: func() int { n, _, _ := f.b.calls(); return n }} |
| f.a.BeforeExec = l.hook |
| @@ -254,6 +256,7 @@ func TestBeforeExecRunsBeforeAnOverlapSuccessor(t *testing.T) { |
| // #13: a planned boot records B before its exec; a refused record execs |
| // nothing and records nothing against B. |
| func TestBeforeExecRunsBeforeAPlannedBoot(t *testing.T) { |
| + t.Parallel() |
| dir := t.TempDir() |
| next := stagedID(t, dir, "v2") |
| a, b, _ := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap}) |
| @@ -289,6 +292,7 @@ func TestBeforeExecRunsBeforeAPlannedBoot(t *testing.T) { |
| // again before its own exec. A refused record starts no good build; B's own |
| // failure (it did fail) stays its own, and nothing falls back. |
| func TestBeforeExecRunsBeforeAPlannedFallback(t *testing.T) { |
| + t.Parallel() |
| dir := t.TempDir() |
| good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| a, b, evs := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good}) |
| @@ -336,6 +340,7 @@ func TestBeforeExecRunsBeforeAPlannedFallback(t *testing.T) { |
| // #5: a worker wake records the identity it is about to exec, after its |
| // publication; a refused record execs nothing and fails the wake. |
| func TestBeforeExecRunsBeforeAWorkerWake(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _, _ := stagedActivator(t) |
| host := "bot.x.internal" |
| l := &execLog{spawns: b.workerStarts} |
| @@ -365,6 +370,7 @@ func TestBeforeExecRunsBeforeAWorkerWake(t *testing.T) { |
| // refused record is a refusal: nothing exec'd, no "failed to respawn" |
| // member error event, and the sweep asks again. |
| func TestBeforeExecRunsBeforeAWorkerRecycle(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _, _ := stagedActivator(t) |
| sink := &lockedSink{} |
| a.Logs = sink |
diff --git a/internal/activator/childwait_test.go b/internal/activator/childwait_test.go| new file mode 100644 |
| index 000000000..cc7240d06 |
| --- /dev/null |
| +++ b/internal/activator/childwait_test.go |
| @@ -0,0 +1,135 @@ |
| +package activator |
| + |
| +import ( |
| + "bytes" |
| + "context" |
| + "os" |
| + "path/filepath" |
| + "strconv" |
| + "time" |
| +) |
| + |
| +// The fake apps these tests launch (stageRecordingLive, stageHelperApp and |
| +// their kin) record what they were handed to a file and exit. They never |
| +// bind their socket, so a Start against one waits out its whole health |
| +// context, and a test that wants to see that NO app ran used to wait out a |
| +// fixed timeout for a file that was never coming. Both waits had a real |
| +// condition behind them, and the helpers here wait for that instead: |
| +// |
| +// - the app has recorded: its file exists AND it has exited, so whatever |
| +// it was going to write is written; |
| +// - nothing can still record: no child of this test binary is running, |
| +// so no process exists that could write the file later. |
| +// |
| +// Both read /proc, not the code under test, so they rest on what the kernel |
| +// says ran, never on the launch's own account of itself. Where /proc cannot |
| +// answer, or an unrelated child of this binary is still running, they fall |
| +// back to the original bound, which only makes them slower. |
| + |
| +// runningChildren reports how many children of this test binary are still |
| +// running. A zombie (exited, not yet reaped) is not running: it has written |
| +// all it ever will. ok is false when /proc cannot be read. |
| +func runningChildren() (n int, ok bool) { |
| + entries, err := os.ReadDir("/proc") |
| + if err != nil { |
| + return 0, false |
| + } |
| + self := strconv.Itoa(os.Getpid()) |
| + for _, e := range entries { |
| + name := e.Name() |
| + if name == "" || name[0] < '0' || name[0] > '9' { |
| + continue |
| + } |
| + raw, err := os.ReadFile(filepath.Join("/proc", name, "stat")) |
| + if err != nil { |
| + continue // gone since the listing |
| + } |
| + // "pid (comm) state ppid ...": comm may hold spaces and parens. |
| + i := bytes.LastIndexByte(raw, ')') |
| + if i < 0 { |
| + continue |
| + } |
| + f := bytes.Fields(raw[i+1:]) |
| + if len(f) < 2 || string(f[1]) != self { |
| + continue |
| + } |
| + if s := string(f[0]); s != "Z" && s != "X" { |
| + n++ |
| + } |
| + } |
| + return n, true |
| +} |
| + |
| +// noChildRunning is runningChildren's "none, for certain". |
| +func noChildRunning() bool { |
| + n, ok := runningChildren() |
| + return ok && n == 0 |
| +} |
| + |
| +// awaitChildFile returns path's contents once it exists, or "" once no |
| +// child of this test binary is running, so nothing is left to write it, |
| +// or once within has passed (the old fixed wait, still the bound). Call it |
| +// only after the launch under test has returned: a launch still in flight |
| +// could spawn after the check. |
| +func awaitChildFile(path string, within time.Duration) string { |
| + deadline := time.Now().Add(within) |
| + for { |
| + if raw, err := os.ReadFile(path); err == nil { |
| + return string(raw) |
| + } |
| + if noChildRunning() { |
| + // A child that exited between the read and the check wrote its |
| + // file before it exited. |
| + if raw, err := os.ReadFile(path); err == nil { |
| + return string(raw) |
| + } |
| + return "" |
| + } |
| + if !time.Now().Before(deadline) { |
| + return "" |
| + } |
| + time.Sleep(5 * time.Millisecond) |
| + } |
| +} |
| + |
| +// recordedCtx is the bounded context a fake-app test hands Start, also |
| +// cancelled the moment the fake app has recorded: every path in paths |
| +// exists and no child of this test binary is still running. The health |
| +// wait the app never answers has nothing left to show after that; before, |
| +// the test sat out the whole timeout. Start then fails on the cancel |
| +// rather than the deadline, which these tests ignore. |
| +func recordedCtx(within time.Duration, paths ...string) (context.Context, context.CancelFunc) { |
| + return ctxUntil(within, func() bool { return allExist(paths) && noChildRunning() }) |
| +} |
| + |
| +// ctxUntil is a context bounded by within and also cancelled once done |
| +// reports true (polled every 5 ms): the condition a test was really waiting |
| +// for when it sat out the timeout. |
| +func ctxUntil(within time.Duration, done func() bool) (context.Context, context.CancelFunc) { |
| + ctx, cancel := context.WithTimeout(context.Background(), within) |
| + go func() { |
| + t := time.NewTicker(5 * time.Millisecond) |
| + defer t.Stop() |
| + for { |
| + select { |
| + case <-ctx.Done(): |
| + return |
| + case <-t.C: |
| + } |
| + if done() { |
| + cancel() |
| + return |
| + } |
| + } |
| + }() |
| + return ctx, cancel |
| +} |
| + |
| +func allExist(paths []string) bool { |
| + for _, p := range paths { |
| + if _, err := os.Stat(p); err != nil { |
| + return false |
| + } |
| + } |
| + return true |
| +} |
diff --git a/internal/activator/claimgen_test.go b/internal/activator/claimgen_test.go| index 5e49ec4bf..69a692689 100644 |
| --- a/internal/activator/claimgen_test.go |
| +++ b/internal/activator/claimgen_test.go |
| @@ -46,6 +46,7 @@ func claimedObs(host string) hostclaim.Observed { |
| // Without a runtime that builds per-generation starts, a claims-on swap is |
| // refused before anything is raised, stopped or spawned. |
| func TestClaimsOnSwapNeedsGenerations(t *testing.T) { |
| + t.Parallel() |
| for _, mode := range []SwapMode{ModeOverlap, ModeHold} { |
| f := newSwapFixture(t) |
| f.a.cfg.HostWrites = hostwrite.Config{Enabled: true, Selector: &swapSelector{work: &swapWork{}}} |
| @@ -138,6 +139,7 @@ func TestClaimGenAt(t *testing.T) { |
| // attempt is abandoned, never SwapFailed: a failure record would halt the |
| // cohort for a build that never executed. |
| func TestAClaimsRefusalAtTheExecIsNoFailure(t *testing.T) { |
| + t.Parallel() |
| for _, refusal := range []error{hostguard.ErrStampMismatch, hostguard.ErrFenced, hostguard.ErrNoClaim} { |
| f := newSwapFixture(t) |
| f.a.cfg.HostWrites = hostwrite.Config{Enabled: true, Selector: &swapSelector{work: &swapWork{}}} |
| @@ -368,6 +370,7 @@ func TestARetiringClaimedGenerationDrainsBeforeItsKill(t *testing.T) { |
| // parent raise, no stop of the incumbent, no spawn. The guard opens only |
| // with a working isolated launch, which this base does not have. |
| func TestEveryClaimsBackedSwapIsRefusedBeforeItsSideEffects(t *testing.T) { |
| + t.Parallel() |
| runtimes := []struct { |
| name string |
| r *ClaimRuntime |
| @@ -421,6 +424,7 @@ func TestEveryClaimsBackedSwapIsRefusedBeforeItsSideEffects(t *testing.T) { |
| // it: production must always meet claimsSwapRefusal's refusal. No |
| // non-test file of the package assigns the field. |
| func TestClaimSpawnIsATestSeam(t *testing.T) { |
| + t.Parallel() |
| fset := token.NewFileSet() |
| pkgs, err := parser.ParseDir(fset, ".", func(fi fs.FileInfo) bool { return !strings.HasSuffix(fi.Name(), "_test.go") }, 0) |
| if err != nil { |
diff --git a/internal/activator/claimloop_linux_test.go b/internal/activator/claimloop_linux_test.go| index 159b1b862..f51a84446 100644 |
| --- a/internal/activator/claimloop_linux_test.go |
| +++ b/internal/activator/claimloop_linux_test.go |
| @@ -79,11 +79,16 @@ func driveLoop(t *testing.T, policy startPolicy) (*loopRun, error) { |
| // Five starts, then the loop gives up, with the delay between starts |
| // doubling: 1, 2, 4 and 8 seconds. The loop's backoff is time.NewTimer |
| // inline (claimwake.go), with no clock or sleeper to substitute, so this |
| -// reads real time and takes 15 s; -short skips it. |
| +// reads real time. It runs on a quarter of the production base |
| +// (claimStartBackoff), so 0.25, 0.5, 1 and 2 s, each delay checked to |
| +// within 200 ms, which keeps every step apart from its neighbours: the |
| +// doubling is what is proven, on 3.75 s of waiting rather than 15. |
| func TestStartAdmittedLoopStopsAfterFiveStartsWithDoublingBackoff(t *testing.T) { |
| - if testing.Short() { |
| - t.Skip("takes 15 s of real backoff") |
| + if claimStartBackoff != time.Second { |
| + t.Fatalf("claimStartBackoff = %v: the production schedule is 1, 2, 4, 8 s", claimStartBackoff) |
| } |
| + base := 250 * time.Millisecond |
| + shrinkClaimStartBackoff(t, base) |
| run, err := driveLoop(t, startPolicy{MaxStarts: 5, Window: 30 * time.Second, Deadline: time.Now().Add(time.Hour), Probe: true}) |
| if !errors.Is(err, ErrClaimIsolationUnavailable) || errors.Is(err, ErrAttemptOver) { |
| t.Fatalf("the last start's refusal should end the loop: %v", err) |
| @@ -92,8 +97,8 @@ func TestStartAdmittedLoopStopsAfterFiveStartsWithDoublingBackoff(t *testing.T) |
| if len(at) != 5 || run.failed != 5 { |
| t.Fatalf("starts=%d failed=%d, want 5 and 5", len(at), run.failed) |
| } |
| - for i, want := range []time.Duration{time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second} { |
| - if gap := at[i+1].Sub(at[i]); gap < want || gap > want+400*time.Millisecond { |
| + for i, want := range []time.Duration{base, 2 * base, 4 * base, 8 * base} { |
| + if gap := at[i+1].Sub(at[i]); gap < want || gap > want+200*time.Millisecond { |
| t.Errorf("delay before start %d = %v, want %v", i+2, gap, want) |
| } |
| } |
diff --git a/internal/activator/claimproducer_test.go b/internal/activator/claimproducer_test.go| index e908d6504..ac788b7c9 100644 |
| --- a/internal/activator/claimproducer_test.go |
| +++ b/internal/activator/claimproducer_test.go |
| @@ -61,6 +61,7 @@ func TestClaimProducerCancellationStopsAndJoinsProcess(t *testing.T) { |
| } |
| func TestClaimProducerCancellationRetriesRootSettlement(t *testing.T) { |
| + t.Parallel() |
| p := &claimProcess{root: true, done: make(chan struct{})} |
| close(p.done) |
| var attempts int |
diff --git a/internal/activator/claimretry_instancekey_linux_test.go b/internal/activator/claimretry_instancekey_linux_test.go| index c29cc11b6..155eeb736 100644 |
| --- a/internal/activator/claimretry_instancekey_linux_test.go |
| +++ b/internal/activator/claimretry_instancekey_linux_test.go |
| @@ -25,6 +25,7 @@ import ( |
| // was established. This pins that shape of startAdmitted: one Prepare, |
| // and Materialize before each attempt's start. |
| func TestStartAdmittedRetriesReuseTheirInputsAndMaterializeFirst(t *testing.T) { |
| + t.Parallel() |
| var ( |
| mu sync.Mutex |
| events []string |
diff --git a/internal/activator/claimwake.go b/internal/activator/claimwake.go| index 77ee02923..414a71c00 100644 |
| --- a/internal/activator/claimwake.go |
| +++ b/internal/activator/claimwake.go |
| @@ -388,6 +388,11 @@ func (a *Service) doClaimWake(ctx context.Context, e *actEntry) (*wokenInstance, |
| return a.claimedPlanResult(e, e.claimResult), nil |
| } |
| +// claimStartBackoff is startAdmitted's wait before its second start; the |
| +// wait doubles before each start after that (1, 2, 4, 8 s). A var so tests |
| +// can shrink it. |
| +var claimStartBackoff = time.Second |
| + |
| func (a *Service) startAdmitted(ctx context.Context, e *actEntry, admitted hostclaim.Observed, policy startPolicy) (result error) { |
| r := a.cfg.Claims |
| if !a.cfg.HostWrites.Enabled || r == nil || r.Read == nil || r.Prepare == nil || e == nil || admitted.Host != e.host || policy.MaxStarts < 1 || policy.MaxStarts > 5 || policy.Window <= 0 || policy.Window > 30*time.Second { |
| @@ -695,7 +700,7 @@ func (a *Service) startAdmitted(ctx context.Context, e *actEntry, admitted hostc |
| if policy.Probe && !budget.opened.IsZero() { |
| waitCtx, cancelWait = context.WithDeadline(ctx, budget.opened.Add(policy.Window)) |
| } |
| - timer := time.NewTimer(time.Second << attempt) |
| + timer := time.NewTimer(claimStartBackoff << attempt) |
| select { |
| case <-waitCtx.Done(): |
| timer.Stop() |
diff --git a/internal/activator/cleanup_test.go b/internal/activator/cleanup_test.go| index 06e30357c..45672ddbc 100644 |
| --- a/internal/activator/cleanup_test.go |
| +++ b/internal/activator/cleanup_test.go |
| @@ -7,6 +7,7 @@ import ( |
| ) |
| func TestCleanupBudgetSharedAfterCancellation(t *testing.T) { |
| + t.Parallel() |
| parent, stop := context.WithCancel(t.Context()) |
| ctx, finish := WithCleanupBudget(parent, 80*time.Millisecond) |
| defer finish() |
diff --git a/internal/activator/cmd_swap_test.go b/internal/activator/cmd_swap_test.go| index d957f3b6a..2e066f9a4 100644 |
| --- a/internal/activator/cmd_swap_test.go |
| +++ b/internal/activator/cmd_swap_test.go |
| @@ -112,6 +112,7 @@ func TestARunStartInFlightRefusesAnOverlapSwap(t *testing.T) { |
| // and then runs the successor's build, admitted against the successor's |
| // generation (merge ruling M2). |
| func TestARunWaitsForAPendingSwitch(t *testing.T) { |
| + t.Parallel() |
| f, cb := newCmdSwapFixture(t) |
| f.b.gmu.Lock() |
| f.b.genGate = make(chan struct{}) |
| @@ -164,6 +165,7 @@ func TestARunWaitsForAPendingSwitch(t *testing.T) { |
| // restart for a scheduled restart or config change (exit 125, RUN-C75; |
| // merge ruling M2). |
| func TestARunOnARetiredGenerationIsInterrupted(t *testing.T) { |
| + t.Parallel() |
| for _, c := range []struct { |
| cause Cause |
| why string |
| @@ -202,6 +204,7 @@ func TestARunOnARetiredGenerationIsInterrupted(t *testing.T) { |
| // restages nor sweeps the bytes the command runs; the settle that proves |
| // it releases the pin (merge ruling M2). |
| func TestARetiredRunKeepsItsBuildPinnedUntilItSettles(t *testing.T) { |
| + t.Parallel() |
| f, cb := newCmdSwapFixture(t) |
| path := release.GenerationsPath(t.TempDir()) |
| f.a.GenerationsFile = path |
| @@ -239,6 +242,7 @@ func TestARetiredRunKeepsItsBuildPinnedUntilItSettles(t *testing.T) { |
| // before or after the switch, and returns its context's error. A Recheck |
| // that refuses after the wait starts nothing either (re-review 5). |
| func TestARunEndedDuringASwitchWaitNeverStarts(t *testing.T) { |
| + t.Parallel() |
| for _, c := range []struct { |
| name string |
| ctx func() (context.Context, context.CancelFunc) |
| @@ -322,6 +326,7 @@ func TestARunEndedDuringASwitchWaitNeverStarts(t *testing.T) { |
| // next run is admitted. The run's handle and the pin stay until then |
| // (re-review 6). |
| func TestARetiredRunsFailedStopIsSettledBySweeps(t *testing.T) { |
| + t.Parallel() |
| for _, released := range []bool{false, true} { |
| t.Run(map[bool]string{false: "attached", true: "released"}[released], func(t *testing.T) { |
| f, cb := newCmdSwapFixture(t) |
| @@ -488,6 +493,7 @@ func TestARunRecordsItsServingBuildBeforeItStarts(t *testing.T) { |
| // next run are unaffected. A stop, or the run's own cancel, that lands |
| // during the write starts nothing either. |
| func TestARunWhoseBuildRecordFailsOrIsOvertakenNeverStarts(t *testing.T) { |
| + t.Parallel() |
| t.Run("the write fails", func(t *testing.T) { |
| f, cb := newCmdSwapFixture(t) |
| rec := &buildRecorder{err: errors.New("bucket: 503")} |
diff --git a/internal/activator/cmd_test.go b/internal/activator/cmd_test.go| index e34e7dc6e..82b7f1a02 100644 |
| --- a/internal/activator/cmd_test.go |
| +++ b/internal/activator/cmd_test.go |
| @@ -192,6 +192,7 @@ func TestRunCmdRefusesASecondRun(t *testing.T) { |
| } |
| func TestRunCmdHoldsTheIdleTimer(t *testing.T) { |
| + t.Parallel() |
| a, _, clk := testCmdActivator(t, Config{}) |
| rt := testRoute("c.example.com") |
| h, err := a.RunCmd(context.Background(), rt, CmdSpec{RunID: "run_a"}) |
| @@ -212,6 +213,7 @@ func TestRunCmdHoldsTheIdleTimer(t *testing.T) { |
| // Every stop path stops and settles the run before replication closes, and |
| // the run learns why (run spec §5.3, codex 1). |
| func TestEveryStopPathSettlesTheRunBeforeReplicationCloses(t *testing.T) { |
| + t.Parallel() |
| cases := []struct { |
| name string |
| why string |
| @@ -270,6 +272,7 @@ func TestEveryStopPathSettlesTheRunBeforeReplicationCloses(t *testing.T) { |
| // The hold stamps the first stop asked for with the activator's clock, so |
| // the run can tell an instance stop that came after its command finished. |
| func TestRunCmdInterruptedSaysWhenTheStopWasAsked(t *testing.T) { |
| + t.Parallel() |
| a, _, clk := testCmdActivator(t, Config{}) |
| rt := testRoute("c.example.com") |
| h, err := a.RunCmd(context.Background(), rt, CmdSpec{RunID: "run_a"}) |
| @@ -306,6 +309,7 @@ func TestRunCmdNeverStartsUnderAStoppingEntry(t *testing.T) { |
| } |
| func TestAStopDuringTheStartWaitsForIt(t *testing.T) { |
| + t.Parallel() |
| a, cb, _ := testCmdActivator(t, Config{}) |
| cb.gate = make(chan struct{}) |
| cb.entered = make(chan struct{}) |
| @@ -345,6 +349,7 @@ func TestAStopDuringTheStartWaitsForIt(t *testing.T) { |
| // closes replication and releases the lease (merge ruling M1). That holds |
| // with generations off and on. |
| func TestAnUnprovenRunStopKeepsReplicationAndTheLease(t *testing.T) { |
| + t.Parallel() |
| for _, gens := range []bool{false, true} { |
| t.Run(map[bool]string{false: "generations off", true: "generations on"}[gens], func(t *testing.T) { |
| a, cb, _ := testCmdActivator(t, Config{}) |
| @@ -400,6 +405,7 @@ func TestAnUnprovenRunStopKeepsReplicationAndTheLease(t *testing.T) { |
| // before replication closes, and holds replication and the lease until |
| // it has. |
| func TestAReleasedRunThatNeverSettledStillHoldsTheStop(t *testing.T) { |
| + t.Parallel() |
| for _, c := range []struct { |
| name string |
| stopFirst bool |
| @@ -469,6 +475,7 @@ func TestRunCmdRefusesAnythingButAnInstance(t *testing.T) { |
| // no unit has started, so the stop is prompt and complete, Prepare's |
| // context is cancelled, and the run ends interrupted with no unit started. |
| func TestAStopDuringASlowPrepareDoesNotWaitForIt(t *testing.T) { |
| + t.Parallel() |
| a, cb, _ := testCmdActivator(t, Config{}) |
| rt := testRoute("c.example.com") |
| inPrepare, unblock := make(chan struct{}), make(chan struct{}) |
| @@ -506,6 +513,7 @@ func TestAStopDuringASlowPrepareDoesNotWaitForIt(t *testing.T) { |
| // the unit running: RunCmd settles it, and a unit that will not settle |
| // keeps the instance from another run and makes its next stop uncertain. |
| func TestAFailedStartThatMayHaveRunIsSettled(t *testing.T) { |
| + t.Parallel() |
| for _, c := range []struct { |
| name string |
| settleErr error |
| @@ -556,6 +564,7 @@ func TestALostCmdResponseStopsTheCmdRoleBeforeReplicationCloses(t *testing.T) { |
| cgroupErr error |
| }{{"cgroup empties", nil}, {"cgroup unreadable", errors.New("cgroup.events: permission denied")}} { |
| t.Run(c.name, func(t *testing.T) { |
| + shrinkCmdAcceptGrace(t, 50*time.Millisecond) // the stop drains two streams the fake unit never connected |
| a, cb, _ := testCmdActivator(t, Config{}) |
| ctl := &fakeControl{cmdErr: errors.New("502 mid-roll")} |
| unit := &fakeUnit{} |
| @@ -622,6 +631,7 @@ func TestALostCmdResponseStopsTheCmdRoleBeforeReplicationCloses(t *testing.T) { |
| // file leaves the run the live link. A caller's own setting never |
| // survives: RunCmd decides. |
| func TestRunCmdRunsTheServingGenerationsBuild(t *testing.T) { |
| + t.Parallel() |
| rt := testRoute("c.example.com") |
| staged := &instancectl.StagedExe{Path: "/opt/carlos/versions/x", Version: "v9", Digest: strings.Repeat("0", 64)} |
| want := instancectl.StagedExe{Path: "/opt/carlos/versions/v2/blog", Version: "v2", Digest: strings.Repeat("a", 64)} |
diff --git a/internal/activator/cmdunit.go b/internal/activator/cmdunit.go| index a219649be..402813706 100644 |
| --- a/internal/activator/cmdunit.go |
| +++ b/internal/activator/cmdunit.go |
| @@ -351,8 +351,8 @@ func (s *cmdStream) copy(c net.Conn) { |
| } |
| // cmdAcceptGrace is how long Drain waits for a connection still queued in |
| -// the listener's backlog. |
| -const cmdAcceptGrace = time.Second |
| +// the listener's backlog. A var so tests can shrink it. |
| +var cmdAcceptGrace = time.Second |
| // Drain waits up to bound for the stream to reach EOF (its unit has |
| // stopped, so every writer has closed), then closes the listener. The unit |
diff --git a/internal/activator/cmdunit_test.go b/internal/activator/cmdunit_test.go| index ce15edbcd..2f87e3a18 100644 |
| --- a/internal/activator/cmdunit_test.go |
| +++ b/internal/activator/cmdunit_test.go |
| @@ -74,7 +74,18 @@ func startTestCmd(t *testing.T) (*ExecBackend, registry.Route, *fakeControl, *fa |
| return b, rt, ctl, unit, u, out, errs |
| } |
| +// shrinkCmdAcceptGrace sets how long a cmd stream's Drain waits for a |
| +// connection that never came, for one test. The fake units here never |
| +// connect their streams, so every Drain waits the whole grace. |
| +func shrinkCmdAcceptGrace(t *testing.T, d time.Duration) { |
| + t.Helper() |
| + old := cmdAcceptGrace |
| + cmdAcceptGrace = d |
| + t.Cleanup(func() { cmdAcceptGrace = old }) |
| +} |
| + |
| func TestStartCmdWritesTheLaunchFileAndStartsTheUnit(t *testing.T) { |
| + shrinkCmdAcceptGrace(t, 50*time.Millisecond) // the cleanup drains two unconnected streams |
| b, rt, ctl, _, u, _, _ := startTestCmd(t) |
| t.Cleanup(func() { u.StopAndSettle(context.Background()) }) |
| if u.Invocation() != "inv1" || !strings.Contains(ctl.joined(), "cmd "+rt.Host) { |
| @@ -205,6 +216,7 @@ func TestCmdStreamDrainTakesAConnectionStillInTheBacklog(t *testing.T) { |
| } |
| func TestCmdStreamDrainOfAnUnconnectedStreamEndsAfterTheGrace(t *testing.T) { |
| + shrinkCmdAcceptGrace(t, 100*time.Millisecond) |
| dir, err := os.MkdirTemp("", "cs") |
| if err != nil { |
| t.Fatal(err) |
diff --git a/internal/activator/config_gate_test.go b/internal/activator/config_gate_test.go| index 5c4aa5c17..c07c6ed62 100644 |
| --- a/internal/activator/config_gate_test.go |
| +++ b/internal/activator/config_gate_test.go |
| @@ -90,6 +90,7 @@ func TestActivatorConfigRefusal503(t *testing.T) { |
| // has to stand the worker down (stop it, release the lease) rather than leave |
| // a childless entry holding the lease from a sibling that could run it. |
| func TestWorkerRespawnIsRefusedByTheConfigGateAndStandsDown(t *testing.T) { |
| + t.Parallel() |
| for _, why := range []string{"crash", "max_run"} { |
| t.Run(why, func(t *testing.T) { |
| a, b, clk, _ := testActivator(t, time.Hour) |
| @@ -137,6 +138,7 @@ func TestWorkerRespawnIsRefusedByTheConfigGateAndStandsDown(t *testing.T) { |
| // worker would be cycled onto the new build. With the gate refusing, it stands |
| // down instead of starting the new build without its config. |
| func TestWorkerConvergeRespawnIsRefusedByTheConfigGateAndStandsDown(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, ver := shippingActivator(t, time.Hour) |
| host := "bot.x.internal" |
| if err := a.WakeWorker(alwaysOnWorkerRoute(host)); err != nil { |
| @@ -170,6 +172,7 @@ func TestWorkerConvergeRespawnIsRefusedByTheConfigGateAndStandsDown(t *testing.T |
| // And a gate that allows it changes nothing: the respawn runs in place and |
| // keeps its lease, as before. |
| func TestWorkerRespawnProceedsWhenTheConfigGateAllows(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil { |
| t.Fatalf("WakeWorker: %v", err) |
diff --git a/internal/activator/converge_adopted_test.go b/internal/activator/converge_adopted_test.go| index be7ed7f55..b8044e496 100644 |
| --- a/internal/activator/converge_adopted_test.go |
| +++ b/internal/activator/converge_adopted_test.go |
| @@ -16,6 +16,7 @@ import ( |
| // 110-200s per deploy on the flagship). ConvergeAdopted is the agent finishing |
| // the half it delegated: no grace, no sweep. |
| func TestConvergeAdoptedCyclesAStaleInstanceWithoutTheGrace(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) // idle far away — only convergence can stop it |
| var ver atomic.Value |
| ver.Store("v1") |
| @@ -130,6 +131,7 @@ func TestConvergeAdoptedLeavesAnUnknownSpawnVersionAlone(t *testing.T) { |
| // the symlink" means something else there. The sweep's own worker converge |
| // (one move, behind the grace) still covers it. |
| func TestConvergeAdoptedLeavesWorkersToTheSweep(t *testing.T) { |
| + t.Parallel() |
| a, _, _, ver := shippingActivator(t, time.Hour) |
| host := "bot.x.internal" |
| if err := a.WakeWorker(alwaysOnWorkerRoute(host)); err != nil { |
| @@ -152,6 +154,7 @@ func TestConvergeAdoptedLeavesWorkersToTheSweep(t *testing.T) { |
| // else got there first — ConvergeAdopted finishing a root pass's adoption and |
| // a request re-waking the instance, while a slower pass was still mid-host. |
| func TestBounceAdoptedSparesAChildAlreadyOnTheLiveBuild(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| var ver atomic.Value |
| ver.Store("v2") |
| @@ -186,6 +189,7 @@ func TestBounceAdoptedSparesAChildAlreadyOnTheLiveBuild(t *testing.T) { |
| // that cannot name its build, or a symlink that cannot be read, must not turn |
| // an adoption into a silent no-op over a child still on the old binary. |
| func TestBounceAdoptedBouncesWhenTheBuildIsUnknown(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| a.liveVer = func(string) (string, error) { return "v2", nil } |
| b.execVerFn = func() string { return "" } |
| @@ -210,6 +214,7 @@ func TestBounceAdoptedBouncesWhenTheBuildIsUnknown(t *testing.T) { |
| // before any waiting request is let through, so every request lands on a |
| // child started after the bounce was asked for. |
| func TestBounceDuringAWakeIsAppliedWhenTheWakeFinishes(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| b.startGate = make(chan struct{}) |
| host := "one.example.com" |
diff --git a/internal/activator/declinedgate_identity_test.go b/internal/activator/declinedgate_identity_test.go| index 79ee5dc7d..05eb4341c 100644 |
| --- a/internal/activator/declinedgate_identity_test.go |
| +++ b/internal/activator/declinedgate_identity_test.go |
| @@ -92,6 +92,7 @@ func TestTheDeclinedGateIsAskedAboutTheCapturedWorkerWake(t *testing.T) { |
| // unreadable, the refusal stands: nothing is recorded or started, and the |
| // instance keeps serving. |
| func TestTheDeclinedGateIsAskedAboutAnUnstagedSidecarsCapture(t *testing.T) { |
| + t.Parallel() |
| host := "one.example.com" |
| setup := func(t *testing.T, refuse map[string]bool, ownErr error) (*Service, *planGenBackend, *liveGate, *execLog) { |
| t.Helper() |
diff --git a/internal/activator/exec_gate_test.go b/internal/activator/exec_gate_test.go| index acae4e4dc..153667835 100644 |
| --- a/internal/activator/exec_gate_test.go |
| +++ b/internal/activator/exec_gate_test.go |
| @@ -439,6 +439,7 @@ func workerGateService(t *testing.T, flip func()) (*Service, *ExecBackend, regis |
| // publication dropped. Before the fix the gate rode the before-exec hook, |
| // asked once before the cgroup preparation, and the worker started. |
| func TestAWorkerStartIsGatedAfterItsCgroupPreparation(t *testing.T) { |
| + t.Parallel() |
| var refuse atomic.Bool |
| a, _, rt, out, dir := workerGateService(t, func() { refuse.Store(true) }) |
| lost := errors.New("lease renewal failed") |
| @@ -483,6 +484,7 @@ func TestAWorkerStartIsGatedAfterItsCgroupPreparation(t *testing.T) { |
| // this same gate. Before the clone3 start left (amadan #272) this asked |
| // twice, at the clone3 start and its retry. |
| func TestAWorkerStartFallbackAsksTheGate(t *testing.T) { |
| + t.Parallel() |
| a, eb, rt, out, dir := workerGateService(t, nil) |
| var asked atomic.Int32 |
| eb.launchHook = func(string) { asked.Add(1) } |
| @@ -769,6 +771,7 @@ func TestALiveExecGateGatesTheNestedLiveLinkStart(t *testing.T) { |
| // refusal publishes and runs nothing. A recycle onto the captured build |
| // (onto set) is not a move onto the live link, and is not asked. |
| func TestALiveExecGateGatesAWorkerSpawnFromTheLiveLink(t *testing.T) { |
| + t.Parallel() |
| b := testExecBackend(t) |
| host := "bot.example.com" |
| out := filepath.Join(t.TempDir(), "argv0") |
| @@ -812,6 +815,7 @@ func TestALiveExecGateGatesAWorkerSpawnFromTheLiveLink(t *testing.T) { |
| // before it retires the old child, so a refusal leaves the worker running |
| // its current build instead of stopping it into a refused spawn. |
| func TestADeclinedWorkerIsNotConvergedOntoAHeldBuild(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _, ver := stagedActivator(t) |
| g := &liveGate{refuse: map[string]bool{"v2": true}} |
| a.SetLiveExecGate(g.gate) |
| @@ -850,6 +854,7 @@ func TestADeclinedWorkerIsNotConvergedOntoAHeldBuild(t *testing.T) { |
| // names a held build respawns its captured build, however long the drift |
| // lasts: the recycle's converge is refused before anything is stopped. |
| func TestADeclinedWorkerCrashRecyclesItsCapturedBuildWhenTheLiveLinkIsHeld(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _, ver := stagedActivator(t) |
| a.SetLiveExecGate((&liveGate{refuse: map[string]bool{"v2": true}}).gate) |
| host := "bot.x.internal" |
| @@ -878,6 +883,7 @@ func TestADeclinedWorkerCrashRecyclesItsCapturedBuildWhenTheLiveLinkIsHeld(t *te |
| // respawned from it: a crashed one stays down until a wake starts it, and |
| // one whose max_run falls due keeps running. |
| func TestADeclinedUncapturedWorkerIsNotRespawnedOntoAHeldBuild(t *testing.T) { |
| + t.Parallel() |
| t.Run("crashed", func(t *testing.T) { |
| a, b, clk, _ := testActivator(t, time.Hour) |
| a.liveVer = func(string) (string, error) { return "v2", nil } |
| @@ -932,6 +938,7 @@ func TestADeclinedUncapturedWorkerIsNotRespawnedOntoAHeldBuild(t *testing.T) { |
| // starts nothing (the supervisor's backoff keeps it down). With the gate nil |
| // the sidecar execs the link, as before. |
| func TestALiveExecGateGatesTheLegacySidecarStart(t *testing.T) { |
| + t.Parallel() |
| sidecarEnv := func(t *testing.T, b *ExecBackend, rt registry.Route) { |
| t.Helper() |
| if err := os.WriteFile(b.pathsFor(rt).sidecarEnv, nil, 0o600); err != nil { |
| @@ -1008,6 +1015,7 @@ func TestALiveExecGateGatesTheLegacySidecarStart(t *testing.T) { |
| // is asked first: a refusal keeps it running and spends no budget. A build |
| // the gate allows cycles as before. |
| func TestADeclinedSweepDoesNotCycleOntoAHeldBuild(t *testing.T) { |
| + t.Parallel() |
| t.Run("instance", func(t *testing.T) { |
| a, b, clk, _ := testActivator(t, time.Hour) |
| var ver atomic.Value |
diff --git a/internal/activator/fenceowner_test.go b/internal/activator/fenceowner_test.go| index 58dbe8dd4..6aaa0cfe6 100644 |
| --- a/internal/activator/fenceowner_test.go |
| +++ b/internal/activator/fenceowner_test.go |
| @@ -13,6 +13,7 @@ import ( |
| // another owner's at the same generation) refuses with ErrFenceHeld and |
| // stays exactly as it was, with nothing stopped. |
| func TestFenceAndStopAsMeetsOnlyItsOwnFence(t *testing.T) { |
| + t.Parallel() |
| t.Run("own fence, re-collected", func(t *testing.T) { |
| a, e, _, _, _ := maintenanceAwake(t) |
| proof, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 12, "owner-1") |
| @@ -118,6 +119,7 @@ func TestLiftFenceAsLiftsOnlyTheOwnersFence(t *testing.T) { |
| // answers ErrStopUnproven and leaves the fence as the other caller left |
| // it: still plain and unowned, or still lifted, never the owner's again. |
| func TestAnOwnedCallNeverTakesBackAFenceReplacedWhileItWaited(t *testing.T) { |
| + t.Parallel() |
| for _, meanwhile := range []string{"a plain fence", "a plain lift"} { |
| t.Run(meanwhile, func(t *testing.T) { |
| a, e, _, _, _ := maintenanceAwake(t) |
diff --git a/internal/activator/finalreview1_failedwake_test.go b/internal/activator/finalreview1_failedwake_test.go| index 1318f3f3d..bb86282b4 100644 |
| --- a/internal/activator/finalreview1_failedwake_test.go |
| +++ b/internal/activator/finalreview1_failedwake_test.go |
| @@ -19,6 +19,7 @@ import ( |
| // nested, keeps its own proof (ruling c). Here the host cgroup stays |
| // populated, so a proven wake is retained and its lease kept. |
| func TestAFailedWakeIsProvenByTheLayoutItsLaunchUsed(t *testing.T) { |
| + t.Parallel() |
| for _, row := range []struct { |
| name string |
| launch, cleanup Layout |
diff --git a/internal/activator/generations_backend_test.go b/internal/activator/generations_backend_test.go| index a4dacf0e4..3cfb77e1f 100644 |
| --- a/internal/activator/generations_backend_test.go |
| +++ b/internal/activator/generations_backend_test.go |
| @@ -66,6 +66,7 @@ func TestStartGenWithGenerationsOffIsStart(t *testing.T) { |
| }() |
| gotArgv := strings.Fields(waitForFile(t, argvPath, time.Second)) |
| gotEnv := strings.TrimSpace(waitForFile(t, envPath, time.Second)) |
| + cancel() // recorded: end the health wait the recorder never answers |
| <-done |
| if proc != nil { |
| defer proc.Stop(context.Background()) |
diff --git a/internal/activator/generations_instances_test.go b/internal/activator/generations_instances_test.go| index 07cb8195d..54b852740 100644 |
| --- a/internal/activator/generations_instances_test.go |
| +++ b/internal/activator/generations_instances_test.go |
| @@ -72,6 +72,7 @@ func breakGenerationsFile(t *testing.T, path string) { |
| } |
| func TestTheGenerationsFileFollowsEveryChange(t *testing.T) { |
| + t.Parallel() |
| dir := t.TempDir() |
| good := stagedID(t, dir, "v1") |
| a, _, _ := planActivator(t, WakePlan{Boot: good, Mode: ModeOverlap}) |
| @@ -83,7 +84,7 @@ func TestTheGenerationsFileFollowsEveryChange(t *testing.T) { |
| if err != nil || g == nil || len(g.Hosts) != 0 { |
| t.Fatalf("before any spawn: %+v, %v; want an empty file, written", g, err) |
| } |
| - done, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| + done, err := a.beginRequest(context.Background(), genTestRoute(t, "one.example.com")) |
| if err != nil { |
| t.Fatal(err) |
| } |
| @@ -108,6 +109,7 @@ func TestTheGenerationsFileFollowsEveryChange(t *testing.T) { |
| } |
| func TestTheGenerationsFileNamesThePlannedBuildBeforeItSpawns(t *testing.T) { |
| + t.Parallel() |
| dir := t.TempDir() |
| good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| a, b, _ := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good}) |
| @@ -115,7 +117,7 @@ func TestTheGenerationsFileNamesThePlannedBuildBeforeItSpawns(t *testing.T) { |
| b.gate = make(chan struct{}) |
| woke := make(chan error, 1) |
| go func() { |
| - done, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| + done, err := a.beginRequest(context.Background(), genTestRoute(t, "one.example.com")) |
| if err == nil { |
| done() |
| } |
| @@ -136,6 +138,7 @@ func TestTheGenerationsFileNamesThePlannedBuildBeforeItSpawns(t *testing.T) { |
| // execs; when it cannot be, it is not exec'd, and that is no failure of |
| // either build. |
| func TestAReplannedBuildIsPublishedBeforeItExecs(t *testing.T) { |
| + t.Parallel() |
| for _, broken := range []bool{false, true} { |
| name := "published" |
| if broken { |
| @@ -162,7 +165,7 @@ func TestAReplannedBuildIsPublishedBeforeItExecs(t *testing.T) { |
| return WakePlan{Boot: v3, Mode: ModeOverlap} |
| } |
| if broken { |
| - _, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| + _, err := a.beginRequest(context.Background(), genTestRoute(t, "one.example.com")) |
| if !errors.Is(err, errStagedBuildNotPublished) { |
| t.Fatalf("err = %v, want errStagedBuildNotPublished", err) |
| } |
| @@ -177,7 +180,7 @@ func TestAReplannedBuildIsPublishedBeforeItExecs(t *testing.T) { |
| b.gate = make(chan struct{}) |
| woke := make(chan error, 1) |
| go func() { |
| - done, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| + done, err := a.beginRequest(context.Background(), genTestRoute(t, "one.example.com")) |
| if err == nil { |
| done() |
| } |
| @@ -221,6 +224,7 @@ func TestARetainedFailedWakeStaysPinned(t *testing.T) { |
| // Spec §8.5: every generation of the operation set is named, the successor |
| // before its exec, and the retiring one until its termination is proven. |
| func TestTheGenerationsFileFollowsAnOverlapSwap(t *testing.T) { |
| + t.Parallel() |
| f := newSwapFixture(t) |
| f.a.GenerationsFile = release.GenerationsPath(t.TempDir()) |
| gate := make(chan struct{}) |
| @@ -244,12 +248,13 @@ func TestTheGenerationsFileFollowsAnOverlapSwap(t *testing.T) { |
| // spawn whose build could not be published must not exec it. The refusal is |
| // no failure of the build, takes no lease, and is never down. |
| func TestAnUnpublishedBuildIsNeverSpawned(t *testing.T) { |
| + t.Parallel() |
| t.Run("planned wake", func(t *testing.T) { |
| dir := t.TempDir() |
| good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| a, b, evs := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good}) |
| a.GenerationsFile = unwritableGenerationsFile(t) |
| - _, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| + _, err := a.beginRequest(context.Background(), genTestRoute(t, "one.example.com")) |
| if !errors.Is(err, errStagedBuildNotPublished) { |
| t.Fatalf("err = %v, want errStagedBuildNotPublished", err) |
| } |
| @@ -369,13 +374,14 @@ func claimedPlanActivator(t *testing.T, plan WakePlan) (*Service, *atomic.Int32, |
| // is listed by version and by staged path; an instance, which runs from the |
| // live link, adds nothing. |
| func TestTheGenerationsFileIsUnchangedWithTheFlagOff(t *testing.T) { |
| + t.Parallel() |
| a, _, _, _, _ := stagedActivator(t) // generations off, no planner |
| a.GenerationsFile = release.GenerationsPath(t.TempDir()) |
| const bot = "bot.x.internal" |
| if err := a.WakeWorker(alwaysOnWorkerRoute(bot)); err != nil { |
| t.Fatal(err) |
| } |
| - done, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| + done, err := a.beginRequest(context.Background(), genTestRoute(t, "one.example.com")) |
| if err != nil { |
| t.Fatal(err) |
| } |
| @@ -438,6 +444,7 @@ func TestAWorkerAttemptsBuildsArePublished(t *testing.T) { |
| // sweeping), so Start replaces it with this incarnation's snapshot, which is |
| // empty: nothing has spawned yet. |
| func TestStartReplacesAStaleGenerationsFile(t *testing.T) { |
| + t.Parallel() |
| stale := func(t *testing.T, path string) { |
| t.Helper() |
| if err := release.WriteGenerations(path, release.Generations{ |
diff --git a/internal/activator/gensocket_rev19_test.go b/internal/activator/gensocket_rev19_test.go| index 975120111..65793d674 100644 |
| --- a/internal/activator/gensocket_rev19_test.go |
| +++ b/internal/activator/gensocket_rev19_test.go |
| @@ -117,6 +117,7 @@ func TestFlatGenerationLaunchMatchesTheFlagOffLaunch(t *testing.T) { |
| }() |
| argv := strings.Split(strings.TrimSpace(waitForFile(t, argvPath, 3*time.Second)), "\n") |
| env := envMap(t, strings.Split(strings.TrimSpace(waitForFile(t, envPath, 3*time.Second)), "\n")) |
| + cancel() // recorded: end the health wait the recorder never answers |
| <-done |
| if proc != nil { |
| _ = proc.Stop(context.Background()) |
| @@ -393,6 +394,7 @@ func TestOverLongFlatHostClearsALeftoverCanonicalLink(t *testing.T) { |
| // goes once failedWakeStopped's whole proof holds, the host cgroup's |
| // included where the generation had no child of its own, and never before. |
| func TestFailedWakeRemovesItsDirectoryOnlyOnceProven(t *testing.T) { |
| + t.Parallel() |
| running := &sigtermIgnorer{} |
| exited := &sigtermIgnorer{} |
| exited.exited.Store(true) |
diff --git a/internal/activator/identity_test.go b/internal/activator/identity_test.go| index d4af652c3..836d79b2e 100644 |
| --- a/internal/activator/identity_test.go |
| +++ b/internal/activator/identity_test.go |
| @@ -591,6 +591,7 @@ func TestADeclinedLinkOutsideTheStoreAsksRootForNothing(t *testing.T) { |
| // the gate answered still seeds with v1. A held build seeds nothing: no |
| // process runs and no database is created. |
| func TestADeclinedFlatSeedRunsThePinnedFile(t *testing.T) { |
| + t.Parallel() |
| b := testExecBackend(t) |
| host := "seedpin.example.com" |
| marker := filepath.Join(t.TempDir(), "argv0") |
diff --git a/internal/activator/instancekey_delivery_test.go b/internal/activator/instancekey_delivery_test.go| index 35ffc7d9d..627a0724f 100644 |
| --- a/internal/activator/instancekey_delivery_test.go |
| +++ b/internal/activator/instancekey_delivery_test.go |
| @@ -168,7 +168,7 @@ func TestFirstStartWaitsForTheConfiguredKey(t *testing.T) { |
| // The helper exits once it has recorded its outcome, so Start's health |
| // wait never sees a live socket: the outcome is the proof, as in the |
| // replica tests, and the error only must not be the deferral. |
| - ctx2, cancel2 := context.WithTimeout(context.Background(), 2*time.Second) // ctx was spent waiting for the first outcome |
| + ctx2, cancel2 := recordedCtx(2*time.Second, out) // ctx was spent waiting for the first outcome |
| defer cancel2() |
| proc, _, err = b.Start(ctx2, rt) |
| if proc != nil { |
diff --git a/internal/activator/instancekey_test.go b/internal/activator/instancekey_test.go| index c632af017..7c9301a64 100644 |
| --- a/internal/activator/instancekey_test.go |
| +++ b/internal/activator/instancekey_test.go |
| @@ -646,9 +646,10 @@ func TestEnsureInstanceKeySymlinkErrorNamesThePathOnce(t *testing.T) { |
| } |
| // startRecorded starts host on b with the recording binary and returns the |
| -// environment the child saw, or nil when no child ran within a second. The |
| -// route's socket is a temp path, so the row classifies as foreign; |
| -// startRecordedRoute takes the row as given. |
| +// environment the child saw, or nil when no child ran (none recorded within a |
| +// second, or none is left running that could). The route's socket is a temp |
| +// path, so the row classifies as foreign; startRecordedRoute takes the row as |
| +// given. |
| func startRecorded(t *testing.T, b *ExecBackend, host string) (map[string]string, error) { |
| t.Helper() |
| return startRecordedRoute(t, b, registry.Route{Host: host, Kind: "instance", Socket: tmpSock(t)}) |
| @@ -657,26 +658,23 @@ func startRecorded(t *testing.T, b *ExecBackend, host string) (map[string]string |
| func startRecordedRoute(t *testing.T, b *ExecBackend, rt registry.Route) (map[string]string, error) { |
| t.Helper() |
| _, envPath := stageRecordingLiveFullEnv(t, b, rt.Host, "v1") |
| - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) |
| + ctx, cancel := recordedCtx(2*time.Second, envPath) |
| defer cancel() |
| proc, _, err := b.Start(ctx, rt) // the fake binary never binds: the health wait fails, as in the sibling tests |
| if proc != nil { |
| defer proc.Stop(context.Background()) |
| } |
| - deadline := time.Now().Add(time.Second) |
| - for time.Now().Before(deadline) { |
| - if raw, rerr := os.ReadFile(envPath); rerr == nil { |
| - child := map[string]string{} |
| - for _, line := range strings.Split(string(raw), "\n") { |
| - if k, v, ok := strings.Cut(line, "="); ok { |
| - child[k] = v |
| - } |
| - } |
| - return child, err |
| + raw := awaitChildFile(envPath, time.Second) |
| + if raw == "" { |
| + return nil, err |
| + } |
| + child := map[string]string{} |
| + for _, line := range strings.Split(raw, "\n") { |
| + if k, v, ok := strings.Cut(line, "="); ok { |
| + child[k] = v |
| } |
| - time.Sleep(10 * time.Millisecond) |
| } |
| - return nil, err |
| + return child, err |
| } |
| // Start hands the child the key it minted, and a second Start (a restart, a |
diff --git a/internal/activator/intent_events_test.go b/internal/activator/intent_events_test.go| index c164caaec..22440cdad 100644 |
| --- a/internal/activator/intent_events_test.go |
| +++ b/internal/activator/intent_events_test.go |
| @@ -71,6 +71,7 @@ func TestAPlannedWakeRefusedAtItsLeaseIsCancelledUnrun(t *testing.T) { |
| // A plan refused by its check before the exec, whose replan's check refuses |
| // too: both plans are cancelled unrun, each once, by its own token. |
| func TestARefusedReplanCancelsBothPlans(t *testing.T) { |
| + t.Parallel() |
| dir := t.TempDir() |
| good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| a, b, evs := planActivator(t, WakePlan{}) |
| @@ -101,6 +102,7 @@ func TestARefusedReplanCancelsBothPlans(t *testing.T) { |
| // A plan replaced by a replan that boots: the replaced plan is cancelled |
| // unrun, and the replan's own events follow. |
| func TestAReplacedPlanIsCancelledUnrun(t *testing.T) { |
| + t.Parallel() |
| dir := t.TempDir() |
| good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| a, b, evs := planActivator(t, WakePlan{}) |
| @@ -136,6 +138,7 @@ func TestAReplacedPlanIsCancelledUnrun(t *testing.T) { |
| // wokenInstance at all; it still ends with its cancellation. And the flag |
| // off (no OnGen) hears nothing, whatever the plan carries. |
| func TestAWakeWithNoResultIsStillCancelled(t *testing.T) { |
| + t.Parallel() |
| plan := WakePlan{Boot: stagedID(t, t.TempDir(), "v2"), Mode: ModeHold, FirstWake: true, Token: 81} |
| a, _, evs := planActivator(t, plan) |
| e := &actEntry{host: "one.example.com", route: testRoute("one.example.com"), plan: plan} |
diff --git a/internal/activator/logsink_test.go b/internal/activator/logsink_test.go| index e0ef72ae2..01c152426 100644 |
| --- a/internal/activator/logsink_test.go |
| +++ b/internal/activator/logsink_test.go |
| @@ -119,6 +119,7 @@ func (f *fakeSink) Event(acct, app, host, level, msg string) { |
| // one member-facing event per outcome — the #131 fact, at its source. The |
| // route carries app+account context so the sink can key the record. |
| func TestWakeOutcomesReachTheSink(t *testing.T) { |
| + t.Parallel() |
| // Failure: Start refuses, the event is the member-safe rendering. |
| a, b, _, _ := testActivator(t, time.Hour) |
| sink := &fakeSink{} |
diff --git a/internal/activator/maintenance_test.go b/internal/activator/maintenance_test.go| index 0ecf5d35c..d4a526228 100644 |
| --- a/internal/activator/maintenance_test.go |
| +++ b/internal/activator/maintenance_test.go |
| @@ -77,6 +77,7 @@ func maintenanceAwake(t *testing.T) (*Service, *actEntry, *maintenanceProc, *mai |
| return a, e, main, side, rep |
| } |
| func TestMaintenanceFenceChecksEveryStopOutcome(t *testing.T) { |
| + t.Parallel() |
| for _, scenario := range []string{"success", "main-error", "sidecar-error", "replica-error", "main-alive", "sidecar-alive", "missing-main", "missing-replica", "uncertain-sidecar"} { |
| t.Run(scenario, func(t *testing.T) { |
| a, e, main, side, rep := maintenanceAwake(t) |
| @@ -124,6 +125,7 @@ func TestMaintenanceFenceChecksEveryStopOutcome(t *testing.T) { |
| } |
| } |
| func TestMaintenanceFenceBlocksEveryAdmissionIncludingStaleRoute(t *testing.T) { |
| + t.Parallel() |
| a, _, _, reg := testActivator(t, time.Hour) |
| defer reg.Close() |
| host := "absent.example" |
| @@ -212,6 +214,7 @@ func TestMaintenanceFenceContextAndGenerationDoNotRestoreAdmission(t *testing.T) |
| } |
| } |
| func TestMaintenanceOrdinaryStopFailureRemainsObservable(t *testing.T) { |
| + t.Parallel() |
| a, e, main, _, _ := maintenanceAwake(t) |
| main.err = errors.New("stop refused") |
| a.mu.Lock() |
| @@ -225,6 +228,7 @@ func TestMaintenanceOrdinaryStopFailureRemainsObservable(t *testing.T) { |
| } |
| func TestMaintenanceLaterOrdinaryStopCannotEraseUnresolvedCleanup(t *testing.T) { |
| + t.Parallel() |
| a, e, main, _, _ := maintenanceAwake(t) |
| main.err = errors.New("earlier process may still run") |
| a.mu.Lock() |
| @@ -317,6 +321,7 @@ func (l *maintenanceLease) Replicate(context.Context, string, time.Duration) (Re |
| return l.b.rep, l.b.replicateFailure |
| } |
| func TestMaintenanceFenceDuringWakeOwnsItsCleanup(t *testing.T) { |
| + t.Parallel() |
| for _, phase := range []string{"acquire", "replicate"} { |
| t.Run(phase, func(t *testing.T) { |
| a, base, _, reg := testActivator(t, time.Hour) |
| @@ -368,6 +373,7 @@ func TestMaintenanceFenceDuringWakeOwnsItsCleanup(t *testing.T) { |
| } |
| } |
| func TestMaintenanceFailedBackendStartRemainsUnproven(t *testing.T) { |
| + t.Parallel() |
| a, base, _, reg := testActivator(t, time.Hour) |
| defer reg.Close() |
| a.backend = &maintenanceBackend{fakeBackend: base, startFailure: errors.New("hidden backend cleanup failed")} |
| @@ -391,6 +397,7 @@ func TestMaintenanceFailedBackendStartRemainsUnproven(t *testing.T) { |
| // but a stalled sidecar still keeps a checked stop from completing |
| // instantly, which is the property this test exists to cover. |
| func TestMaintenanceFenceSerializesSidecarSweepReplacement(t *testing.T) { |
| + t.Parallel() |
| a, e, _, side, rep := maintenanceAwake(t) |
| gate, entered := make(chan struct{}), make(chan struct{}) |
| base := a.backend.(*fakeBackend) |
| @@ -435,6 +442,7 @@ func (d *maintenanceDue) DueInstances(int64) ([]registry.Route, error) { |
| return []registry.Route{d.route}, nil |
| } |
| func TestMaintenanceFenceStopsStaleScheduledWake(t *testing.T) { |
| + t.Parallel() |
| a, b, _, reg := testActivator(t, time.Hour) |
| defer reg.Close() |
| rt := testRoute("scheduled.example") |
| @@ -457,6 +465,7 @@ func TestMaintenanceFenceStopsStaleScheduledWake(t *testing.T) { |
| } |
| } |
| func TestMaintenanceAwakeFenceSuppressesSidecarAdoption(t *testing.T) { |
| + t.Parallel() |
| a, e, _, side, _ := maintenanceAwake(t) |
| base := a.backend.(*fakeBackend) |
| base.sidecarStaged = true |
| @@ -472,6 +481,7 @@ func TestMaintenanceAwakeFenceSuppressesSidecarAdoption(t *testing.T) { |
| } |
| func TestMaintenanceFailedStartCleansReturnedHandleBeforeRelease(t *testing.T) { |
| + t.Parallel() |
| a, base, _, reg := testActivator(t, time.Hour) |
| defer reg.Close() |
| gate, entered := make(chan struct{}), make(chan struct{}) |
| @@ -509,6 +519,7 @@ func TestMaintenanceFailedStartCleansReturnedHandleBeforeRelease(t *testing.T) { |
| } |
| } |
| func TestMaintenanceFailedSidecarHandlesRemainOwned(t *testing.T) { |
| + t.Parallel() |
| for _, phase := range []string{"wake", "sweep"} { |
| t.Run(phase, func(t *testing.T) { |
| a, base, _, reg := testActivator(t, time.Hour) |
| @@ -556,6 +567,7 @@ func TestMaintenanceFailedSidecarHandlesRemainOwned(t *testing.T) { |
| } |
| func TestMaintenanceFailedReplicationClosesReturnedReplicaWithoutDoubleRelease(t *testing.T) { |
| + t.Parallel() |
| for _, closeFails := range []bool{false, true} { |
| t.Run(fmt.Sprint(closeFails), func(t *testing.T) { |
| a, base, _, reg := testActivator(t, time.Hour) |
| @@ -612,6 +624,7 @@ func TestMaintenanceFailedReplicationClosesReturnedReplicaWithoutDoubleRelease(t |
| } |
| func TestMaintenanceFailedAcquireReleasesReturnedLease(t *testing.T) { |
| + t.Parallel() |
| a, base, _, reg := testActivator(t, time.Hour) |
| defer reg.Close() |
| a.backend = &maintenanceBackend{fakeBackend: base, acquireFailure: errors.New("lease returned with acquisition failure")} |
| @@ -680,6 +693,7 @@ func TestFenceGenerationReportsTheHeldGeneration(t *testing.T) { |
| // It must answer false for a host it has never seen at all, since that is |
| // not evidence of anything. |
| func TestStopUncertainCoversLiveEntriesAndIncompleteHistory(t *testing.T) { |
| + t.Parallel() |
| a, e, main, _, _ := maintenanceAwake(t) |
| if !a.StopUncertain(e.host) { |
| @@ -722,6 +736,7 @@ func TestStopUncertainCoversLiveEntriesAndIncompleteHistory(t *testing.T) { |
| // lock that installs, so a fence of any generation, lower, equal or |
| // higher, refuses it and stays exactly as it was, and nothing is stopped. |
| func TestFenceAndStopIfUnfencedRefusesAnyHeldFence(t *testing.T) { |
| + t.Parallel() |
| for _, held := range []uint64{5, 10, 20} { |
| t.Run(fmt.Sprint(held), func(t *testing.T) { |
| a, e, main, _, _ := maintenanceAwake(t) |
| @@ -748,6 +763,7 @@ func TestFenceAndStopIfUnfencedRefusesAnyHeldFence(t *testing.T) { |
| // TestFenceAndStopIfUnfencedIsFenceAndStopOnAnUnfencedHost: with no fence |
| // held it installs, stops and proves exactly as FenceAndStop does. |
| func TestFenceAndStopIfUnfencedIsFenceAndStopOnAnUnfencedHost(t *testing.T) { |
| + t.Parallel() |
| a, e, _, _, _ := maintenanceAwake(t) |
| proof, err := a.FenceAndStopIfUnfenced(maintenanceContext(t, time.Second), e.host, 7) |
| if err != nil || !proof.Matches(e.host, 7) { |
diff --git a/internal/activator/opset_test.go b/internal/activator/opset_test.go| index e19d1f95e..19939c959 100644 |
| --- a/internal/activator/opset_test.go |
| +++ b/internal/activator/opset_test.go |
| @@ -84,6 +84,7 @@ func (s *stopSignal) Event(_, _, _, _, msg string) { |
| // held open by a gate; until it closes, neither the instance's own stop |
| // nor the lease release may happen. |
| func TestSwapWitnessBarrierJoinsBeforeTheLease(t *testing.T) { |
| + t.Parallel() |
| stops := []struct { |
| name string |
| stop func(x barrierEnv) |
| @@ -390,6 +391,7 @@ func TestAStopKeepsTheLeaseWhileItsLeaderLivesOn(t *testing.T) { |
| // (spec §7.4): its process may still run against the database. The lease |
| // went with the stop before. |
| func TestAStopKeepsTheLeaseWhileAStraySidecarRuns(t *testing.T) { |
| + t.Parallel() |
| x := awakeForBarrier(t, true) |
| stray := &fakeProc{label: "sidecar", rec: x.b, stuck: true} |
| x.a.mu.Lock() |
| @@ -473,6 +475,7 @@ func TestAClaimsOnMemberNeedsItsChildEmptied(t *testing.T) { |
| // that accepts the leader's exit alone releases the lease over a surviving |
| // descendant. This is the witness for mutation M8. |
| func TestSwapWitnessStopProvesTheCurrentGenerationEmpty(t *testing.T) { |
| + t.Parallel() |
| x := awakeForBarrier(t, true) |
| kb := &killGenBackend{fakeBackend: x.b, err: errors.New("the child still holds a process")} |
| x.a.backend = kb |
| @@ -513,6 +516,7 @@ func TestSwapWitnessStopProvesTheCurrentGenerationEmpty(t *testing.T) { |
| // wake and waits for it. Its child is stopped and its lease given back |
| // before Shutdown returns. |
| func TestSwapWitnessShutdownJoinsAWakingEntry(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| a.gens = true |
| gate := make(chan struct{}) |
| @@ -558,6 +562,7 @@ func TestSwapWitnessShutdownJoinsAWakingEntry(t *testing.T) { |
| // lands while Start is held, so the wake's context is cancelled before it |
| // releases. Flag off: the fix is not gated on generations. |
| func TestAFencedWakeReleasesUnderALiveContext(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| gate := make(chan struct{}) |
| b.startGate = gate |
| @@ -600,6 +605,7 @@ func TestAFencedWakeReleasesUnderALiveContext(t *testing.T) { |
| // cancelled context, so Shutdown must wait it out, stop what it woke and |
| // give the lease back under a live context. |
| func TestShutdownJoinsAWakingEntryWithSwapOff(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| gate := make(chan struct{}) |
| b.startGate = gate |
| @@ -799,6 +805,7 @@ var ( |
| // instancectl.ErrGenUnproven exist only under plan 2's GenContainment, |
| // which can be on while swaps are off. This is the witness for M9. |
| func TestSwapWitnessUnprovenWakeKeepsTheLease(t *testing.T) { |
| + t.Parallel() |
| lost := errors.New("the unit exited before it was observed") |
| rows := []struct { |
| name string |
| @@ -1066,6 +1073,7 @@ var _ GenBackend = (*fencedStartBackend)(nil) |
| // FenceAndStop reports the stop unproven, and the sweep gives the lease back |
| // once the leader has exited. |
| func TestSwapWitnessAFencedWakeKeepsItsLeaseWhileItsProcessRuns(t *testing.T) { |
| + t.Parallel() |
| for _, gens := range []bool{false, true} { |
| t.Run(fmt.Sprintf("generations %v", gens), func(t *testing.T) { |
| a, b, _, _ := testActivator(t, time.Hour) |
| @@ -1182,6 +1190,7 @@ var _ hostKiller = (*uncontainedStartBackend)(nil) |
| // empty the host cgroup: while it cannot, the wake stays retained with its |
| // lease, and once it can the lease goes back. |
| func TestARetainedFlagOffWakeProvesItsHostCgroupEmpty(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| a.gens = false |
| leader := &sigtermIgnorer{} |
| @@ -1243,6 +1252,7 @@ func TestARetainedFlagOffWakeProvesItsHostCgroupEmpty(t *testing.T) { |
| // a contained generation are as before, and so is a box with no delegated |
| // cgroup, where the wake ran unaccounted and nothing could be proven. |
| func TestFailedWakeStoppedProvesTheHostCgroupWithoutContainment(t *testing.T) { |
| + t.Parallel() |
| exited := &sigtermIgnorer{} |
| exited.exited.Store(true) |
| for _, row := range []struct { |
| @@ -1404,6 +1414,7 @@ func TestARealFlatHealthFailedWakeProvesItsHostCgroup(t *testing.T) { |
| // Ruling c: a nested row and a unit-backed row have their own proof and are |
| // never KillHost'd; their failed wake releases as before. |
| func TestAFlagOffFailedWakeProvesTheHostCgroupOnlyForAnExecChild(t *testing.T) { |
| + t.Parallel() |
| for _, row := range []struct { |
| name string |
| layout Layout |
diff --git a/internal/activator/outcome_fence_test.go b/internal/activator/outcome_fence_test.go| index 13105bc32..582479c85 100644 |
| --- a/internal/activator/outcome_fence_test.go |
| +++ b/internal/activator/outcome_fence_test.go |
| @@ -75,6 +75,7 @@ func secondPlanner(o *blockedOnGen, first WakePlan, saw chan<- bool) func(contex |
| // record in place. Before the fence the second wake planned B again in the |
| // gap, before the hold-down existed. |
| func TestANewWakeWaitsForAFailedFirstWakesOutcome(t *testing.T) { |
| + t.Parallel() |
| dir := t.TempDir() |
| next := stagedID(t, dir, "v2") |
| a, b, _ := planActivator(t, WakePlan{}) |
diff --git a/internal/activator/plan14_generation_workers_test.go b/internal/activator/plan14_generation_workers_test.go| index 6d28b29ef..7f24f7380 100644 |
| --- a/internal/activator/plan14_generation_workers_test.go |
| +++ b/internal/activator/plan14_generation_workers_test.go |
| @@ -12,6 +12,7 @@ import ( |
| ) |
| func TestPlan14GenerationWorkerStartBoundary(t *testing.T) { |
| + t.Parallel() |
| for _, claimed := range []bool{false, true} { |
| for _, staged := range []bool{false, true} { |
| for _, pinned := range []bool{false, true} { |
| @@ -64,6 +65,7 @@ func TestPlan14GenerationWorkerStartBoundary(t *testing.T) { |
| } |
| func TestPlan14GenerationWorkerRealBackendRefusal(t *testing.T) { |
| + t.Parallel() |
| for _, pinned := range []bool{false, true} { |
| name := "live" |
| if pinned { |
| @@ -138,6 +140,7 @@ func TestPlan14GenerationWorkerReplacementBoundary(t *testing.T) { |
| } |
| func TestPlan14GenerationWorkerSupervisionBoundary(t *testing.T) { |
| + t.Parallel() |
| for _, claimed := range []bool{false, true} { |
| for _, cause := range []string{"crash", "max-run", "converge"} { |
| name := "legacy/" + cause |
diff --git a/internal/activator/probewake_test.go b/internal/activator/probewake_test.go| index f4517ae11..ac3c85bf2 100644 |
| --- a/internal/activator/probewake_test.go |
| +++ b/internal/activator/probewake_test.go |
| @@ -266,6 +266,7 @@ func TestProbeWakeRefusesUnheldRowOrForeignExec(t *testing.T) { |
| // RoleLaunch.Start, which main still refuses |
| // (TestStartAdmittedLoopWindowAndReadinessAtTheBoundary). |
| func TestProbeWakeEndsAtTheCutDeadline(t *testing.T) { |
| + t.Parallel() |
| r := &pwRuntime{observed: pwFlip()} |
| a, _ := pwService(t, r) |
| cfg := pwConfig() |
| @@ -306,6 +307,7 @@ func TestProbeAttemptEndsAtTheWindow(t *testing.T) { |
| // probe entry while the registry row is still held by the probe's hold op, |
| // and does once it is not (H18). |
| func TestRetireExceptKeepsTheHeldProbeEntry(t *testing.T) { |
| + t.Parallel() |
| a, _, _, reg := testActivator(t, time.Hour) |
| a.Routes = reg |
| held := registry.Route{Host: "one.example.com", Kind: "instance", Socket: "/tmp/nope.sock", DBPath: "/tmp/one.example.com.db", Hibernate: true} |
| @@ -464,6 +466,7 @@ func TestOpenProbeEntryMarksTheLiveProbe(t *testing.T) { |
| // the open marked: its row is ready, and the next reload manages it. Once |
| // the edge manages it, it is an ordinary serving entry. |
| func TestRetireExceptKeepsTheOpenedProbeAcrossTheOpen(t *testing.T) { |
| + t.Parallel() |
| a, _, _, reg := testActivator(t, time.Hour) |
| a.Routes = reg |
| if err := reg.Upsert(testRoute("one.example.com")); err != nil { |
| @@ -511,6 +514,7 @@ func TestRetireExceptKeepsTheOpenedProbeAcrossTheOpen(t *testing.T) { |
| // Task 7, the idle stop: a held probe entry gets no traffic, so the idle |
| // sweep must not stop it before the open; once opened it idles as any entry. |
| func TestIdleSweepSparesAHeldProbe(t *testing.T) { |
| + t.Parallel() |
| a, _, clk, _ := testActivator(t, time.Minute) |
| e := pwHeldEntry(t, a) |
| a.mu.Lock() |
| @@ -603,6 +607,7 @@ func TestRetireExceptOverlappingReloadsAcrossTheOpen(t *testing.T) { |
| // once the release is confirmed. A failed release leaves the entry |
| // protected through an idle sweep; a retried open that succeeds ends it. |
| func TestIdleProtectionHoldsUntilTheReleaseIsConfirmed(t *testing.T) { |
| + t.Parallel() |
| a, _, clk, _ := testActivator(t, time.Minute) |
| e := pwHeldEntry(t, a) |
| a.mu.Lock() |
diff --git a/internal/activator/recoveryprobe_sweep_test.go b/internal/activator/recoveryprobe_sweep_test.go| index fa83dfd95..7853e7ddb 100644 |
| --- a/internal/activator/recoveryprobe_sweep_test.go |
| +++ b/internal/activator/recoveryprobe_sweep_test.go |
| @@ -44,6 +44,7 @@ func waitProbe(t *testing.T, a *Service, predicate func() bool) { |
| } |
| func TestRecoveryProbeSweepRetiresFailedBehindCutFence(t *testing.T) { |
| + t.Parallel() |
| for _, mode := range []string{"exit", "lease", "join-exit", "join-lease"} { |
| t.Run(mode, func(t *testing.T) { |
| a, r, e, proc, rep, admitted := sweptProbe(t) |
| @@ -86,6 +87,7 @@ func TestRecoveryProbeSweepRetiresFailedBehindCutFence(t *testing.T) { |
| } |
| func TestRecoveryProbeSweepOpensOnlyExactSessionAndFence(t *testing.T) { |
| + t.Parallel() |
| for _, phase := range []string{hostclaim.PhaseProbing, hostclaim.PhaseProbed, hostclaim.PhaseOpened, "serving", "new-session", "foreign-executor", "unreadable"} { |
| t.Run(phase, func(t *testing.T) { |
| a, r, e, proc, _, admitted := sweptProbe(t) |
| @@ -160,6 +162,7 @@ func TestRecoveryProbeSweepOpensOnlyExactSessionAndFence(t *testing.T) { |
| } |
| func TestRecoveryProbeSweepPreservesReplacementFence(t *testing.T) { |
| + t.Parallel() |
| for _, afterRead := range []bool{false, true} { |
| t.Run(map[bool]string{false: "before-read", true: "during-read"}[afterRead], func(t *testing.T) { |
| a, r, e, proc, rep, admitted := sweptProbe(t) |
| @@ -213,6 +216,7 @@ func TestRecoveryProbeRefusesAnotherOperationsFence(t *testing.T) { |
| // the slower activator sweep observes opened/serving. The exact session |
| // still owns its process; a missing fence alone is not an opening proof. |
| func TestRecoveryProbeSweepAcceptsOwnerLiftOnlyWhenOpened(t *testing.T) { |
| + t.Parallel() |
| for _, phase := range []string{hostclaim.PhaseOpened, "serving", hostclaim.PhaseProbing} { |
| for _, duringRead := range []bool{false, true} { |
| t.Run(phase+map[bool]string{false: "/before", true: "/during"}[duringRead], func(t *testing.T) { |
diff --git a/internal/activator/recoveryprobe_test.go b/internal/activator/recoveryprobe_test.go| index db34ba5c6..58270463f 100644 |
| --- a/internal/activator/recoveryprobe_test.go |
| +++ b/internal/activator/recoveryprobe_test.go |
| @@ -139,6 +139,7 @@ func probeService(t *testing.T, obs hostclaim.Observed) (*Service, *ClaimRuntime |
| } |
| func TestRecoveryProbeRunsTheClaimRuntimeWithTheProbePolicyAndExactInputs(t *testing.T) { |
| + t.Parallel() |
| for _, kind := range []string{hostclaim.KindRestoring, hostclaim.KindReverting} { |
| t.Run(kind, func(t *testing.T) { |
| obs := probeObserved(probeClaim(kind, hostclaim.PhaseProbing, "box")) |
| @@ -197,6 +198,7 @@ func TestRecoveryProbeK6StopsForeignBeforeAnyAdmission(t *testing.T) { |
| } |
| func TestRecoveryProbeRefusesWrongKindOrPhaseBeforePrerequisites(t *testing.T) { |
| + t.Parallel() |
| for _, c := range []*hostclaim.Claim{ |
| probeClaim(hostclaim.KindRestoring, hostclaim.PhaseClaimed, "box"), |
| probeClaim(hostclaim.KindRestoring, hostclaim.PhaseProbed, "box"), |
| @@ -221,6 +223,7 @@ func TestRecoveryProbeRefusesWrongKindOrPhaseBeforePrerequisites(t *testing.T) { |
| } |
| func TestRecoveryProbeRefusesMissingRuntimeOrPolicyBeforeAnyRead(t *testing.T) { |
| + t.Parallel() |
| obs := probeObserved(probeClaim(hostclaim.KindRestoring, hostclaim.PhaseProbing, "box")) |
| in := probeInputs() |
| runtimes := map[string]func(*Service, *ClaimRuntime){ |
| @@ -272,6 +275,7 @@ func TestRecoveryProbeRefusesMissingRuntimeOrPolicyBeforeAnyRead(t *testing.T) { |
| } |
| func TestRecoveryProbeRefusesAnOrdinaryEntryAndShutdown(t *testing.T) { |
| + t.Parallel() |
| obs := probeObserved(probeClaim(hostclaim.KindRestoring, hostclaim.PhaseProbing, "box")) |
| for _, state := range []actState{actWaking, actAwake, actStopping} { |
| a, _, run := probeService(t, obs) |
| @@ -293,6 +297,7 @@ func TestRecoveryProbeRefusesAnOrdinaryEntryAndShutdown(t *testing.T) { |
| // One probe per host: an identical second call joins the one in flight and |
| // shares its outcome; a different policy is refused; a request never joins. |
| func TestRecoveryProbeCoalescesTheSamePolicyAndRefusesOthers(t *testing.T) { |
| + t.Parallel() |
| obs := probeObserved(probeClaim(hostclaim.KindRestoring, hostclaim.PhaseProbing, "box")) |
| a, r, run := probeService(t, obs) |
| entered, release := make(chan struct{}), make(chan struct{}) |
| @@ -782,6 +787,7 @@ func TestRecoveryProbeGenerationRoutingAfterOpening(t *testing.T) { |
| } |
| func TestRecoveryHoldSurvivesAdoptionAtProbed(t *testing.T) { |
| + t.Parallel() |
| for _, kind := range []string{hostclaim.KindReverting, hostclaim.KindRestoring} { |
| for _, fenced := range []bool{false, true} { |
| for _, api := range []string{"Bounce", "BounceAdopted", "ConvergeAdopted"} { |
diff --git a/internal/activator/replica_instancekey_linux_test.go b/internal/activator/replica_instancekey_linux_test.go| index 66c5cfe36..ad0d04c5a 100644 |
| --- a/internal/activator/replica_instancekey_linux_test.go |
| +++ b/internal/activator/replica_instancekey_linux_test.go |
| @@ -44,7 +44,7 @@ func TestKeylessReplicaWithAControlSocketLeavesTheHelperToRefuse(t *testing.T) { |
| t.Fatal(err) |
| } |
| out := stageHelperApp(t, b, rt.Host, "v1") |
| - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) |
| + ctx, cancel := recordedCtx(2*time.Second, out) |
| defer cancel() |
| proc, _, _ := b.Start(ctx, rt) |
| if proc != nil { |
diff --git a/internal/activator/replica_instancekey_test.go b/internal/activator/replica_instancekey_test.go| index 2bc52c544..0e15f8b18 100644 |
| --- a/internal/activator/replica_instancekey_test.go |
| +++ b/internal/activator/replica_instancekey_test.go |
| @@ -96,16 +96,10 @@ func stageHelperApp(t *testing.T, b *ExecBackend, host, ver string) string { |
| } |
| // outcomeOf waits up to two seconds for the helper's outcome; "" is no app |
| -// ran at all. |
| +// ran at all: none recorded, and no child of this test binary is left |
| +// running that could (awaitChildFile). Call it once the launch returned. |
| func outcomeOf(path string) string { |
| - deadline := time.Now().Add(2 * time.Second) |
| - for time.Now().Before(deadline) { |
| - if raw, err := os.ReadFile(path); err == nil { |
| - return string(raw) |
| - } |
| - time.Sleep(10 * time.Millisecond) |
| - } |
| - return "" |
| + return awaitChildFile(path, 2*time.Second) |
| } |
| // noControlSocket is a socket path too long to bind, so the launch's |
| @@ -198,7 +192,7 @@ func TestReplicaLaunchWithoutAControlSocket(t *testing.T) { |
| } |
| b.ConfiguredKey = freshConfig(tc) |
| out := stageHelperApp(t, b, rt.Host, "v1") |
| - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) |
| + ctx, cancel := recordedCtx(2*time.Second, out) |
| defer cancel() |
| proc, _, err := b.Start(ctx, rt) |
| if proc != nil { |
diff --git a/internal/activator/scheduled_token_test.go b/internal/activator/scheduled_token_test.go| index 0acea3a8b..b1fe98d24 100644 |
| --- a/internal/activator/scheduled_token_test.go |
| +++ b/internal/activator/scheduled_token_test.go |
| @@ -30,6 +30,7 @@ func TestScheduledTokenFenceRefusesPreparation(t *testing.T) { |
| } |
| func TestScheduledTokenPreparationOrdersFence(t *testing.T) { |
| + t.Parallel() |
| const host = "token.example.com" |
| a := &Service{stopHistory: map[string]*stopObservation{host: { |
| mainStop: true, mainExited: true, sidecarStop: true, sidecarExited: true, replicaClosed: true, |
diff --git a/internal/activator/seed_exclusion_test.go b/internal/activator/seed_exclusion_test.go| index 02e68c684..8ccd11d01 100644 |
| --- a/internal/activator/seed_exclusion_test.go |
| +++ b/internal/activator/seed_exclusion_test.go |
| @@ -11,6 +11,7 @@ import ( |
| ) |
| func TestSeedExclusionWaitsForExistingPreparation(t *testing.T) { |
| + t.Parallel() |
| for _, kind := range []string{"request", "timer", "stop"} { |
| t.Run(kind, func(t *testing.T) { |
| a := &Service{entries: map[string]*actEntry{}} |
diff --git a/internal/activator/seed_rollback_stop_test.go b/internal/activator/seed_rollback_stop_test.go| index 42244dc95..590ebb6ff 100644 |
| --- a/internal/activator/seed_rollback_stop_test.go |
| +++ b/internal/activator/seed_rollback_stop_test.go |
| @@ -269,6 +269,7 @@ func mustRoute(t *testing.T, reg *registry.Registry, host string) *registry.Rout |
| // the rollback's fence cancels the start, #292's proof stops the unit at |
| // once, and the same pass restores the original route. |
| func TestImportRollbackAfterAnAbortedStartProvenAtOnce(t *testing.T) { |
| + t.Parallel() |
| x := newImportRollback(t) |
| x.queueStart(t) |
| woke := x.wake(t) |
| @@ -292,6 +293,7 @@ func TestImportRollbackAfterAnAbortedStartProvenAtOnce(t *testing.T) { |
| // once the sweep's retry proves the unit gone, the next pass restores the |
| // original route, with no agent restart in between. |
| func TestImportRollbackAfterAnAbortedStartProvenLater(t *testing.T) { |
| + t.Parallel() |
| x := newImportRollback(t) |
| x.b.mu.Lock() |
| x.b.legacyStopErr = errors.New("stop carlos-instance@a.shop.example.com: instance control socket: connection refused") |
| @@ -324,6 +326,7 @@ func TestImportRollbackAfterAnAbortedStartProvenLater(t *testing.T) { |
| // While #292's proof keeps failing, the rollback deletes nothing, however |
| // many passes and retries run. |
| func TestImportRollbackKeepsEverythingWhileTheStopIsUnproven(t *testing.T) { |
| + t.Parallel() |
| x := newImportRollback(t) |
| x.b.mu.Lock() |
| x.b.legacyStopErr = errors.New("stop carlos-instance@a.shop.example.com: unit still active") |
| @@ -416,6 +419,7 @@ func TestALaunchRefusedBeforeExecIsNotUncertain(t *testing.T) { |
| // A retained start's proof discharges only its own uncertainty: one that |
| // another source added to stays. |
| func TestTheLegacyProofDischargesOnlyItsOwnUncertainty(t *testing.T) { |
| + t.Parallel() |
| for _, other := range []bool{false, true} { |
| a, b, _, reg := testActivator(t, time.Hour) |
| host := "own.example.com" |
| @@ -467,6 +471,7 @@ func TestOnlyAWakeThatStartedNothingIsRecordedNotStarted(t *testing.T) { |
| // The sidecar's start (step 6) is the wake's last gate; a replica that |
| // opens and then fails is the stage before it. |
| func TestALateRefusalMarksNothingHeld(t *testing.T) { |
| + t.Parallel() |
| refused := launchRefused{ErrFenced} |
| t.Run("the sidecar's gate", func(t *testing.T) { |
| @@ -523,6 +528,7 @@ func TestALateRefusalMarksNothingHeld(t *testing.T) { |
| // it must leave the host's stop history as it was, and the rollback |
| // restores the original route in the same pass, with no restart. |
| func TestImportRollbackAfterACachedRowWakeRefusedBeforeItsLease(t *testing.T) { |
| + t.Parallel() |
| for _, history := range []string{"none", "a clean stop"} { |
| t.Run(history, func(t *testing.T) { |
| x := newImportRollback(t) |
diff --git a/internal/activator/settle_test.go b/internal/activator/settle_test.go| index 207943785..61df391ef 100644 |
| --- a/internal/activator/settle_test.go |
| +++ b/internal/activator/settle_test.go |
| @@ -9,6 +9,7 @@ import ( |
| ) |
| func TestObserveSettle(t *testing.T) { |
| + t.Parallel() |
| if r := observeSettle(context.Background(), &fakeProc{}, 20*time.Millisecond); r != settleOK { |
| t.Fatalf("a process that stayed up: %v", r) |
| } |
diff --git a/internal/activator/shutdown_lease_test.go b/internal/activator/shutdown_lease_test.go| index 4b7c24297..51fbc15f3 100644 |
| --- a/internal/activator/shutdown_lease_test.go |
| +++ b/internal/activator/shutdown_lease_test.go |
| @@ -28,6 +28,7 @@ import ( |
| // let the job launch a writer nobody holds a lease for. So the abort stops |
| // the unit and proves it gone first, and only then gives the lease back. |
| func TestShutdownAbortsAWakeAndReleasesItsLease(t *testing.T) { |
| + t.Parallel() |
| a, b, reqErr := arrangeQueuedRewake(t, "restart.example.com") |
| ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) |
| @@ -108,12 +109,15 @@ func arrangeQueuedRewake(t *testing.T, host string) (*Service, *fakeBackend, <-c |
| // stopped KEEPS its lease. A lease left to its TTL is a 15-minute stall; a |
| // lease given back over a queued start job is two writers. |
| func TestAnAbortedStartNotProvenStoppedKeepsItsLease(t *testing.T) { |
| + t.Parallel() |
| a, b, _ := arrangeQueuedRewake(t, "unproven.example.com") |
| b.mu.Lock() |
| b.legacyStopErr = errors.New("stop carlos-instance@unproven: instance control socket: connection refused") |
| b.mu.Unlock() |
| - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) |
| + // An unproven start keeps Shutdown polling until its budget ends, so |
| + // the budget is all this test waits: half a second is five of its polls. |
| + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) |
| defer cancel() |
| a.Shutdown(ctx) |
| @@ -140,6 +144,7 @@ func TestAnAbortedStartNotProvenStoppedKeepsItsLease(t *testing.T) { |
| // its process exits, and its lease goes back then, not once every wake |
| // has joined. |
| func TestShutdownFinishesPendingStopsWhileJoiningWakes(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| stuckHost, wakingHost := "stuck.example.com", "waking.example.com" |
| b.procStuck = true |
| @@ -184,6 +189,7 @@ func TestShutdownFinishesPendingStopsWhileJoiningWakes(t *testing.T) { |
| // that fails gives its lease back, the store refuses the DELETE, and |
| // Shutdown retries it (conditionally) inside its own budget. |
| func TestShutdownRetriesAFailedLeaseRelease(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| b.startErr = errors.New("start failed") |
| b.releaseErrs = []error{errors.New("delete lease: context deadline exceeded")} |
| @@ -269,6 +275,7 @@ func lockedLog(t *testing.T) *lockedBuf { |
| // is closed, as it is on a running agent, and the request is already past |
| // the startup gate (queued on a Bounce's stop) when the shutdown begins. |
| func TestNoWakeOnceShutdownHasBegun(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| started := make(chan struct{}) |
| close(started) |
| @@ -418,6 +425,7 @@ func TestOwedReleasesAreRetiredOnceTheirRecordsExpire(t *testing.T) { |
| // as well as its own 30 s, whichever ends first, so no proof outlives |
| // Shutdown. Unproven, the lease is still kept. |
| func TestTheLegacyProofEndsWithShutdownsBudget(t *testing.T) { |
| + t.Parallel() |
| a, b, _ := arrangeQueuedRewake(t, "slowproof.example.com") |
| b.mu.Lock() |
| b.legacyBlock = true |
| @@ -442,6 +450,7 @@ func TestTheLegacyProofEndsWithShutdownsBudget(t *testing.T) { |
| // ends when Shutdown's budget does. Cut short, it is unproven, and the |
| // lease is kept. |
| func TestAProofRunningBeforeShutdownEndsWithItsBudget(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| b.mu.Lock() |
| b.startLegacyFail, b.legacyBlock = true, true |
diff --git a/internal/activator/startup_test.go b/internal/activator/startup_test.go| index 8a36e7674..8a06481f2 100644 |
| --- a/internal/activator/startup_test.go |
| +++ b/internal/activator/startup_test.go |
| @@ -91,6 +91,7 @@ func (s startupRoutes) Get(host string) (*registry.Route, error) { |
| // wake in progress, so the retry is routed afresh, never sent to the old |
| // socket. |
| func TestStartupHeldRequestRefusesStaleUpstream(t *testing.T) { |
| + t.Parallel() |
| for _, moved := range []bool{false, true} { |
| a := New(Config{}, &fakeBackend{}, nil) |
| ready := make(chan struct{}) |
diff --git a/internal/activator/stop_returned_sidecar_test.go b/internal/activator/stop_returned_sidecar_test.go| index 904599c87..43b0a44ee 100644 |
| --- a/internal/activator/stop_returned_sidecar_test.go |
| +++ b/internal/activator/stop_returned_sidecar_test.go |
| @@ -173,6 +173,7 @@ func TestANestedStopKeepsTheLeaseOverAReturnedSidecarItCannotProve(t *testing.T) |
| // stop cannot prove its exit, is held to the same rule: the lease is kept |
| // until it has exited. |
| func TestANestedStopKeepsTheLeaseOverItsCoStoppedSidecarUntilItExits(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _ := testActivator(t, time.Hour) |
| h := &hostEmptyFake{fakeBackend: b} |
| a.backend = h |
diff --git a/internal/activator/swap_executed_test.go b/internal/activator/swap_executed_test.go| index 3d02cdb81..1e800a15e 100644 |
| --- a/internal/activator/swap_executed_test.go |
| +++ b/internal/activator/swap_executed_test.go |
| @@ -23,6 +23,7 @@ import ( |
| // outcome. These cases cover each mode's refusals before the exec, its |
| // abandonments after the exec, and its executed outcomes. |
| func TestSwapResultReportsWhetherTheAttemptsBuildExecuted(t *testing.T) { |
| + t.Parallel() |
| for _, tc := range []struct { |
| name string |
| mode SwapMode |
diff --git a/internal/activator/swap_hold_exec_test.go b/internal/activator/swap_hold_exec_test.go| index 3543c97dd..fc7f0933c 100644 |
| --- a/internal/activator/swap_hold_exec_test.go |
| +++ b/internal/activator/swap_hold_exec_test.go |
| @@ -41,6 +41,7 @@ func (b *prepHookBackend) GenTier(rt registry.Route, build string) cgroup.Limit |
| // Restore or the rest of the preparation, or a fence set there, execs |
| // nothing, and the attempt is abandoned (NoOwnership). |
| func TestSwapHoldChecksItsLeaseAndAttemptAtTheExec(t *testing.T) { |
| + t.Parallel() |
| lose := func(f *swapFixture) func() { |
| var once sync.Once |
| return func() { |
| @@ -93,6 +94,7 @@ func TestSwapHoldChecksItsLeaseAndAttemptAtTheExec(t *testing.T) { |
| // unproven-start sentinel, or a process that then fails its health check, |
| // executed: the hold is down. |
| func TestSwapHoldExecEvidenceIsTheSpawnResult(t *testing.T) { |
| + t.Parallel() |
| for _, tc := range []struct { |
| name string |
| arrange func(f *swapFixture) |
diff --git a/internal/activator/swap_hold_test.go b/internal/activator/swap_hold_test.go| index b518623d2..ae11efc82 100644 |
| --- a/internal/activator/swap_hold_test.go |
| +++ b/internal/activator/swap_hold_test.go |
| @@ -22,6 +22,7 @@ import ( |
| // generation, which execs the attempt's staged build while the live link |
| // names something else. This is the witness for mutation M3. |
| func TestSwapWitnessHoldExecsTheStagedBuild(t *testing.T) { |
| + t.Parallel() |
| f := newSwapFixture(t) |
| f.b.execVerFn = func() string { return "v3" } |
| gate := make(chan struct{}) |
| @@ -188,6 +189,7 @@ func TestSwapHoldFailureIsDown(t *testing.T) { |
| // hold build that executed is held down). Reported down, plan 5 records a |
| // failure of a build that never ran and halts its cohort. |
| func TestSwapHoldRefusedBeforeTheExecIsAbandoned(t *testing.T) { |
| + t.Parallel() |
| for _, tc := range []struct { |
| name string |
| arrange func(f *swapFixture) |
| @@ -259,6 +261,7 @@ func (s *lockedSink) errors() []string { |
| // pinned build that ran and failed is reported in the wording any failed |
| // wake already uses. |
| func TestSwapHoldWakeEventsCarryNoSwapText(t *testing.T) { |
| + t.Parallel() |
| t.Run("superseded before the exec emits nothing", func(t *testing.T) { |
| f := newSwapFixture(t) |
| sink := &lockedSink{} |
| @@ -381,6 +384,7 @@ func (k *killingSwapBackend) setFail(f func(uint64) bool) { |
| // releases the lease over a surviving descendant and lets a new wake start |
| // beside it. |
| func TestSwapHoldFailedWakeKeepsTheLeaseUntilItsChildIsEmpty(t *testing.T) { |
| + t.Parallel() |
| f := newSwapFixture(t) |
| kb := &killingSwapBackend{fakeSwapBackend: f.b} |
| kb.setFail(func(n uint64) bool { return n != f.incumbent.N }) // the hold's new generation only |
| @@ -406,6 +410,7 @@ func TestSwapHoldFailedWakeKeepsTheLeaseUntilItsChildIsEmpty(t *testing.T) { |
| // sweep's retried stop and proof hold. Retaining only when the leader had |
| // exited released the lease over a live process. |
| func TestSwapHoldFailedWakeWhoseLeaderNeverStopsKeepsTheLease(t *testing.T) { |
| + t.Parallel() |
| f := newSwapFixture(t) |
| f.b.gmu.Lock() |
| f.b.genStuck = true |
| @@ -428,6 +433,7 @@ func TestSwapHoldFailedWakeWhoseLeaderNeverStopsKeepsTheLease(t *testing.T) { |
| // newer attempt or the agent's shutdown abandons the hold, and nothing is |
| // spawned. |
| func TestSwapHoldRefusedAtTheStop(t *testing.T) { |
| + t.Parallel() |
| for _, tc := range []struct { |
| name string |
| act func(t *testing.T, f *swapFixture) |
| @@ -498,6 +504,7 @@ func TestSwapHoldClaimsOnNeedsPins(t *testing.T) { |
| // pinned wake failed and the hold answered SwapDown for a build that never |
| // ran. Swaps off refuses a hold as it refuses an overlap. |
| func TestSwapHoldRefusesARowThatCannotRunAStagedBuild(t *testing.T) { |
| + t.Parallel() |
| for _, tc := range []struct { |
| name string |
| arrange func(f *swapFixture) |
diff --git a/internal/activator/swap_overlap_test.go b/internal/activator/swap_overlap_test.go| index d2d3fc582..d95a8de93 100644 |
| --- a/internal/activator/swap_overlap_test.go |
| +++ b/internal/activator/swap_overlap_test.go |
| @@ -69,6 +69,7 @@ func TestSwapOverlapSwitchesAfterHealthAndRetiresTheIncumbent(t *testing.T) { |
| // not (512 MiB). Summing one tier twice gives a 1 GiB parent (or 2 GiB), |
| // and restoring to the live tier leaves the wrong limit after a failure. |
| func TestSwapOverlapUsesEachArtifactsTier(t *testing.T) { |
| + t.Parallel() |
| const gib, half = int64(1 << 30), int64(1 << 29) |
| t.Run("switched: restored to the successor's tier", func(t *testing.T) { |
| f := newSwapFixture(t) |
| @@ -101,6 +102,7 @@ func TestSwapOverlapUsesEachArtifactsTier(t *testing.T) { |
| // The "swapped" event carries the retiring generation's drain time (spec |
| // §9.4), so the result measures it: from SIGTERM to the proven exit. |
| func TestSwapOverlapReportsTheDrainTime(t *testing.T) { |
| + t.Parallel() |
| f := newSwapFixture(t) |
| gate := make(chan struct{}) |
| f.b.lastProc().setStopGate(gate) |
| @@ -117,6 +119,7 @@ func TestSwapOverlapReportsTheDrainTime(t *testing.T) { |
| // A successor that fails its health check, fails to spawn, or exits before |
| // the switch leaves the incumbent current and untouched (spec §8.1). |
| func TestSwapOverlapFailureKeepsTheIncumbent(t *testing.T) { |
| + t.Parallel() |
| for _, tc := range []struct { |
| name string |
| arrange func(f *swapFixture) |
| @@ -160,6 +163,7 @@ func TestSwapOverlapFailureKeepsTheIncumbent(t *testing.T) { |
| // Every refusal spawns nothing. Swaps off is the flag-off test. |
| func TestSwapRefusals(t *testing.T) { |
| + t.Parallel() |
| for _, tc := range []struct { |
| name string |
| arrange func(t *testing.T, f *swapFixture, at *Attempt) |
| @@ -420,6 +424,7 @@ func useClaimSpawn(f *swapFixture) { |
| // Claims on: a refused selection or a refused check, before the spawn or |
| // at the switch, abandons the attempt and is never retried as claims off. |
| func TestSwapClaimsOn(t *testing.T) { |
| + t.Parallel() |
| for _, tc := range []struct { |
| name string |
| sel func() hostwrite.Selector |
diff --git a/internal/activator/swap_parent_repair_test.go b/internal/activator/swap_parent_repair_test.go| index c1443aefa..7d005ea61 100644 |
| --- a/internal/activator/swap_parent_repair_test.go |
| +++ b/internal/activator/swap_parent_repair_test.go |
| @@ -44,6 +44,7 @@ func TestSwapOverlapRestoresAPartlyRaisedParent(t *testing.T) { |
| // admitted onto it (it would raise again under a pending restore), and |
| // the sweep retries the restore, outside a.mu, until it lands. |
| func TestSwapOverlapRetriesAFailedParentRestore(t *testing.T) { |
| + t.Parallel() |
| f := newSwapFixture(t) |
| f.b.gmu.Lock() |
| f.b.restoreFails = 1 |
| @@ -75,6 +76,7 @@ func TestSwapOverlapRetriesAFailedParentRestore(t *testing.T) { |
| // delegated manager) changed nothing: no restore is owed, and the next |
| // overlap is not held busy by an obligation that could never clear. |
| func TestSwapOverlapOwesNoRestoreForARaiseRefusedBeforeAnyWrite(t *testing.T) { |
| + t.Parallel() |
| for _, refusal := range []error{errNoGenContainment, cgroup.ErrNoDelegation} { |
| f := newSwapFixture(t) |
| f.b.gmu.Lock() |
diff --git a/internal/activator/swap_parent_retry_join_test.go b/internal/activator/swap_parent_retry_join_test.go| index 37ae739cf..ae3e40f9e 100644 |
| --- a/internal/activator/swap_parent_retry_join_test.go |
| +++ b/internal/activator/swap_parent_retry_join_test.go |
| @@ -66,6 +66,7 @@ func joinSignal(f *swapFixture) chan struct{} { |
| // old entry's single-generation tier after a later overlap raised the |
| // host's parent for two children. |
| func TestAParentRestoreRetryIsJoinedByTheEntrysStop(t *testing.T) { |
| + t.Parallel() |
| t.Run("a terminal stop joins the retry in flight", func(t *testing.T) { |
| f := newSwapFixture(t) |
| owedRestore(t, f) |
diff --git a/internal/activator/swap_review_test.go b/internal/activator/swap_review_test.go| index 98c74fc34..df2a4c59b 100644 |
| --- a/internal/activator/swap_review_test.go |
| +++ b/internal/activator/swap_review_test.go |
| @@ -19,6 +19,7 @@ import ( |
| // boots the installed build through the live link, and nothing of the swap |
| // survives the restart. This is the witness for mutations M1, M2 and M4. |
| func TestSwapWitnessAgentRestartMidSwap(t *testing.T) { |
| + t.Parallel() |
| restart := func(t *testing.T, f *swapFixture) { |
| t.Helper() |
| f.b.execVerFn = func() string { return "v2" } // root installed v2 |
diff --git a/internal/activator/swap_schedule_test.go b/internal/activator/swap_schedule_test.go| index 4ade5e8d4..247ab07b7 100644 |
| --- a/internal/activator/swap_schedule_test.go |
| +++ b/internal/activator/swap_schedule_test.go |
| @@ -22,6 +22,7 @@ import ( |
| // running. The successor's writes apply throughout, and a hibernating stop |
| // after the swap retires nothing. |
| func TestRetiredGenNamesOnlyTheRetiringGeneration(t *testing.T) { |
| + t.Parallel() |
| f := newSwapFixture(t) |
| duringHealth := map[string]bool{} |
| f.a.genHealth = func(_ context.Context, g Gen) error { |
| @@ -94,6 +95,7 @@ func (a *Service) schedSwitchWaiting(host string) bool { |
| // generation is current), and once the switch has happened every write of |
| // the retired generation is refused. |
| func TestScheduleWriteIsSerializedWithTheSwitch(t *testing.T) { |
| + t.Parallel() |
| f := newSwapFixture(t) |
| entered, release := make(chan struct{}), make(chan struct{}) |
| wrote := make(chan bool, 1) |
diff --git a/internal/activator/swap_sidecar_test.go b/internal/activator/swap_sidecar_test.go| index 388b19ddf..326f91549 100644 |
| --- a/internal/activator/swap_sidecar_test.go |
| +++ b/internal/activator/swap_sidecar_test.go |
| @@ -58,6 +58,7 @@ func (f *swapFixture) generationSwitched() bool { |
| // A generation that names its file runs it whatever the flag (ruling |
| // R-EXEC-SIDECAR; TestLegacySidecarOwnershipCheckRunsWithSwapOff). |
| func TestLegacySidecarUsesTheLiveLinkWithSwapOff(t *testing.T) { |
| + t.Parallel() |
| f := newSwapFixture(t) |
| f.a.gens = false |
| f.b.sidecarStaged = true |
| @@ -144,6 +145,7 @@ func TestLegacySidecarOwnershipCheckRunsWithSwapOff(t *testing.T) { |
| // wake tail all exec the CURRENT generation's captured staged path (spec |
| // §7.5). The live link names v3 throughout and is never used. |
| func TestLegacySidecarRunsTheServingGeneration(t *testing.T) { |
| + t.Parallel() |
| t.Run("late attachment and the supervisor", func(t *testing.T) { |
| f := newSwapFixture(t) |
| f.b.execVerFn = func() string { return "v3" } |
| @@ -200,6 +202,7 @@ func TestLegacySidecarRunsTheServingGeneration(t *testing.T) { |
| // restart that falls due meanwhile cannot start a second sidecar (review |
| // focus 4). |
| func TestSwapSidecarReplacementHoldsTheSupervisorOff(t *testing.T) { |
| + t.Parallel() |
| f := newSwapFixture(t) |
| f.b.sidecarStaged = true |
| f.a.sweepOnce() |
| @@ -238,6 +241,7 @@ func TestSwapSidecarReplacementHoldsTheSupervisorOff(t *testing.T) { |
| // replaceSidecar) leaves the supervisor's old-build sidecar installed with |
| // no replacement pending: the third spawn never comes. |
| func TestSwapSidecarReplacementJoinsAnInFlightSupervisor(t *testing.T) { |
| + t.Parallel() |
| f := newSwapFixture(t) |
| f.b.sidecarStaged = true |
| f.a.sweepOnce() |
| @@ -287,6 +291,7 @@ func TestSwapSidecarReplacementJoinsAnInFlightSupervisor(t *testing.T) { |
| // close its join at once, so the replacement started beside a sidecar |
| // still draining on the old build. |
| func TestSwapSidecarNeverRunsBesideTheRetiredGenerationsSidecar(t *testing.T) { |
| + t.Parallel() |
| f := newSwapFixture(t) |
| f.b.sidecarStaged = true |
| f.a.sweepOnce() |
| @@ -330,6 +335,7 @@ func TestSwapSidecarNeverRunsBesideTheRetiredGenerationsSidecar(t *testing.T) { |
| // close its join at once, so the stop released the lease while the new |
| // sidecar was still running. |
| func TestSwapStopJoinsTheSidecarAStartLeftBehind(t *testing.T) { |
| + t.Parallel() |
| f := newSwapFixture(t) |
| f.b.sidecarStaged = true |
| f.a.sweepOnce() |
| @@ -385,6 +391,7 @@ func TestSwapStopJoinsTheSidecarAStartLeftBehind(t *testing.T) { |
| // stops it again until it exits. Logging it and moving on let the next wake |
| // run a second sidecar beside a live one. |
| func TestAStraySidecarIsNeverForgotten(t *testing.T) { |
| + t.Parallel() |
| f := newSwapFixture(t) |
| f.a.Bounce(f.host) |
| waitFor(t, "the instance asleep", func() bool { return f.a.entryCount() == 0 }) |
| @@ -439,6 +446,7 @@ func (a *Service) stopPending(host string) bool { |
| // gone. Accepting Exited() started the new sidecar beside the descendant |
| // and released the lease under it. |
| func TestASidecarsDescendantBlocksItsReplacementAndTheLease(t *testing.T) { |
| + t.Parallel() |
| f := newSwapFixture(t) |
| f.b.sidecarStaged = true |
| f.a.sweepOnce() |
| @@ -485,6 +493,7 @@ func TestASidecarsDescendantBlocksItsReplacementAndTheLease(t *testing.T) { |
| // exits but a process it forked is left in the cgroup, so the stop keeps the |
| // lease (releasable) until the sweep proves the cgroup empty. |
| func TestATerminalStopKeepsTheLeaseOverASidecarsDescendant(t *testing.T) { |
| + t.Parallel() |
| f := newSwapFixture(t) |
| f.b.sidecarStaged = true |
| f.a.sweepOnce() |
| @@ -514,6 +523,7 @@ func TestATerminalStopKeepsTheLeaseOverASidecarsDescendant(t *testing.T) { |
| // too: if something it forked is left in the cgroup, the old one becomes a |
| // tracked stray and no new sidecar starts beside it. |
| func TestASupervisedRestartWaitsForTheCrashedSidecarsDescendant(t *testing.T) { |
| + t.Parallel() |
| f := newSwapFixture(t) |
| f.b.sidecarStaged = true |
| f.a.sweepOnce() |
| @@ -540,6 +550,7 @@ func TestASupervisedRestartWaitsForTheCrashedSidecarsDescendant(t *testing.T) { |
| // A legacy sidecar left on a retired generation's build (a replacement whose |
| // stop of it was unproven) is replaced by the sweep's retry. |
| func TestSweepReplacesASidecarLeftOnARetiredGeneration(t *testing.T) { |
| + t.Parallel() |
| f := newSwapFixture(t) |
| f.b.sidecarStaged = true |
| f.a.sweepOnce() |
| @@ -608,6 +619,7 @@ func TestStartSidecarGenKeepsTheRoleSidecarChild(t *testing.T) { |
| // reads the live link to decide, because a repoint can land between that |
| // read and the start. |
| func TestExecBackendStartSidecarGen(t *testing.T) { |
| + t.Parallel() |
| stage := func(t *testing.T, b *ExecBackend, host string) (v2, marker string) { |
| t.Helper() |
| marker = filepath.Join(t.TempDir(), "argv0") |
diff --git a/internal/activator/unitproc_gen_test.go b/internal/activator/unitproc_gen_test.go| index ac0854730..028878c1e 100644 |
| --- a/internal/activator/unitproc_gen_test.go |
| +++ b/internal/activator/unitproc_gen_test.go |
| @@ -103,6 +103,7 @@ func TestGenerationUnitProcStopsAndWatchesItsOwnUnit(t *testing.T) { |
| } |
| func TestGenerationUnitProcConsumesOnlyItsOwnRecordAndLaunchFile(t *testing.T) { |
| + t.Parallel() |
| ctl := &fakeControl{} |
| unit := &fakeUnit{} |
| b, rt := nestedBackend(t, ctl, unit) |
| @@ -295,6 +296,7 @@ func genProcFixture(t *testing.T) (*fakeControl, *fakeUnit, *ExecBackend, regist |
| // gone: its launch file here, and the control service's drop-in and exe |
| // layout through a stop of the inactive unit. A later Stop sends nothing. |
| func TestGenerationNaturalExitFinalisesItsFiles(t *testing.T) { |
| + t.Parallel() |
| ctl, unit, b, rt, launch := genProcFixture(t) |
| unit.set("active", "inv7", true, true) |
| up := b.newUnitProcFor(rt, instancectl.RoleRun, 7, b.pathsFor(rt), "inv7", nil, nil) |
| @@ -318,6 +320,7 @@ func TestGenerationNaturalExitFinalisesItsFiles(t *testing.T) { |
| // is proven its files go too. The stop that proved it already removed the |
| // control service's files, so it is not asked twice. |
| func TestGenerationForeignInvocationExitFinalisesItsFiles(t *testing.T) { |
| + t.Parallel() |
| ctl, unit, b, rt, launch := genProcFixture(t) |
| ctl.stopHook = func(string, instancectl.Role) { unit.set("inactive", "", false, false) } |
| unit.set("active", "inv7", true, true) |
diff --git a/internal/activator/unitproc_test.go b/internal/activator/unitproc_test.go| index cfaca58bb..d65dca226 100644 |
| --- a/internal/activator/unitproc_test.go |
| +++ b/internal/activator/unitproc_test.go |
| @@ -514,10 +514,16 @@ func TestStopClosesListenersAfterDoneWithNoSecondStopCall(t *testing.T) { |
| // TestSeedUnitFailureStopsBeforeCleanupAndKeepsTheDBWhenProcessesRemain |
| // covers the ordering rule: Control.Stop for the seed unit runs, and is |
| // proven by an empty cgroup, BEFORE any database cleanup, and when the |
| -// cgroup never reports empty, nothing is removed at all. The |
| -// wait is bounded by actStopTimeout, a const this package does not let a |
| -// test shrink, so this test genuinely waits out that bound. |
| +// cgroup never reports empty, nothing is removed at all. The wait is |
| +// bounded by roleEmptyBound (actStopTimeout in production), which this test |
| +// genuinely waits out, shrunk to a fraction of a second. |
| func TestSeedUnitFailureStopsBeforeCleanupAndKeepsTheDBWhenProcessesRemain(t *testing.T) { |
| + if roleEmptyBound != actStopTimeout { |
| + t.Fatalf("roleEmptyBound = %v, want actStopTimeout (%v) outside a test", roleEmptyBound, actStopTimeout) |
| + } |
| + old := roleEmptyBound |
| + roleEmptyBound = 300 * time.Millisecond |
| + t.Cleanup(func() { roleEmptyBound = old }) |
| ctl := &fakeControl{seed: instancectl.SeedResponse{InvocationID: "s1", Result: "exit-code", Status: "3", Code: "exited"}} |
| unit := &fakeUnit{} |
| b, rt := nestedBackend(t, ctl, unit) |
| @@ -531,7 +537,7 @@ func TestSeedUnitFailureStopsBeforeCleanupAndKeepsTheDBWhenProcessesRemain(t *te |
| } |
| } |
| // The cgroup stays populated for the whole wait: a process from the |
| - // seed survived SIGKILL and is still there when actStopTimeout elapses. |
| + // seed survived SIGKILL and is still there when the bound elapses. |
| unit.set("active", "s1", true, true) |
| err := b.Seed(context.Background(), rt) |
| if err == nil || !strings.Contains(err.Error(), "processes remain") { |
diff --git a/internal/activator/wakecheck.go b/internal/activator/wakecheck.go| index 803840d9b..e476eebc9 100644 |
| --- a/internal/activator/wakecheck.go |
| +++ b/internal/activator/wakecheck.go |
| @@ -52,8 +52,8 @@ type HeldWake struct { |
| } |
| // heldPoll is how often a held wake re-reads the live link while it waits |
| -// for root to install its target. |
| -const heldPoll = 250 * time.Millisecond |
| +// for root to install its target. A var so tests can shrink it. |
| +var heldPoll = 250 * time.Millisecond |
| // wakeCheck is settings spec §4.3 rule 3's wake half, with its own admission |
| // rule so it holds with CARLOS_SWAP off and no planner (Astra I3 #2, #5): |
diff --git a/internal/activator/wakecheck_test.go b/internal/activator/wakecheck_test.go| index a7275ea85..fff482043 100644 |
| --- a/internal/activator/wakecheck_test.go |
| +++ b/internal/activator/wakecheck_test.go |
| @@ -33,6 +33,16 @@ func shrinkWakeWait(t *testing.T, d time.Duration) { |
| t.Cleanup(func() { wakeWaitBound = old }) |
| } |
| +// shrinkHeldPoll sets how often a held wake polls while it waits, for one |
| +// test. It asks for the answer again every heldReask polls, so a test that |
| +// waits for a re-ask waits heldReask*d. |
| +func shrinkHeldPoll(t *testing.T, d time.Duration) { |
| + t.Helper() |
| + old := heldPoll |
| + heldPoll = d |
| + t.Cleanup(func() { heldPoll = old }) |
| +} |
| + |
| // asks records RequestStage calls as host@build. |
| type asks struct { |
| mu sync.Mutex |
| @@ -202,6 +212,7 @@ func TestWithNoPlannerARetainedCopyRunsWithoutARepoint(t *testing.T) { |
| // the drift is gone and the instance keeps serving. The control (root never |
| // repoints) shows the sweep does cycle it. |
| func TestARetainedWakeIsNotCycledOnceRootRepoints(t *testing.T) { |
| + t.Parallel() |
| for _, repoint := range []bool{true, false} { |
| a, b, clk, _ := testActivator(t, time.Hour) |
| b.setIdentity("v3") |
| @@ -486,6 +497,7 @@ func TestAHeldWakeEndsItsWaitWhenTheAnswerChanges(t *testing.T) { |
| for _, c := range cases { |
| t.Run(c.name, func(t *testing.T) { |
| shrinkWakeWait(t, 20*time.Second) |
| + shrinkHeldPoll(t, 25*time.Millisecond) // the re-ask comes 8 polls in |
| a, b := identityActivator(t, "v3") |
| var asked atomic.Int32 |
| a.HeldTarget = func(string) HeldWake { |
| @@ -563,7 +575,10 @@ func TestAHeldUnitRowRefusesALinkThatMovedAfterTheWait(t *testing.T) { |
| // to come back (the agent's refresh reads the bucket, up to its own 5 s |
| // budget) never holds the wake past wakeWaitBound. |
| func TestAHeldWakesReaskNeverOutlastsTheWait(t *testing.T) { |
| - shrinkWakeWait(t, 1900*time.Millisecond) |
| + // The re-ask comes 8 polls (175 ms) into a 400 ms wait, and its answer |
| + // takes 5 s: the wait must end at its own bound. |
| + shrinkHeldPoll(t, 25*time.Millisecond) |
| + shrinkWakeWait(t, 400*time.Millisecond) |
| a, b := identityActivator(t, "v3") |
| var asked atomic.Int32 |
| a.HeldTarget = func(string) HeldWake { |
| @@ -586,7 +601,7 @@ func TestAHeldWakesReaskNeverOutlastsTheWait(t *testing.T) { |
| t.Fatalf("asked %d times: the wait never asked again", asked.Load()) |
| } |
| if took >= 3*time.Second { |
| - t.Fatalf("the wait ran %v, past its 1.9 s bound: the re-ask outlasted it", took) |
| + t.Fatalf("the wait ran %v, past its 0.4 s bound: the re-ask outlasted it", took) |
| } |
| if b.startCount() != 0 { |
| t.Fatal("a wake that never saw its target ran something") |
| @@ -643,7 +658,10 @@ func TestAWakeRootHasNotDecidedForWaitsForRoot(t *testing.T) { |
| } |
| for _, c := range cases { |
| t.Run(c.name, func(t *testing.T) { |
| - shrinkWakeWait(t, 3*time.Second) |
| + // Root's answer is asked again 8 polls (200 ms) in; a wake root |
| + // never decides for fails at the bound. |
| + shrinkHeldPoll(t, 25*time.Millisecond) |
| + shrinkWakeWait(t, time.Second) |
| a, b := identityActivator(t, "v3") |
| var asked atomic.Bool |
| a.HeldTarget = func(string) HeldWake { |
diff --git a/internal/activator/wakeidentity_route_test.go b/internal/activator/wakeidentity_route_test.go| index 2e3cc47e1..f1894599b 100644 |
| --- a/internal/activator/wakeidentity_route_test.go |
| +++ b/internal/activator/wakeidentity_route_test.go |
| @@ -17,6 +17,7 @@ import ( |
| // reloads. The wake refreshes the row, finds the legacy route, and stops: |
| // no lease is taken and nothing is restored. The edge is never reloaded. |
| func TestAVerifyOnACachedImportRowAfterItsRollbackWakesNothing(t *testing.T) { |
| + t.Parallel() |
| a, b, _, reg := testActivator(t, time.Hour) |
| a.Routes = reg |
| const host = "old.shop.example.com" |
| @@ -128,6 +129,7 @@ func TestWakeRouteChanged(t *testing.T) { |
| // must still wake; a sidecar whose identity changed under the wake is still |
| // refused before any lease. |
| func TestWakeWorkerWithTheRegistryWired(t *testing.T) { |
| + t.Parallel() |
| a, b, _, reg := testActivator(t, 10*time.Second) |
| a.Routes = reg |
| rt := testWorkerRoute("bot.shop.example.com") |
diff --git a/internal/activator/wakeidentity_test.go b/internal/activator/wakeidentity_test.go| index 0b220a689..6c317d1fd 100644 |
| --- a/internal/activator/wakeidentity_test.go |
| +++ b/internal/activator/wakeidentity_test.go |
| @@ -261,6 +261,7 @@ func (a *Service) entryGen(host string) (Gen, bool) { |
| // --- the wake --- |
| func TestTheBuildStateIsWrittenBeforeTheWake(t *testing.T) { |
| + t.Parallel() |
| a, b := identityActivator(t, "v1") |
| l := &execLog{spawns: b.startCount} |
| a.BeforeExec = l.hook |
| @@ -287,6 +288,7 @@ func TestTheBuildStateIsWrittenBeforeTheWake(t *testing.T) { |
| } |
| func TestAFailedWriteBlocksTheSpawn(t *testing.T) { |
| + t.Parallel() |
| a, b := identityActivator(t, "v1") |
| b.restoreFresh = true |
| a.BeforeExec = (&execLog{fail: failAll}).hook |
| @@ -347,6 +349,7 @@ func TestTheSeedRunsTheRecordedIdentity(t *testing.T) { |
| // that build (never the live link), and startPinnedAt does not write it |
| // again. |
| func TestAPlannedWakeRecordsBeforeTheSeed(t *testing.T) { |
| + t.Parallel() |
| next := stagedID(t, t.TempDir(), "v3") |
| a, b, _ := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap}) |
| b.fakeBackend.setIdentity("v1") |
| @@ -376,6 +379,7 @@ func TestAPlannedWakeRecordsBeforeTheSeed(t *testing.T) { |
| // chooses the identity, so a replan lands first and the seed and the boot |
| // run one build, recorded once. |
| func TestAReplanBeforeTheSeedSeedsAndBootsOneBuild(t *testing.T) { |
| + t.Parallel() |
| dir := t.TempDir() |
| stale, next := stagedID(t, dir, "v2"), stagedID(t, dir, "v3") |
| a, b, _ := planActivator(t, WakePlan{}) |
| @@ -411,6 +415,7 @@ func TestAReplanBeforeTheSeedSeedsAndBootsOneBuild(t *testing.T) { |
| // execs. The first plan's boot never ran, so its pending record is never |
| // written (amadan #272). |
| func TestAReplanAfterTheRecordRecordsTheNewBuild(t *testing.T) { |
| + t.Parallel() |
| dir := t.TempDir() |
| stale, next := stagedID(t, dir, "v2"), stagedID(t, dir, "v3") |
| a, b, _ := planActivator(t, WakePlan{}) |
| @@ -468,6 +473,7 @@ func TestANilHookRunsTheCapturedFileAndRecordsNothing(t *testing.T) { |
| // live-link seed, start and sidecar, nothing published or recorded. With a |
| // hook it is refused. |
| func TestAnUnversionedLinkWithNoHookWakesAsToday(t *testing.T) { |
| + t.Parallel() |
| a, b := sidecarActivator(t, "") |
| b.fakeBackend.setIdentity("") |
| b.mu.Lock() |
| @@ -601,6 +607,7 @@ func TestANestedUnversionedLinkWithNoHookWakesAsToday(t *testing.T) { |
| // A refused record starts nothing on any wake kind: no seed, no instance, |
| // no worker, no planned build, and the member reads the fence's wording. |
| func TestARefusingHookStartsNothing(t *testing.T) { |
| + t.Parallel() |
| a, b := identityActivator(t, "v1") |
| b.restoreFresh, b.sidecarStaged = true, true |
| a.BeforeExec = (&execLog{fail: failAll}).hook |
| @@ -735,6 +742,7 @@ func TestAFlagOnUnplannedWakeRunsTheRecordedFileAtGenN(t *testing.T) { |
| // before its build state does, so root cannot sweep the file between the |
| // capture and the exec. A refused wake's pending build is dropped again. |
| func TestTheColdWakePublishesBeforeItRecords(t *testing.T) { |
| + t.Parallel() |
| a, b := identityActivator(t, "v1") |
| a.GenerationsFile = filepath.Join(t.TempDir(), "zdd", "generations.json") |
| path := fakeIdentity("v1").StagedPath |
| @@ -802,6 +810,7 @@ func TestAPinnedWakeSeedsThePin(t *testing.T) { |
| // repoint during the record cannot change the worker's build, and its |
| // exec-time hook does not write the same build again. |
| func TestAWorkerWakeExecsTheIdentityItRecorded(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _, _ := stagedActivator(t) |
| b.fakeBackend.setIdentity("v1") |
| l := &execLog{} |
| @@ -887,6 +896,7 @@ func TestAFlagOffSidecarRunsTheRecordedFile(t *testing.T) { |
| // A supervisor restart is a new exec against the instance's data: it |
| // records its generation's build first, and runs that file. |
| func TestASidecarRestartRecordsItsBuild(t *testing.T) { |
| + t.Parallel() |
| a, b := sidecarActivator(t, "v1") |
| l := &execLog{spawns: func() int { return len(b.sidecarsOn()) }} |
| a.BeforeExec = l.hook |
| @@ -911,6 +921,7 @@ func TestASidecarRestartRecordsItsBuild(t *testing.T) { |
| // A refused sidecar record starts no sidecar and is logged as a sidecar |
| // start failure; the instance keeps serving, at the wake and at a restart. |
| func TestARefusedSidecarRecordStartsNoSidecar(t *testing.T) { |
| + t.Parallel() |
| a, b := sidecarActivator(t, "v1") |
| l := &execLog{} |
| a.BeforeExec = l.hook |
| @@ -967,6 +978,7 @@ func TestARefusedSidecarRecordStartsNoSidecar(t *testing.T) { |
| // this binary): its sidecar resolves the live link once, records that |
| // build, and runs that file by path. |
| func TestASidecarOnAnUnstagedGenerationResolvesRecordsAndRunsByPath(t *testing.T) { |
| + t.Parallel() |
| a, b := sidecarActivator(t, "") |
| host := "one.example.com" |
| done, err := a.beginRequest(context.Background(), identityRoute(t, host)) |
| @@ -998,6 +1010,7 @@ func TestASidecarOnAnUnstagedGenerationResolvesRecordsAndRunsByPath(t *testing.T |
| // after an overlap switch each record their generation's build before |
| // their exec (the replacement runs off any wake's context, so it writes). |
| func TestASidecarReplacementRecordsTheNewBuild(t *testing.T) { |
| + t.Parallel() |
| f := newSwapFixture(t) |
| f.b.sidecarStaged = true |
| l := &execLog{spawns: func() int { return len(f.b.sidecarGenPaths()) }} |
| @@ -1021,6 +1034,7 @@ func TestASidecarReplacementRecordsTheNewBuild(t *testing.T) { |
| // A sidecar restart asks ownership again after its record: a lease lost |
| // while the build state was written execs nothing. |
| func TestASidecarExecsOnlyIfStillOwnedAfterItsRecord(t *testing.T) { |
| + t.Parallel() |
| a, b := sidecarActivator(t, "v1") |
| host := "one.example.com" |
| var lose atomic.Bool |
diff --git a/internal/activator/wakeplan_test.go b/internal/activator/wakeplan_test.go| index 686d6dc5b..b188a6a45 100644 |
| --- a/internal/activator/wakeplan_test.go |
| +++ b/internal/activator/wakeplan_test.go |
| @@ -307,6 +307,7 @@ func TestAPlannedWakeRechecksItsBuildAtTheExec(t *testing.T) { |
| // never execs, the wake is abandoned (errPlanStale), and nothing is |
| // recorded against the build. |
| func TestThePlanCheckAtTheExecRefusesWithoutRecording(t *testing.T) { |
| + t.Parallel() |
| dir := t.TempDir() |
| good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| a, b, evs := planActivator(t, WakePlan{}) |
| @@ -343,6 +344,7 @@ func TestThePlanCheckAtTheExecRefusesWithoutRecording(t *testing.T) { |
| // gate. A renewal that fails after the restore execs nothing: no |
| // generation starts, nothing is recorded, nothing falls back. |
| func TestAPlannedWakeWhoseLeaseIsLostExecsNothing(t *testing.T) { |
| + t.Parallel() |
| dir := t.TempDir() |
| good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| a, b, evs := planActivator(t, WakePlan{}) |
| @@ -533,12 +535,24 @@ func TestAClaimedFirstWakeReportsLikeAnUnclaimedOne(t *testing.T) { |
| } |
| } |
| +// shrinkClaimStartBackoff sets startAdmitted's backoff base for one test. |
| +func shrinkClaimStartBackoff(t *testing.T, d time.Duration) { |
| + t.Helper() |
| + old := claimStartBackoff |
| + claimStartBackoff = d |
| + t.Cleanup(func() { claimStartBackoff = old }) |
| +} |
| + |
| // PF17: every claims-on launch refuses on this base (RoleLaunch.Start |
| // answers ErrClaimIsolationUnavailable before its exec hook). A planned |
| // claimed wake carries the plan into the start policy (the pin, the good |
| // build, the health cap, the ownership gate and the selection gate), meets |
| // the closed launch, and records nothing: no failure event, no fallback. |
| +// The refused start is retried on startAdmitted's backoff, whose schedule |
| +// TestStartAdmittedLoopStopsAfterFiveStartsWithDoublingBackoff pins; here |
| +// it is shrunk so the retries do not take 15 s. |
| func TestAPlannedClaimedWakeRefusesAtTheLaunchRecordingNothing(t *testing.T) { |
| + shrinkClaimStartBackoff(t, 10*time.Millisecond) |
| dir := t.TempDir() |
| good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| const host = "x.example" |
| @@ -759,6 +773,7 @@ func TestAPlannedFirstWakeOnTheRealExecBackendFallsBackInTime(t *testing.T) { |
| // wake fell back, that is the good build, by its staged path, never B's |
| // and never the live link. |
| func TestAFallenBackFirstWakeBindsItsSidecarToTheGoodBuild(t *testing.T) { |
| + t.Parallel() |
| dir := t.TempDir() |
| good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| a, b, evs := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good}) |
| @@ -1040,6 +1055,7 @@ func TestTheWorkerPublicationSentinelKeepsMainsText(t *testing.T) { |
| // event, no fallback, so no halted cohort and no hold-down. spawnPlanned |
| // that marked every error with a process as a failure recorded one. |
| func TestACancelledFirstWakeRecordsNothing(t *testing.T) { |
| + t.Parallel() |
| dir := t.TempDir() |
| good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| a, b, evs := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good}) |
| @@ -1153,6 +1169,7 @@ func TestWithoutAPlannerTheWakeIsTodays(t *testing.T) { |
| } |
| func TestDriftFeedsTheSchedulerInsteadOfStopping(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _ := testActivator(t, time.Hour) |
| a.liveVer = func(string) (string, error) { return "v2", nil } |
| b.execVerFn = func() string { return "v1" } |
diff --git a/internal/activator/wakerecord_recheck_test.go b/internal/activator/wakerecord_recheck_test.go| index a221e7c25..a19034af7 100644 |
| --- a/internal/activator/wakerecord_recheck_test.go |
| +++ b/internal/activator/wakerecord_recheck_test.go |
| @@ -57,6 +57,7 @@ func TestAnExecGateRefusingDuringTheRecordStartsNothing(t *testing.T) { |
| // P1, the fence: a maintenance fence raised while a cold wake's record is |
| // being written starts nothing, at the seed and at the spawn. |
| func TestAFenceRaisedDuringTheRecordStartsNothing(t *testing.T) { |
| + t.Parallel() |
| for _, seeded := range []bool{false, true} { |
| name := "spawn" |
| if seeded { |
diff --git a/internal/activator/worker_gen_test.go b/internal/activator/worker_gen_test.go| index 773fb1382..7e3c02e58 100644 |
| --- a/internal/activator/worker_gen_test.go |
| +++ b/internal/activator/worker_gen_test.go |
| @@ -226,6 +226,7 @@ func TestWorkerTerminationProvesItsHostCgroup(t *testing.T) { |
| // switch says. A respawn whose lease has been lost execs nothing, with |
| // generations on or off. |
| func TestWorkerRespawnAsksOwnershipAtItsExec(t *testing.T) { |
| + t.Parallel() |
| for _, on := range []bool{true, false} { |
| a, b, _, _ := testActivator(t, time.Hour) |
| a.gens = on |
diff --git a/internal/activator/worker_generations_test.go b/internal/activator/worker_generations_test.go| index f990a6277..22659f7e2 100644 |
| --- a/internal/activator/worker_generations_test.go |
| +++ b/internal/activator/worker_generations_test.go |
| @@ -50,6 +50,7 @@ func stringOf(m map[string][]string) string { |
| // naming it across a crash recycle while live/<host> names another build, |
| // follows a converge, and drops it once the worker stops. |
| func TestWorkerGenerationsFileFollowsTheCapturedBuild(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _, ver := stagedActivator(t) |
| path := release.GenerationsPath(t.TempDir()) |
| a.GenerationsFile = path |
| @@ -91,6 +92,7 @@ func TestWorkerGenerationsFileFollowsTheCapturedBuild(t *testing.T) { |
| // hold, is not listed by version, which would make the file unwritable; it |
| // is listed by its staged path instead (ruling R19 item 2). |
| func TestWorkerGenerationsListOnlyVersionStoreBuilds(t *testing.T) { |
| + t.Parallel() |
| a, _, _, _, ver := stagedActivator(t) |
| path := release.GenerationsPath(t.TempDir()) |
| a.GenerationsFile = path |
| @@ -344,6 +346,7 @@ func describeGenerations(gs []*release.Generations) string { |
| // and for a converge onto a newly installed build. Once the spawn has |
| // installed, the captured build replaces the pending one. |
| func TestWorkerSpawnPublishesItsBuildBeforeTheExec(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _, ver := stagedActivator(t) |
| path := release.GenerationsPath(t.TempDir()) |
| a.GenerationsFile = path |
| @@ -383,6 +386,7 @@ func TestWorkerSpawnPublishesItsBuildBeforeTheExec(t *testing.T) { |
| // can be written again, the sweep's retry publishes the set without the |
| // refused build. |
| func TestWorkerSpawnRefusedWhenItsBuildCannotBePublished(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _, _ := stagedActivator(t) |
| data := filepath.Join(t.TempDir(), "data") |
| if err := os.WriteFile(data, []byte("a file where the data dir should be"), 0o644); err != nil { |
| @@ -439,6 +443,7 @@ func TestWorkerSpawnRefusedWhenItsBuildCannotBePublished(t *testing.T) { |
| // versions/v1/app) has Version "", and is published by its staged path, |
| // which root's pin set then holds without any registry row. |
| func TestWorkerFromAChainedLinkIsPublishedByPath(t *testing.T) { |
| + t.Parallel() |
| eb := testExecBackend(t) |
| fb := &fakeBackend{} |
| reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
diff --git a/internal/activator/worker_identity_test.go b/internal/activator/worker_identity_test.go| index eef3d1d80..25c52618f 100644 |
| --- a/internal/activator/worker_identity_test.go |
| +++ b/internal/activator/worker_identity_test.go |
| @@ -46,6 +46,7 @@ func (a *Service) entryServing(host string) release.StagedIdentity { |
| // Spec §7.9: a crash recycle respawns the serving build, not the installed one. |
| func TestWorkerCrashRecyclesTheCapturedServingBuild(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _, ver := stagedActivator(t) |
| host := "bot.x.internal" |
| if err := a.WakeWorker(alwaysOnWorkerRoute(host)); err != nil { |
| @@ -72,6 +73,7 @@ func TestWorkerCrashRecyclesTheCapturedServingBuild(t *testing.T) { |
| // Spec §7.9: so does an always-on worker's max_run recycle. |
| func TestWorkerMaxRunRecyclesTheCapturedServingBuild(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _, ver := stagedActivator(t) |
| host := "bot.x.internal" |
| rt := alwaysOnWorkerRoute(host) |
| @@ -91,6 +93,7 @@ func TestWorkerMaxRunRecyclesTheCapturedServingBuild(t *testing.T) { |
| // Until plan 5 queues it, the drift check is the one path that moves a |
| // worker to a different installed build, and it reads the live link. |
| func TestWorkerConvergeStillMovesOntoTheInstalledBuild(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _, ver := stagedActivator(t) |
| host := "bot.x.internal" |
| if err := a.WakeWorker(alwaysOnWorkerRoute(host)); err != nil { |
| @@ -111,6 +114,7 @@ func TestWorkerConvergeStillMovesOntoTheInstalledBuild(t *testing.T) { |
| // Plan 5's hook: an admissible queued attempt takes the spawn. |
| func TestWorkerRecycleDefersToCoalesceRecycle(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _, _ := stagedActivator(t) |
| host := "bot.x.internal" |
| var asked []string |
| @@ -131,6 +135,7 @@ func TestWorkerRecycleDefersToCoalesceRecycle(t *testing.T) { |
| // Review Focus 4, and spec §7.9: the report is display metadata; registry |
| // Version is root's install stamp. |
| func TestNoteWorkerReadyIsDisplayOnly(t *testing.T) { |
| + t.Parallel() |
| a, b, _, reg := testActivator(t, time.Hour) |
| host := "bot.x.internal" |
| if err := reg.Upsert(testWorkerRoute(host)); err != nil { |
| @@ -161,6 +166,7 @@ func TestNoteWorkerReadyIsDisplayOnly(t *testing.T) { |
| // Spec §7.9: no spawn path writes registry Version: not a wake, a crash |
| // recycle or a converge. |
| func TestWorkerSpawnsLeaveTheInstalledStampAlone(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, reg, ver := shippingActivatorWithRegistry(t, time.Hour) |
| host := "bot.x.internal" |
| rt := alwaysOnWorkerRoute(host) |
| @@ -197,6 +203,7 @@ func TestWorkerSpawnsLeaveTheInstalledStampAlone(t *testing.T) { |
| // Review Focus 4: an app-chosen report changes nothing about what recycles, |
| // and a new child's report starts out unknown. |
| func TestWorkerReportingAnAppChosenVersionStillRecyclesItsCapturedBuild(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _, _ := stagedActivator(t) |
| host := "bot.x.internal" |
| if err := a.WakeWorker(alwaysOnWorkerRoute(host)); err != nil { |
| @@ -235,6 +242,7 @@ func waitForSpawnCalls(t *testing.T, a *Service, b *stagedFake, host string, n i |
| // for what live/<host> names. Here live/<host> still names v1, so there is |
| // no drift that could admit a converge either. |
| func TestWorkerRecycleRefusedForAChangedCopyStaysRefused(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _, _ := stagedActivator(t) |
| host := "bot.x.internal" |
| if err := a.WakeWorker(alwaysOnWorkerRoute(host)); err != nil { |
| @@ -273,6 +281,7 @@ func TestWorkerRecycleRefusedForAChangedCopyStaysRefused(t *testing.T) { |
| // Any other spawn failure keeps the captured build too. |
| func TestWorkerRecycleFailingForAnotherReasonKeepsTheCapturedBuild(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _, _ := stagedActivator(t) |
| host := "bot.x.internal" |
| if err := a.WakeWorker(alwaysOnWorkerRoute(host)); err != nil { |
| @@ -300,6 +309,7 @@ func TestWorkerRecycleFailingForAnotherReasonKeepsTheCapturedBuild(t *testing.T) |
| // converge. Recycles onto the captured build keep the drift clock, and the |
| // recycle that falls due past ConvergeGrace converges instead. |
| func TestWorkerMaxRunRecyclesConvergeOnceTheGraceHasPassed(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _, ver := stagedActivator(t) |
| host := "bot.x.internal" |
| rt := alwaysOnWorkerRoute(host) |
| @@ -355,6 +365,7 @@ func (a *Service) entryStaleSince(host string) time.Time { |
| // converges at most maxConvergeCyclesPerSweep of them, the other recycles its |
| // captured build and keeps its drift age, and converges on a later sweep. |
| func TestWorkerRecycleConvergesShareTheSweepsConvergeBudget(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _, ver := stagedActivator(t) |
| hosts := []string{"bot1.x.internal", "bot2.x.internal"} |
| for _, h := range hosts { |
| @@ -405,6 +416,7 @@ func TestWorkerRecycleConvergesShareTheSweepsConvergeBudget(t *testing.T) { |
| // build changes. A false answer recycles the captured build, and the drift |
| // keeps its age. |
| func TestWorkerRecycleNeverConvergesWhenTheSchedulerIsPresent(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _, ver := stagedActivator(t) |
| host := "bot.x.internal" |
| a.CoalesceRecycle = func(string) bool { return false } |
| @@ -508,6 +520,7 @@ func TestAnArmedBoxNeverRecyclesAnUncapturedWorkerFromTheLiveLink(t *testing.T) |
| // restart measures the drift itself, and a crash loop converges once the |
| // grace has passed. |
| func TestWorkerCrashLoopConvergesOnceTheGraceHasPassed(t *testing.T) { |
| + t.Parallel() |
| a, b, clk, _, ver := stagedActivator(t) |
| host := "bot.x.internal" |
| if err := a.WakeWorker(alwaysOnWorkerRoute(host)); err != nil { |
diff --git a/internal/activator/worker_respawn_exec_test.go b/internal/activator/worker_respawn_exec_test.go| index f170a9f33..428ca3a66 100644 |
| --- a/internal/activator/worker_respawn_exec_test.go |
| +++ b/internal/activator/worker_respawn_exec_test.go |
| @@ -20,6 +20,7 @@ import ( |
| // the child it made is shed. With generations on or off, on a backend with |
| // or without the staged capability. |
| func TestWorkerRespawnRechecksOwnershipAtTheExecAndTheInstall(t *testing.T) { |
| + t.Parallel() |
| type rig struct { |
| a *Service |
| b *fakeBackend |
diff --git a/internal/activator/worker_retained_pin_test.go b/internal/activator/worker_retained_pin_test.go| index 426450e13..f97ccf9b2 100644 |
| --- a/internal/activator/worker_retained_pin_test.go |
| +++ b/internal/activator/worker_retained_pin_test.go |
| @@ -105,6 +105,7 @@ func TestARetainedFailedWorkerWakeKeepsItsBuildPinned(t *testing.T) { |
| // build pinned while the member is retained, and drops it once the reaper |
| // proves the child gone. The serving build stays pinned throughout. |
| func TestARetainedShedRespawnKeepsItsBuildPinned(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _, ver := stagedActivator(t) |
| a.gens = true |
| hk := &stagedHostKill{stagedFake: b} |
diff --git a/internal/activator/worker_staged_test.go b/internal/activator/worker_staged_test.go| index 830629f63..9939d4243 100644 |
| --- a/internal/activator/worker_staged_test.go |
| +++ b/internal/activator/worker_staged_test.go |
| @@ -145,6 +145,7 @@ func TestExecBackendWorkerExecsTheStagedPathItResolved(t *testing.T) { |
| // Spec §7.9: a recycle execs the captured build even after root has |
| // repointed the link. |
| func TestExecBackendWorkerRecycleExecsTheCapturedBuildAfterARepoint(t *testing.T) { |
| + t.Parallel() |
| b := testExecBackend(t) |
| host := "bot.x.internal" |
| out1 := filepath.Join(t.TempDir(), "argv0-v1") |
| @@ -247,6 +248,7 @@ func TestExecBackendWorkerRunsTheBeforeExecHookFirst(t *testing.T) { |
| // Ruling R19 item 3: a before-exec hook that fails means no child, for a |
| // wake from the live link and for a recycle onto a captured build. |
| func TestExecBackendWorkerDoesNotExecWhenTheHookFails(t *testing.T) { |
| + t.Parallel() |
| b := testExecBackend(t) |
| host := "bot.x.internal" |
| out := filepath.Join(t.TempDir(), "argv0") |
diff --git a/internal/activator/workerswap_test.go b/internal/activator/workerswap_test.go| index d60463da3..d6a3a8332 100644 |
| --- a/internal/activator/workerswap_test.go |
| +++ b/internal/activator/workerswap_test.go |
| @@ -464,6 +464,7 @@ func TestReplaceWorkerWithoutOwnershipSpawnsNothing(t *testing.T) { |
| // A superseded attempt touches nothing. |
| func TestReplaceWorkerSupersededSpawnsNothing(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _, _ := stagedActivator(t) |
| if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil { |
| t.Fatal(err) |
| @@ -539,6 +540,7 @@ func TestReplaceWorkerAnIntentionalStopDuringTheObservationIsNoVerdict(t *testin |
| // manager wrote it, and a flag-off box showed "unknown", or a stale build |
| // after the flag was turned off. |
| func TestAWorkersServingIdentityIsPublishedWithTheFlagOff(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _, _ := stagedActivator(t) |
| var mu sync.Mutex |
| var got []string |
| @@ -568,6 +570,7 @@ func TestAWorkersServingIdentityIsPublishedWithTheFlagOff(t *testing.T) { |
| // replayed after the respawn published incarnation 2: it is dropped. An |
| // unconditional write left the older version on display indefinitely. |
| func TestAnOlderWorkerPublicationIsDropped(t *testing.T) { |
| + t.Parallel() |
| a, b, _, _, _ := stagedActivator(t) |
| var mu sync.Mutex |
| var got []string |
| @@ -600,6 +603,7 @@ func TestAnOlderWorkerPublicationIsDropped(t *testing.T) { |
| // so the row stops naming the build the previous child ran. Skipping empty |
| // versions left ServingVersion=v1 on display indefinitely. |
| func TestAnUnknownVersionClearsTheServingVersion(t *testing.T) { |
| + t.Parallel() |
| a, _, _, _, _ := stagedActivator(t) |
| var mu sync.Mutex |
| var got []string |
diff --git a/internal/provision/plan14_root_reads_test.go b/internal/provision/plan14_root_reads_test.go| index f99e6b6d8..217c65123 100644 |
| --- a/internal/provision/plan14_root_reads_test.go |
| +++ b/internal/provision/plan14_root_reads_test.go |
| @@ -1083,6 +1083,24 @@ func TestPlan14RootReadCensus(t *testing.T) { |
| t.Fatal(err) |
| } |
| seal = "940ed1bba8d911e0f57af83ecac253c77ba5e14dad590258e0ee03ec98ccb818" |
| + // Census overlay ci-speed-activator (amadan #291: CI speed), loaded last, |
| + // after moves-offline-precopy, under census-ruling-ci-speed-activator |
| + // (Paul, as census owner, 2026-10-21, accepted as proposed). Four |
| + // activator timings become package vars with the same values and |
| + // moveprobe gains a link-time grace override; only tests change them. |
| + // ExecBackend.waitRoleCgroupEmpty and |
| + // Service.startAdmitted run in the agent (user carlos), make no root |
| + // decision and are re-frozen at main's class and task with new digests; |
| + // their sites are unchanged. The internal/activator and cmd/moveprobe |
| + // envelopes are re-pinned. Two frozen declarations are added, so the seal |
| + // moves; moves-offline-precopy's is checked first. |
| + if errs := checkRootReadLegacy(inv, seal); len(errs) != 0 { |
| + t.Fatal(errs) |
| + } |
| + if err = loadRootReadInventory(filepath.Join(root, "internal/provision/testdata/plan14-root-reads-ci-speed-activator.tsv"), &inv, true); err != nil { |
| + t.Fatal(err) |
| + } |
| + seal = "3737ebbd0ee87811839624724e552e0af02f63d70baaa0516a024440d848fdae" |
| errs := append(compareRootReads(got, inv), checkRootReadLegacy(inv, seal)...) |
| errs = append(errs, checkRootReadDeclarations(got, inv)...) |
| if len(errs) > 0 { |
diff --git a/internal/provision/testdata/plan14-root-reads-ci-speed-activator.tsv b/internal/provision/testdata/plan14-root-reads-ci-speed-activator.tsv| new file mode 100644 |
| index 000000000..8ffba5a2d |
| --- /dev/null |
| +++ b/internal/provision/testdata/plan14-root-reads-ci-speed-activator.tsv |
| @@ -0,0 +1,11 @@ |
| +# Census overlay ci-speed-activator (amadan #291: CI speed; test speed-ups in internal/activator and cmd/moveprobe), loaded LAST, after moves-offline-precopy. RULED as census-ruling-ci-speed-activator (Paul, as census owner, 2026-10-21), accepted as proposed. |
| +# No root read, file read or record write is added, and no read site moves. Four activator timings become package vars with the same values (claimStartBackoff 1 s, roleEmptyBound = actStopTimeout, cmdAcceptGrace 1 s, heldPoll 250 ms), and cmd/moveprobe gains a link-time workerGraceOverride read through orphanGrace() (a build that does not set it keeps the 60 s workerGrace; moveprobe is a test probe binary). Only tests change these values. Production behaviour is identical. |
| +# ExecBackend.waitRoleCgroupEmpty: executing identity the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos); no root decision. Value-identical: its deadline reads roleEmptyBound (initialised to actStopTimeout) once into a local; the wait, its error text and its two Host reads are unchanged. Re-frozen at main's class and task (accepted-legacy, census-ruling-carlos-run) with its new digest; sites unchanged. |
| +# Service.startAdmitted: executing identity the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos); no root decision. Value-identical: the start backoff time.Second << attempt reads claimStartBackoff (1 s) << attempt, the same 1, 2, 4, 8 s waits. Re-frozen at main's class and task (accepted-legacy, J1,J4 route-and-input) with its new digest; sites and offsets unchanged. |
| +# Envelopes re-pinned: internal/activator (the two units above, plus cmdunit.go's cmdAcceptGrace and wakecheck.go's heldPoll, const to var with the same values; no read site) and cmd/moveprobe (workerGraceOverride and orphanGrace in runWorker; no read site). Two frozen declarations are added (the two re-frozen units), so the seal moves; moves-offline-precopy's is checked first. |
| +@legacy f3cb856ba885e532a2ce4fc2a21f1770075b0758009439df98dce85b05978021 internal/activator/backend_exec.go ExecBackend.waitRoleCgroupEmpty legacy accepted-legacy accepted-legacy;rev3=census-ruling-carlos-run;basis=the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos): body of the frozen waitSeedCgroupEmpty with the role as a parameter (seed or cmd); the agent waits on a unit it asked root to stop; no root decision;rev3=census-ruling-ci-speed-activator;identity=the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos);basis=value-identical: the deadline reads package var roleEmptyBound, initialised to actStopTimeout and changed only by tests, once into a local; the wait, its error text and its two Host reads are unchanged; no root decision registry:Host@302,registry:Host@347 882b526a398e97fa0d8fec05e918d5e3eee8585834f326d460f54fda3ac102f3 |
| +internal/activator/backend_exec.go ExecBackend.waitRoleCgroupEmpty legacy accepted-legacy accepted-legacy;rev3=census-ruling-carlos-run;basis=the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos): body of the frozen waitSeedCgroupEmpty with the role as a parameter (seed or cmd); the agent waits on a unit it asked root to stop; no root decision;rev3=census-ruling-ci-speed-activator;identity=the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos);basis=value-identical: the deadline reads package var roleEmptyBound, initialised to actStopTimeout and changed only by tests, once into a local; the wait, its error text and its two Host reads are unchanged; no root decision registry:Host@302,registry:Host@347 882b526a398e97fa0d8fec05e918d5e3eee8585834f326d460f54fda3ac102f3 |
| +@legacy 108ce5ccff5cfbb6be2dcb94ffc51f3413ef93d40fc5ce6dafb799b1baa304be internal/activator/claimwake.go Service.startAdmitted legacy accepted-legacy accepted-legacy;rev3=J1,J4;basis=route-and-input;rev3=census-ruling-ci-speed-activator;identity=the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos);basis=value-identical: the start backoff time.Second << attempt reads package var claimStartBackoff (1 s, changed only by tests) << attempt; the same 1, 2, 4, 8 s waits; read sites and offsets unchanged; no root decision file:Prepare@1038,file:Prepare@213,file:Read@196,file:Read@812,file:Read@8372,whole:Route@1055,whole:Route@5087,whole:Route@5163,whole:Route@5795,whole:Route@5858 af71d436b7e8129a7486b65e7aedfaf7b184bbf5bd95258ab4b3d6d8061bb6a5 |
| +internal/activator/claimwake.go Service.startAdmitted legacy accepted-legacy accepted-legacy;rev3=J1,J4;basis=route-and-input;rev3=census-ruling-ci-speed-activator;identity=the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos);basis=value-identical: the start backoff time.Second << attempt reads package var claimStartBackoff (1 s, changed only by tests) << attempt; the same 1, 2, 4, 8 s waits; read sites and offsets unchanged; no root decision file:Prepare@1038,file:Prepare@213,file:Read@196,file:Read@812,file:Read@8372,whole:Route@1055,whole:Route@5087,whole:Route@5163,whole:Route@5795,whole:Route@5858 af71d436b7e8129a7486b65e7aedfaf7b184bbf5bd95258ab4b3d6d8061bb6a5 |
| +@coverage cmd/moveprobe e8d78f2c183ff1de2be2ea6e13b5bda6a5ac9f11d8f3ca2b7666ea0f55062a17 |
| +@coverage internal/activator a7691aafa1284962aa4d4b70d7be40209197bb769d937dd7b1670928c77055f6 |