carlos / platform Public

Download this file

1	package activator
2
3	// Ported from lchat activator_test.go.
4	//
5	// Unit tests for the activator state machine (activator.go) with a fake
6	// backend — wake coalescing, idle→sleep, lease-lost stand-down, contention
7	// 503s. No AWS, no processes. The router-level tests use a small in-package
8	// test router that mimics the edge's dispatch (internal/edge) — deliberately
9	// NOT imported: the activator satisfies edge.Activator structurally, and the
10	// dependency must point edge→activator only.
11
12	import (
13	"context"
14	"database/sql"
15	"errors"
16	"fmt"
17	"io"
18	"net"
19	"net/http"
20	"net/http/httptest"
21	"net/http/httputil"
22	"os"
23	"os/exec"
24	"path/filepath"
25	"slices"
26	"strings"
27	"sync"
28	"sync/atomic"
29	"syscall"
30	"testing"
31	"time"
32
33	"golang.org/x/sys/unix"
34
35	"github.com/carlosframework/platform/internal/cgroup"
36	"github.com/carlosframework/platform/internal/edge"
37	"github.com/carlosframework/platform/internal/lease"
38	"github.com/carlosframework/platform/internal/provision"
39	"github.com/carlosframework/platform/internal/registry"
40	"github.com/carlosframework/platform/internal/seeds"
41
42	_ "modernc.org/sqlite"
43	)
44
45	// --- fakes -------------------------------------------------------------------
46
47	type fakeRep struct {
48	mu sync.Mutex
49	err error
50	closed bool
51	closes int
52	// b, when set, is bumped on Close. production's Rep.Close "does the
53	// final checkpoint/sync and releases the lease" (Rep's doc comment), so
54	// a successful Close is the fake's equivalent of the lease coming free
55	// once a wake has gotten past Replicate (before that point, the raw
56	// Lease.Release calls below count it instead: the two are mutually
57	// exclusive per wake, since Replicate spends the Lease).
58	b *fakeBackend
59	// onClose, when set, runs at the start of Close: the moment the lease is
60	// given back. Task 12 checks there that no generation still runs.
61	onClose func()
62	}
63
64	func (r *fakeRep) Err() error {
65	r.mu.Lock()
66	defer r.mu.Unlock()
67	return r.err
68	}
69	func (r *fakeRep) setErr(err error) {
70	r.mu.Lock()
71	defer r.mu.Unlock()
72	r.err = err
73	}
74	func (r *fakeRep) Close(context.Context) error {
75	r.mu.Lock()
76	hook := r.onClose
77	r.mu.Unlock()
78	if hook != nil {
79	hook()
80	}
81	r.mu.Lock()
82	r.closed = true
83	r.closes++ // Close IS the lease release: exactly once, on every path
84	b := r.b
85	r.mu.Unlock()
86	if b != nil {
87	b.mu.Lock()
88	b.releases++
89	b.mu.Unlock()
90	}
91	return nil
92	}
93
94	func (r *fakeRep) setOnClose(fn func()) {
95	r.mu.Lock()
96	r.onClose = fn
97	r.mu.Unlock()
98	}
99
100	func (r *fakeRep) closeCount() int {
101	r.mu.Lock()
102	defer r.mu.Unlock()
103	return r.closes
104	}
105	func (r *fakeRep) isClosed() bool {
106	r.mu.Lock()
107	defer r.mu.Unlock()
108	return r.closed
109	}
110
111	type fakeProc struct {
112	// launchEnv is what LaunchEnv answers (merge ruling M3); nil answers
113	// an empty environment, as a generation launched with no files has.
114	launchEnv map[string]string
115	mu sync.Mutex
116	stopped bool
117	dead bool
118	// stopGate, when non-nil, blocks Stop() until closed — holds an entry in
119	// the stopping state so a test can poke at it.
120	stopGate chan struct{}
121	// crashed marks a NON-ZERO exit — the fact that tells a hibernating
122	// worker's crash from its completion.
123	crashed bool
124	// stuck models a Proc whose termination cannot be proven within Stop's
125	// bounded ctx (design §6.3, unitProc's ErrStillRunning): while stuck and
126	// not yet dead, Stop reports ErrStillRunning instead of succeeding; exit()
127	// clears it. stopAttempts counts every Stop() call, proven or not, so a
128	// test can see the sweeper retrying a stop it hasn't been able to finish.
129	stuck bool
130	stopAttempts int
131	// stopEnters counts Stop calls ENTERED, before the gate: a test waits on
132	// it to know a gated Stop is in flight.
133	stopEnters int
134	// label + rec feed the backend's order log ("instance" / "sidecar"), so
135	// the co-spawn/co-stop tests can assert interleavings.
136	label string
137	rec *fakeBackend
138	// polls counts Exited() calls; exitAt, when > 0, makes the process exit
139	// on that call: a death a test places inside an observation (a settle
140	// window) by the observation's own polling, with no sleep.
141	polls int
142	exitAt int
143	// onStopEnter, when set, runs as Stop is entered, before the gate: a
144	// test records what was already true when the stop began.
145	onStopEnter func()
146	}
147
148	func (p *fakeProc) LaunchEnv() map[string]string {
149	if p.launchEnv == nil {
150	return map[string]string{}
151	}
152	return p.launchEnv
153	}
154
155	func (p *fakeProc) Stop(context.Context) error {
156	p.mu.Lock()
157	p.stopEnters++
158	gate, enter := p.stopGate, p.onStopEnter
159	p.mu.Unlock()
160	if enter != nil {
161	enter()
162	}
163	if gate != nil {
164	<-gate
165	}
166	p.mu.Lock()
167	p.stopAttempts++
168	if p.stuck && !p.dead {
169	p.mu.Unlock()
170	return ErrStillRunning
171	}
172	p.stopped = true
173	// A stopped process HAS exited: modelling that is what lets a test see the
174	// window a converge respawn opens — old child down, new one not yet
175	// installed — which the sweeper must not mistake for a crash.
176	p.dead = true
177	p.mu.Unlock()
178	if p.rec != nil {
179	p.rec.record("stop:" + p.label)
180	}
181	return nil
182	}
183
184	// setStopGate installs (or clears) the gate Stop blocks on, under the proc's
185	// own lock — the sweeper may be reading it concurrently.
186	func (p *fakeProc) setStopGate(gate chan struct{}) {
187	p.mu.Lock()
188	p.stopGate = gate
189	p.mu.Unlock()
190	}
191	func (p *fakeProc) Exited() bool {
192	p.mu.Lock()
193	defer p.mu.Unlock()
194	p.polls++
195	if p.exitAt > 0 && p.polls >= p.exitAt {
196	p.dead = true
197	}
198	return p.dead
199	}
200
201	// exitOnPoll makes the process exit on its n-th Exited() call from now.
202	func (p *fakeProc) exitOnPoll(n int) {
203	p.mu.Lock()
204	p.exitAt = p.polls + n
205	p.mu.Unlock()
206	}
207
208	// pollCount is how many times Exited() has been asked.
209	func (p *fakeProc) pollCount() int {
210	p.mu.Lock()
211	defer p.mu.Unlock()
212	return p.polls
213	}
214
215	// Crashed models the exec child's own answer (backend_exec.go's childProc):
216	// only a definitely non-zero exit is a crash. kill() therefore models a CLEAN
217	// exit — a bot that finished its work — and crash() models a child that fell
218	// over.
219	func (p *fakeProc) Crashed() bool {
220	p.mu.Lock()
221	defer p.mu.Unlock()
222	return p.dead && p.crashed
223	}
224
225	// crash models the child dying with a NON-ZERO status.
226	func (p *fakeProc) crash() {
227	p.mu.Lock()
228	p.dead, p.crashed = true, true
229	p.mu.Unlock()
230	}
231	func (p *fakeProc) isStopped() bool {
232	p.mu.Lock()
233	defer p.mu.Unlock()
234	return p.stopped
235	}
236	func (p *fakeProc) stopEntered() int {
237	p.mu.Lock()
238	defer p.mu.Unlock()
239	return p.stopEnters
240	}
241	func (p *fakeProc) stopCalls() int {
242	p.mu.Lock()
243	defer p.mu.Unlock()
244	return p.stopAttempts
245	}
246
247	// exit marks the process as having died on its own (a crash, or, for a
248	// stuck proc, the delayed termination Stop couldn't prove at the time).
249	func (p *fakeProc) exit() {
250	p.mu.Lock()
251	p.dead = true
252	p.mu.Unlock()
253	}
254
255	type fakeLease struct {
256	rep *fakeRep
257	b *fakeBackend // records the sync interval the wake threads through
258	// owed: a Release failed and RetryRelease has not yet landed. Under b.mu.
259	owed bool
260	}
261
262	func (l *fakeLease) Replicate(_ context.Context, _ string, syncInterval time.Duration) (Rep, error) {
263	if l.b != nil {
264	l.b.mu.Lock()
265	l.b.syncIntervals = append(l.b.syncIntervals, syncInterval)
266	l.b.mu.Unlock()
267	}
268	return l.rep, nil
269	}
270
271	// Err is the held lease's liveness (leaseHealth): the renewal the wake's
272	// replica will inherit. A test loses the lease before Replicate by setting
273	// the error on the rep this lease hands out (b.lastRep().setErr).
274	func (l *fakeLease) Err() error {
275	if l.rep == nil {
276	return nil
277	}
278	return l.rep.Err()
279	}
280
281	// Release counts on the backend: every doWake bail-out must give the lease
282	// back exactly once, and the only way to assert that from outside is to count
283	// the calls the wake made. It also records its context's error: the real
284	// release is an S3 DeleteObject(ctx), which fails at once on a dead context
285	// and spends the handle anyway.
286	func (l *fakeLease) Release(ctx context.Context) error {
287	if l.b != nil {
288	l.b.mu.Lock()
289	defer l.b.mu.Unlock()
290	l.b.releaseCtxErrs = append(l.b.releaseCtxErrs, ctx.Err())
291	if len(l.b.releaseErrs) > 0 {
292	err := l.b.releaseErrs[0]
293	l.b.releaseErrs = l.b.releaseErrs[1:]
294	l.owed = true
295	return err
296	}
297	l.b.releases++
298	l.b.queuedAtRelease = append(l.b.queuedAtRelease, l.b.queuedJobs)
299	}
300	return nil
301	}
302
303	// RetryRelease is lease.Held's: it discharges what a failed Release left
304	// owed, and counts as the release once it lands.
305	func (l *fakeLease) RetryRelease(context.Context) error {
306	if l.b == nil {
307	return nil
308	}
309	l.b.mu.Lock()
310	defer l.b.mu.Unlock()
311	l.b.retries++
312	if !l.owed {
313	return nil
314	}
315	if len(l.b.retryErrs) > 0 {
316	err := l.b.retryErrs[0]
317	l.b.retryErrs = l.b.retryErrs[1:]
318	if l.b.leaseLapsed {
319	l.owed = false
320	return fmt.Errorf("%w: %w", lease.ErrReleaseLapsed, err)
321	}
322	return err
323	}
324	l.owed = false
325	l.b.releases++
326	l.b.queuedAtRelease = append(l.b.queuedAtRelease, l.b.queuedJobs)
327	return nil
328	}
329
330	// LegacyRunGone is ExecBackend's: the legacy run unit's stop, which
331	// replaces a queued start job, and its proof.
332	func (b *fakeBackend) LegacyRunGone(ctx context.Context, _ registry.Route) error {
333	b.mu.Lock()
334	b.legacyStops++
335	block := b.legacyBlock
336	b.mu.Unlock()
337	if block {
338	<-ctx.Done()
339	b.mu.Lock()
340	b.legacyEnded++
341	b.mu.Unlock()
342	return ctx.Err()
343	}
344	b.mu.Lock()
345	defer b.mu.Unlock()
346	if b.legacyStopErr != nil {
347	return b.legacyStopErr
348	}
349	b.queuedJobs = 0
350	return nil
351	}
352
353	type fakeBackend struct {
354	mu sync.Mutex
355	acquires int
356	admits int // AdmitConfig calls
357	restores int
358	starts int
359	seeds int
360	releases int // lease releases — the failure paths' assertion
361	acquireErr error
362	// releaseCtxErrs is ctx.Err() at each fakeLease.Release, in order.
363	releaseCtxErrs []error
364	// admitErr is what AdmitConfig answers: the config gate a retained-lease
365	// respawn must ask again (amadan #213).
366	admitErr error
367	// admitHook, when set, runs at the start of AdmitConfig, off b.mu: a
368	// test's window inside a respawn's preparation, after its first
369	// ownership check and before its exec.
370	admitHook func()
371	// restoreFresh is what Restore reports: a first boot (local DB absent AND
372	// no replica). False by default, so every pre-seed test doubles as the
373	// "a restored instance is never seeded" case.
374	restoreFresh bool
375	restoreErr error
376	seedErr error
377	seedRoutes []registry.Route
378	startErr error
379	startGate chan struct{} // when non-nil, Start() blocks until closed
380	// startQueued, when non-nil, blocks Start() until it is closed or the
381	// wake's context ends. It models a nested legacy start whose systemctl
382	// start job systemd has queued behind an edge restart (#292). The
383	// instance-control client gives up when its context does, but the
384	// queued job SURVIVES that: a cancelled start counts one in queuedJobs
385	// and answers as ExecBackend does (no process, legacyStartUncertain).
386	// Only LegacyRunGone, the unit's stop, clears it.
387	startQueued chan struct{}
388	queuedJobs int
389	// legacyStops counts LegacyRunGone calls; legacyStopErr is what they
390	// answer (an unproven stop) while set.
391	legacyStops int
392	legacyStopErr error
393	// legacyBlock holds every LegacyRunGone until its context ends (a
394	// stop that never proves); legacyEnded counts those that returned.
395	legacyBlock bool
396	legacyEnded int
397	// startLegacyFail fails every Start as a legacy run unit's start root
398	// may still carry out, at once, outside any shutdown.
399	startLegacyFail bool
400	// leaseLapsed: the owed records have passed their expiry, so a retry
401	// that fails again answers lease.ErrReleaseLapsed, as lease.Held does.
402	leaseLapsed bool
403	// queuedAtRelease is queuedJobs at each lease release that landed: a
404	// release over a still-queued job is the single-writer violation.
405	queuedAtRelease []int
406	// releaseErrs fails the next Release calls, in order (a store that
407	// refuses or times out the DELETE); retryErrs does the same for
408	// RetryRelease. retries counts RetryRelease calls.
409	releaseErrs []error
410	retryErrs []error
411	retries int
412	reps []*fakeRep
413	procs []*fakeProc
414	// syncIntervals records, per wake, the replication-class tick doWake
415	// threads into Replicate() — the ladder's seam assertion.
416	syncIntervals []time.Duration
417	dueVal int64 // what the instance reports at the sleep-time next-due probe
418	dueErr error // when non-nil, the probe fails (dead socket, missing token, …)
419	// dueProbeCount counts NextDue calls. A worker has no socket to answer on,
420	// so the assertion its sleep needs is "the probe never ran at all", not
421	// "the probe returned nothing".
422	dueProbeCount int
423
424	// execVerFn models the live/<host> target the child execs at spawn — Start()
425	// returns it as the spawnVersion, exactly as ExecBackend reads the symlink
426	// just before Start(). Nil → "" (unknown), which disables convergence, so
427	// tests that don't care are unaffected. Point it at the same source as the
428	// sweep's liveVer seam to model a symlink faithfully (both read the current
429	// target).
430	execVerFn func() string
431
432	// Sidecar co-spawn. sidecarStaged is HasSidecar's answer (false by
433	// default, so every pre-co-spawn test doubles as the "host without a
434	// sidecar spawns none" case); sidecarErr injects spawn failures.
435	sidecarStaged bool
436	sidecarErr error
437	sidecarStarts int
438	sidecars []*fakeProc
439	// Worker spawn (StartWorker) — mirrors the sidecar fields above for the
440	// same reason: workerStaged... no, a worker has no HasSidecar-style probe
441	// (Task 6's territory); workerErr injects spawn failures, workerEnvs
442	// records what env each call was handed so a caller-side test can assert
443	// the per-wake token actually reached the backend.
444	workerErr error
445	workers []*fakeProc
446	workerEnvs []map[string]string
447	workerStartN int
448	// workerDeadOnStart hands back a child that is ALREADY dead — the sidecar
449	// that crashes on boot, which the settle window exists to catch.
450	workerDeadOnStart bool
451	workerSpawnVer string // what StartWorker reports as the build it execed
452	// workerStartDelay holds StartWorker open for a beat, so a test can land a
453	// sweep INSIDE a converge respawn's window (old child stopped, new one not
454	// yet installed) rather than hoping to hit it.
455	workerStartDelay time.Duration
456	workerEnterN int // StartWorker calls ENTERED (before the delay)
457	// workerStopGate, when set, is installed on every worker child spawned
458	// from then on: its Stop blocks until the gate closes, so a test can hold
459	// a child mid-drain and watch what happens around it.
460	workerStopGate chan struct{}
461	// workerStuck makes every worker child spawned from then on a stuck
462	// fakeProc: its Stop answers ErrStillRunning until exit().
463	workerStuck bool
464	// workerStartHook, when set, runs INSIDE StartWorker (off b.mu), so a test
465	// can model what the child itself does while its wake is still resolving —
466	// notably posting its next due time over the agent socket during the
467	// settle window.
468	workerStartHook func(rt registry.Route)
469	// sidecarGate, when non-nil, blocks StartSidecar() until closed. Models
470	// a stalled restart call so a test can assert it doesn't hold a.mu.
471	sidecarGate chan struct{}
472	// sidecarStartEntered, when non-nil, is closed the first time
473	// StartSidecar is entered (before it waits on sidecarGate). Lets a test
474	// wait for a restart's call to actually begin rather than sleeping a
475	// guessed duration.
476	sidecarStartEntered chan struct{}
477	sidecarStartEnteredOnce sync.Once
478	// sidecarLabel, when set, labels the sidecars StartSidecar returns from
479	// then on ("sidecar" otherwise), so the order log can tell them apart.
480	sidecarLabel string
481	// sidecarStopGate and sidecarStuck, when set, are given to the sidecars
482	// StartSidecar returns from then on (fakeProc's stopGate and stuck): a
483	// returned sidecar whose stop blocks, or cannot prove its exit.
484	sidecarStopGate chan struct{}
485	sidecarStuck bool
486	// order interleaves backend calls and proc stops (start:instance,
487	// start:sidecar, nextDue, stop:sidecar, stop:instance) for the ordering
488	// assertions.
489	order []string
490
491	// procStuck makes every Start() return a fakeProc whose Stop reports
492	// ErrStillRunning until the test calls its exit() (design §6.3).
493	procStuck bool
494	// startedRoutes records the registry.Route Start() actually received,
495	// per call. doWake's route-refresh assertion (design §6.8) needs to see
496	// what the backend was actually handed, not what the caller passed in.
497	startedRoutes []registry.Route
498	// layoutFn, when set, is what Layout reports: a test's way to exercise
499	// doWake's pre-Acquire fence check without a real ExecBackend. Nil means
500	// LayoutFlat, "": every test that never sets this field gets the
501	// behavior it always had, since a flat layout never fences a wake.
502	layoutFn func(rt registry.Route) (Layout, string)
503
504	// ident is the identity capability (settings spec §4.2, Task 5c),
505	// opted into per test (wakeidentity_test.go). Guarded by mu.
506	ident identityFake
507	}
508
509	// Layout is fakeBackend's Classifier implementation, always present so
510	// doWake's type assertion succeeds; layoutFn (above) decides what it says.
511	func (b *fakeBackend) Layout(rt registry.Route) (Layout, string) {
512	if b.layoutFn == nil {
513	return LayoutFlat, ""
514	}
515	return b.layoutFn(rt)
516	}
517
518	func (b *fakeBackend) record(ev string) {
519	b.mu.Lock()
520	b.order = append(b.order, ev)
521	b.mu.Unlock()
522	}
523
524	func (b *fakeBackend) events() []string {
525	b.mu.Lock()
526	defer b.mu.Unlock()
527	return append([]string(nil), b.order...)
528	}
529
530	func (b *fakeBackend) Acquire(context.Context, registry.Route) (Lease, error) {
531	b.mu.Lock()
532	b.acquires++
533	err := b.acquireErr
534	rep := &fakeRep{b: b}
535	b.reps = append(b.reps, rep)
536	b.mu.Unlock()
537	if err != nil {
538	return nil, err
539	}
540	return &fakeLease{rep: rep, b: b}, nil
541	}
542
543	func (b *fakeBackend) AdmitConfig(context.Context, registry.Route) error {
544	b.mu.Lock()
545	hook := b.admitHook
546	b.mu.Unlock()
547	if hook != nil {
548	hook() // a respawn's preparation, held open by the test
549	}
550	b.mu.Lock()
551	defer b.mu.Unlock()
552	b.admits++
553	if b.admitErr != nil {
554	return &errConfigRefused{err: b.admitErr}
555	}
556	return nil
557	}
558
559	func (b *fakeBackend) Restore(context.Context, registry.Route) (bool, error) {
560	b.mu.Lock()
561	defer b.mu.Unlock()
562	b.restores++
563	b.order = append(b.order, "restore")
564	return b.restoreFresh, b.restoreErr
565	}
566
567	func (b *fakeBackend) Seed(_ context.Context, rt registry.Route) error {
568	b.mu.Lock()
569	defer b.mu.Unlock()
570	b.seeds++
571	b.order = append(b.order, "seed")
572	b.seedRoutes = append(b.seedRoutes, rt)
573	return b.seedErr
574	}
575
576	func (b *fakeBackend) Start(ctx context.Context, rt registry.Route) (Proc, string, error) {
577	b.mu.Lock()
578	b.starts++
579	b.order = append(b.order, "start:instance")
580	gate := b.startGate
581	queued := b.startQueued
582	fn := b.execVerFn
583	startErr := b.startErr
584	proc := &fakeProc{label: "instance", rec: b, stuck: b.procStuck}
585	b.procs = append(b.procs, proc)
586	b.startedRoutes = append(b.startedRoutes, rt)
587	b.mu.Unlock()
588	if startErr != nil {
589	return nil, "", startErr
590	}
591	b.mu.Lock()
592	legacyFail := b.startLegacyFail
593	b.mu.Unlock()
594	if legacyFail {
595	return nil, "", markLegacyStartUncertain(markStartUncertain(fmt.Errorf("%s: start unit: root did not answer", rt.Host)))
596	}
597	if gate != nil {
598	<-gate
599	}
600	if queued != nil {
601	select {
602	case <-queued:
603	case <-ctx.Done():
604	b.mu.Lock()
605	b.queuedJobs++
606	b.mu.Unlock()
607	return nil, "", markLegacyStartUncertain(markStartUncertain(fmt.Errorf("%s: start unit: %w", rt.Host, ctx.Err())))
608	}
609	}
610	ver := ""
611	if fn != nil {
612	ver = fn()
613	}
614	return proc, ver, nil
615	}
616
617	func (b *fakeBackend) NextDue(context.Context, registry.Route) (int64, error) {
618	b.mu.Lock()
619	defer b.mu.Unlock()
620	b.dueProbeCount++
621	b.order = append(b.order, "nextDue")
622	if b.dueErr != nil {
623	return 0, b.dueErr
624	}
625	return b.dueVal, nil
626	}
627
628	func (b *fakeBackend) HasSidecar(registry.Route) bool {
629	b.mu.Lock()
630	defer b.mu.Unlock()
631	return b.sidecarStaged
632	}
633
634	func (b *fakeBackend) StartSidecar(context.Context, registry.Route) (Proc, error) {
635	b.mu.Lock()
636	gate := b.sidecarGate
637	entered := b.sidecarStartEntered
638	b.mu.Unlock()
639	if entered != nil {
640	b.sidecarStartEnteredOnce.Do(func() { close(entered) })
641	}
642	if gate != nil {
643	<-gate
644	}
645	b.mu.Lock()
646	defer b.mu.Unlock()
647	b.sidecarStarts++
648	b.order = append(b.order, "start:sidecar")
649	if b.sidecarErr != nil {
650	return nil, b.sidecarErr
651	}
652	label := "sidecar"
653	if b.sidecarLabel != "" {
654	label = b.sidecarLabel
655	}
656	p := &fakeProc{label: label, rec: b, stopGate: b.sidecarStopGate, stuck: b.sidecarStuck}
657	b.sidecars = append(b.sidecars, p)
658	return p, nil
659	}
660
661	func (b *fakeBackend) sidecarStartCount() int {
662	b.mu.Lock()
663	defer b.mu.Unlock()
664	return b.sidecarStarts
665	}
666
667	func (b *fakeBackend) lastSidecar() *fakeProc {
668	b.mu.Lock()
669	defer b.mu.Unlock()
670	return b.sidecars[len(b.sidecars)-1]
671	}
672
673	func (b *fakeBackend) setSidecarErr(err error) {
674	b.mu.Lock()
675	defer b.mu.Unlock()
676	b.sidecarErr = err
677	}
678
679	func (b *fakeBackend) StartWorker(ctx context.Context, rt registry.Route, env map[string]string) (Proc, string, error) {
680	b.mu.Lock()
681	hook := b.workerStartHook
682	delay := b.workerStartDelay
683	b.workerEnterN++ // counted BEFORE the delay: "the exec has begun"
684	b.mu.Unlock()
685	if delay > 0 {
686	// A slow exec — ExecBackend materializes the build from the bucket
687	// when live/<host> dangles — and, like the real one, abandonable: a
688	// cancelled context produces NO child, which is what makes a shutdown
689	// that cancels distinguishable from one that merely outran the spawn.
690	select {
691	case <-time.After(delay):
692	case <-ctx.Done():
693	return nil, "", ctx.Err()
694	}
695	}
696	if hook != nil {
697	hook(rt) // the child, acting while its wake is still in flight
698	}
699	b.mu.Lock()
700	defer b.mu.Unlock()
701	b.workerStartN++
702	b.order = append(b.order, "startWorker:"+rt.Host)
703	b.workerEnvs = append(b.workerEnvs, env)
704	if b.workerErr != nil {
705	return nil, "", b.workerErr
706	}
707	p := &fakeProc{label: "worker", rec: b, dead: b.workerDeadOnStart, stopGate: b.workerStopGate, stuck: b.workerStuck}
708	b.workers = append(b.workers, p)
709	// Same symlink model as Start: execVerFn, when wired, is the live/<host>
710	// target this exec would have read — so a convergence test can move the
711	// symlink and have the respawned child report the new build.
712	ver := b.workerSpawnVer
713	if b.execVerFn != nil {
714	ver = b.execVerFn()
715	}
716	return p, ver, nil
717	}
718
719	// lastWorker is the most recently spawned worker child.
720	func (b *fakeBackend) lastWorker() *fakeProc {
721	b.mu.Lock()
722	defer b.mu.Unlock()
723	if len(b.workers) == 0 {
724	return nil
725	}
726	return b.workers[len(b.workers)-1]
727	}
728
729	// kill models the child dying on its own (a crash), which the sweep notices
730	// through Exited().
731	func (p *fakeProc) kill() {
732	p.mu.Lock()
733	p.dead = true
734	p.mu.Unlock()
735	}
736
737	// workerStarts counts StartWorker calls; startCount counts Start (the socket
738	// path, with its health probe) calls. A worker must only ever appear in the
739	// first.
740	func (b *fakeBackend) workerStarts() int {
741	b.mu.Lock()
742	defer b.mu.Unlock()
743	return b.workerStartN
744	}
745
746	// workerEnters counts spawns that have BEGUN — the only way to know a test is
747	// inside a slow exec rather than merely near one.
748	func (b *fakeBackend) workerEnters() int {
749	b.mu.Lock()
750	defer b.mu.Unlock()
751	return b.workerEnterN
752	}
753
754	func (b *fakeBackend) startCount() int {
755	b.mu.Lock()
756	defer b.mu.Unlock()
757	return b.starts
758	}
759
760	func (b *fakeBackend) dueProbes() int {
761	b.mu.Lock()
762	defer b.mu.Unlock()
763	return b.dueProbeCount
764	}
765
766	func (b *fakeBackend) lastWorkerEnv() map[string]string {
767	b.mu.Lock()
768	defer b.mu.Unlock()
769	if len(b.workerEnvs) == 0 {
770	return nil
771	}
772	return b.workerEnvs[len(b.workerEnvs)-1]
773	}
774
775	func (b *fakeBackend) setDue(v int64) {
776	b.mu.Lock()
777	defer b.mu.Unlock()
778	b.dueVal = v
779	}
780
781	func (b *fakeBackend) counts() (acquires, restores, starts int) {
782	b.mu.Lock()
783	defer b.mu.Unlock()
784	return b.acquires, b.restores, b.starts
785	}
786
787	func (b *fakeBackend) seedCount() int {
788	b.mu.Lock()
789	defer b.mu.Unlock()
790	return b.seeds
791	}
792
793	func (b *fakeBackend) releaseCount() int {
794	b.mu.Lock()
795	defer b.mu.Unlock()
796	return b.releases
797	}
798
799	// releaseCtxErrors is ctx.Err() as each fakeLease.Release saw it.
800	func (b *fakeBackend) releaseCtxErrors() []error {
801	b.mu.Lock()
802	defer b.mu.Unlock()
803	return slices.Clone(b.releaseCtxErrs)
804	}
805
806	func (b *fakeBackend) lastProc() *fakeProc {
807	b.mu.Lock()
808	defer b.mu.Unlock()
809	return b.procs[len(b.procs)-1]
810	}
811
812	func (b *fakeBackend) lastStartedRoute() registry.Route {
813	b.mu.Lock()
814	defer b.mu.Unlock()
815	return b.startedRoutes[len(b.startedRoutes)-1]
816	}
817
818	func (b *fakeBackend) lastRep() *fakeRep {
819	b.mu.Lock()
820	defer b.mu.Unlock()
821	return b.reps[len(b.reps)-1]
822	}
823
824	// fakeClock is the activator's `now` seam.
825	type fakeClock struct {
826	mu sync.Mutex
827	t time.Time
828	}
829
830	func (c *fakeClock) now() time.Time {
831	c.mu.Lock()
832	defer c.mu.Unlock()
833	return c.t
834	}
835	func (c *fakeClock) advance(d time.Duration) {
836	c.mu.Lock()
837	defer c.mu.Unlock()
838	c.t = c.t.Add(d)
839	}
840
841	// The registry IS the DueStore in production — assert the wiring compiles.
842	var _ DueStore = (*registry.Registry)(nil)
843
844	// testActivator wires the fakes to a REAL (temp-dir) registry as the due store,
845	// so the due-time tests exercise the actual routes.next_due SQL, not a fake of it.
846	func testActivator(t *testing.T, idle time.Duration) (*Service, *fakeBackend, *fakeClock, *registry.Registry) {
847	t.Helper()
848	b := &fakeBackend{}
849	clk := &fakeClock{t: time.Unix(1_700_000_000, 0)}
850	reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db"))
851	if err != nil {
852	t.Fatalf("open registry: %v", err)
853	}
854	a := New(Config{Bucket: "test-bucket", Idle: idle}, b, reg)
855	a.now = clk.now
856	return a, b, clk, reg
857	}
858
859	func testRoute(host string) registry.Route {
860	// Hibernate is set because these routes model activator-managed hosts —
861	// the per-host gate (the due sweep's flag filter, ServeHTTP's gate) only
862	// acts on flagged instances.
863	return registry.Route{Host: host, Kind: "instance", Socket: "/tmp/nope.sock",
864	DBPath: filepath.Join("/tmp", host+".db"), Hibernate: true}
865	}
866
867	// testWorkerRoute is a standalone sidecar's row: no socket (nothing can dial
868	// it), an argv of its own, and the hibernate flag that makes the sweeper's
869	// idle clock apply. Status "ready" mirrors the due-sweep's gate.
870	func testWorkerRoute(host string) registry.Route {
871	return registry.Route{Host: host, Kind: "sidecar", Args: `["sidecar","run"]`,
872	DBPath: filepath.Join("/tmp", host+".db"), Hibernate: true, Status: "ready"}
873	}
874
875	// contains: membership in the backend's order log.
876	func contains(events []string, ev string) bool { return slices.Contains(events, ev) }
877
878	// newTestServiceAwake builds a Service against the fake backend and drives
879	// host to actAwake via a real beginRequest/release round trip — the same
880	// wake arrange block used throughout this file (e.g.
881	// TestActivatorShutdownStopsChildren, TestActivatorConvergesOnSymlinkRepoint).
882	func newTestServiceAwake(t *testing.T, host string) *Service {
883	t.Helper()
884	a, _, _, _ := testActivator(t, time.Hour)
885	release, err := a.beginRequest(context.Background(), testRoute(host))
886	if err != nil {
887	t.Fatalf("beginRequest %s: %v", host, err)
888	}
889	release()
890	return a
891	}
892
893	// arrangeAwake drives rt's host to actAwake on an already-built Service via a
894	// real beginRequest/release round trip — the same arrange block
895	// newTestServiceAwake uses, but against a caller-supplied Service/route
896	// instead of building its own.
897	func arrangeAwake(t *testing.T, a *Service, rt registry.Route) {
898	t.Helper()
899	release, err := a.beginRequest(context.Background(), rt)
900	if err != nil {
901	t.Fatalf("beginRequest %s: %v", rt.Host, err)
902	}
903	release()
904	}
905
906	// okHandler is a proxy stand-in that writes 200 OK — for tests asserting
907	// only that a request reached the handler, not what a real instance would
908	// return.
909	func okHandler() http.Handler {
910	return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
911	w.WriteHeader(http.StatusOK)
912	})
913	}
914
915	// armTimer puts one armed schedule row on host, due at next.
916	func armTimer(t *testing.T, reg *registry.Registry, host string, next time.Time) {
917	t.Helper()
918	if err := reg.UpsertTimer(registry.Timer{
919	Host: host, Name: "sweep", Kind: "schedule", Spec: "every 5m",
920	Path: "/jobs/sweep", NextAt: next.Unix(), DueAt: next.Unix(),
921	}); err != nil {
922	t.Fatalf("UpsertTimer %s: %v", host, err)
923	}
924	}
925
926	// waitFor polls until a condition holds — the stop path runs on its own
927	// goroutine, so tests observe its completion rather than assuming it.
928	func waitFor(t *testing.T, what string, cond func() bool) {
929	t.Helper()
930	deadline := time.Now().Add(5 * time.Second)
931	for time.Now().Before(deadline) {
932	if cond() {
933	return
934	}
935	time.Sleep(20 * time.Millisecond)
936	}
937	t.Fatalf("timed out waiting for %s", what)
938	}
939
940	// sidecarSettled reports whether host has no sidecar restart in flight. The
941	// call runs off a.mu (design §6.7), so a test driving several restarts back
942	// to back (each expecting the previous one's count and install to have
943	// already landed) waits on this rather than racing sweepOnce's own, now much
944	// shorter, return.
945	func (a *Service) sidecarSettled(host string) bool {
946	a.mu.Lock()
947	defer a.mu.Unlock()
948	e := a.entries[host]
949	return e == nil || !e.sidecarStarting
950	}
951
952	func (a *Service) entryCount() int {
953	a.mu.Lock()
954	defer a.mu.Unlock()
955	return len(a.entries)
956	}
957
958	// --- the test router ---------------------------------------------------------
959	//
960	// A minimal stand-in for internal/edge's dispatch: routes with a socket get a
961	// unix reverse proxy; hibernate-flagged instance routes go through the
962	// activator's ServeHTTP hook first (the same shape as edge.Edge.ServeHTTP);
963	// reload() re-lists the registry and calls RetireExcept with the managed set,
964	// exactly as edge.Reload does.
965
966	type testUpstream struct {
967	proxy http.Handler
968	route registry.Route
969	hib bool
970	}
971
972	type testRouter struct {
973	reg *registry.Registry
974	act *Service
975	mu sync.Mutex
976	up map[string]testUpstream
977	}
978
979	func newTestRouter(reg *registry.Registry, act *Service) (*testRouter, error) {
980	rt := &testRouter{reg: reg, act: act}
981	if err := rt.reload(); err != nil {
982	return nil, err
983	}
984	return rt, nil
985	}
986
987	func (rt *testRouter) reload() error {
988	routes, err := rt.reg.List()
989	if err != nil {
990	return err
991	}
992	m := make(map[string]testUpstream)
993	managed := make(map[string]bool)
994	for _, t := range routes {
995	hib := rt.act != nil && t.Kind == "instance" && t.Hibernate
996	if hib {
997	managed[t.Host] = true
998	}
999	// Socket wins over Addr, mirroring edge.Reload's upstream choice.
1000	proxy := testUnixProxy(t.Socket)
1001	if t.Socket == "" && t.Addr != "" {
1002	proxy = testTCPProxy(t.Addr)
1003	}
1004	m[t.Host] = testUpstream{proxy: proxy, route: t, hib: hib}
1005	}
1006	rt.mu.Lock()
1007	rt.up = m
1008	rt.mu.Unlock()
1009	if rt.act != nil {
1010	rt.act.RetireExcept(managed)
1011	}
1012	return nil
1013	}
1014
1015	func (rt *testRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
1016	host := r.Host
1017	if h, _, err := net.SplitHostPort(host); err == nil {
1018	host = h
1019	}
1020	rt.mu.Lock()
1021	up, ok := rt.up[host]
1022	rt.mu.Unlock()
1023	if !ok {
1024	http.Error(w, "unknown host", http.StatusNotFound)
1025	return
1026	}
1027	if up.hib && rt.act.ServeHTTP(w, r, up.route, up.proxy) {
1028	return
1029	}
1030	up.proxy.ServeHTTP(w, r)
1031	}
1032
1033	func testUnixProxy(socket string) http.Handler {
1034	dialer := &net.Dialer{}
1035	return &httputil.ReverseProxy{
1036	Rewrite: func(pr *httputil.ProxyRequest) {
1037	pr.Out.URL.Scheme = "http"
1038	pr.Out.URL.Host = "carlos" // placeholder; the unix dialer ignores it
1039	pr.Out.Host = pr.In.Host
1040	},
1041	Transport: &http.Transport{
1042	DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
1043	return dialer.DialContext(ctx, "unix", socket)
1044	},
1045	},
1046	}
1047	}
1048
1049	// testTCPProxy mirrors edge's tcpProxy: the upstream shape for whole-app
1050	// tenants on a TCP Addr (the unix-socket fallback).
1051	func testTCPProxy(addr string) http.Handler {
1052	return &httputil.ReverseProxy{
1053	Rewrite: func(pr *httputil.ProxyRequest) {
1054	pr.Out.URL.Scheme = "http"
1055	pr.Out.URL.Host = addr
1056	pr.Out.Host = pr.In.Host
1057	},
1058	}
1059	}
1060
1061	// proxyGet sends GET base+"/" with an overridden Host header and returns the
1062	// response body + status code.
1063	func proxyGet(t *testing.T, base, host string) (string, int) {
1064	t.Helper()
1065	req, err := http.NewRequest(http.MethodGet, base+"/", nil)
1066	if err != nil {
1067	t.Fatalf("new request: %v", err)
1068	}
1069	req.Host = host
1070	resp, err := http.DefaultClient.Do(req)
1071	if err != nil {
1072	t.Fatalf("request: %v", err)
1073	}
1074	defer resp.Body.Close()
1075	b, _ := io.ReadAll(resp.Body)
1076	return string(b), resp.StatusCode
1077	}
1078
1079	// requireUnixSockets skips a test when the sandbox forbids unix-socket
1080	// creation (probe: net.Listen("unix")), matching internal/host's helper. Most
1081	// tests here fake the backend over TCP; only the ones that bind a real unix
1082	// listener need this.
1083	func requireUnixSockets(t *testing.T) {
1084	t.Helper()
1085	l, err := net.Listen("unix", tmpSock(t))
1086	if err != nil {
1087	if strings.Contains(err.Error(), "operation not permitted") {
1088	t.Skipf("sandbox forbids unix sockets: %v", err)
1089	}
1090	t.Fatalf("unix socket probe: %v", err)
1091	}
1092	l.Close()
1093	}
1094
1095	// tmpSockSeq numbers tmpSock's paths, so tests running in parallel never
1096	// share one.
1097	var tmpSockSeq atomic.Uint64
1098
1099	// tmpSock returns a short unix socket path (the sun_path limit is ~104 bytes,
1100	// so t.TempDir() can be too long) and schedules its removal.
1101	func tmpSock(t *testing.T) string {
1102	t.Helper()
1103	s := filepath.Join(os.TempDir(), fmt.Sprintf("carlos-act-%d-%d.sock", os.Getpid(), tmpSockSeq.Add(1)))
1104	os.Remove(s)
1105	t.Cleanup(func() { os.Remove(s) })
1106	return s
1107	}
1108
1109	// --- tests ---------------------------------------------------------------------
1110
1111	// Many concurrent requests during a wake coalesce into exactly one wake: one
1112	// lease claim, one restore, one spawn.
1113	func TestActivatorWakeCoalesces(t *testing.T) {
1114	t.Parallel()
1115	a, b, _, _ := testActivator(t, time.Minute)
1116	b.startGate = make(chan struct{})
1117
1118	const n = 12
1119	var ok atomic.Int32
1120	var wg sync.WaitGroup
1121	for i := 0; i < n; i++ {
1122	wg.Add(1)
1123	go func() {
1124	defer wg.Done()
1125	release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
1126	if err != nil {
1127	t.Errorf("beginRequest: %v", err)
1128	return
1129	}
1130	ok.Add(1)
1131	release()
1132	}()
1133	}
1134	// Let the requests pile up on the gated wake, then open it.
1135	waitFor(t, "the wake to begin", func() bool { _, _, s := b.counts(); return s == 1 })
1136	time.Sleep(50 * time.Millisecond)
1137	close(b.startGate)
1138	wg.Wait()
1139
1140	if got := ok.Load(); got != n {
1141	t.Fatalf("%d/%d requests succeeded", got, n)
1142	}
1143	acquires, restores, starts := b.counts()
1144	if acquires != 1 || restores != 1 || starts != 1 {
1145	t.Fatalf("wake did not coalesce: acquires=%d restores=%d starts=%d (want 1 each)", acquires, restores, starts)
1146	}
1147	}
1148
1149	// OnWake (customer-fleets plan, Task 6: a fleet box's binding publish) must
1150	// fire after a successful wake, named with the right host, and — the whole
1151	// point of running it via `go a.OnWake(host)` rather than calling it inline —
1152	// must NOT be able to hold the wake open: beginRequest returns and releases
1153	// long before the hook is allowed to finish.
1154	func TestActivatorWakeCallsOnWakeHookAsync(t *testing.T) {
1155	a, _, _, _ := testActivator(t, time.Minute)
1156	block := make(chan struct{})
1157	hookHost := make(chan string, 1)
1158	a.OnWake = func(rt registry.Route) {
1159	<-block // would hang beginRequest forever if invoked synchronously
1160	hookHost <- rt.Host
1161	}
1162
1163	done := make(chan struct{})
1164	go func() {
1165	release, err := a.beginRequest(context.Background(), testRoute("hook.example.com"))
1166	if err != nil {
1167	t.Errorf("beginRequest: %v", err)
1168	return
1169	}
1170	release()
1171	close(done)
1172	}()
1173
1174	select {
1175	case <-done:
1176	case <-time.After(2 * time.Second):
1177	t.Fatal("beginRequest never returned — OnWake appears to be blocking the wake path")
1178	}
1179
1180	close(block)
1181	select {
1182	case host := <-hookHost:
1183	if host != "hook.example.com" {
1184	t.Fatalf("OnWake called with %q, want hook.example.com", host)
1185	}
1186	case <-time.After(2 * time.Second):
1187	t.Fatal("OnWake was never called after a successful wake")
1188	}
1189	}
1190
1191	// A nil OnWake (every non-fleet deployment, and every test above this one)
1192	// must be a pure no-op — no crash, wake behaves exactly as before Task 6.
1193	func TestActivatorNilOnWakeIsNoop(t *testing.T) {
1194	a, _, _, _ := testActivator(t, time.Minute)
1195	release, err := a.beginRequest(context.Background(), testRoute("nohook.example.com"))
1196	if err != nil {
1197	t.Fatalf("beginRequest: %v", err)
1198	}
1199	release()
1200	}
1201
1202	// The wake threads the route's replication class into Replicate() as the
1203	// litestream sync interval — the per-plan durability ladder's seam. Unset
1204	// (every pre-ladder route) must thread 0, i.e. litestream's default: a deploy
1205	// of the ladder changes no host's durability by itself.
1206	func TestActivatorWakeThreadsReplicationClass(t *testing.T) {
1207	a, b, _, _ := testActivator(t, time.Minute)
1208
1209	for i, tc := range []struct {
1210	class string
1211	want time.Duration
1212	}{
1213	{"", 0},
1214	{replicationStream, 0},
1215	{replicationBatched, 5 * time.Minute},
1216	{replicationDaily, 24 * time.Hour},
1217	} {
1218	rt := testRoute(fmt.Sprintf("ladder%d.example.com", i))
1219	rt.Replication = tc.class
1220	release, err := a.beginRequest(context.Background(), rt)
1221	if err != nil {
1222	t.Fatalf("beginRequest(%q): %v", tc.class, err)
1223	}
1224	release()
1225	b.mu.Lock()
1226	got := b.syncIntervals[len(b.syncIntervals)-1]
1227	b.mu.Unlock()
1228	if got != tc.want {
1229	t.Errorf("class %q threaded sync interval %v, want %v", tc.class, got, tc.want)
1230	}
1231	}
1232	}
1233
1234	// --- first-wake seed ---------------------------------------------------------
1235	//
1236	// The seam spec §6 pins: fresh means the local DB was absent AND no replica
1237	// existed — a first boot, never a restore — and the route's seed runs strictly
1238	// between Restore and Start, i.e. before the first health probe ever reaches
1239	// the instance. A brand-new instance is therefore born seeded, and a restored
1240	// one is never re-seeded over its own data.
1241
1242	func TestActivatorSeedsFreshInstanceBeforeStart(t *testing.T) {
1243	a, b, _, _ := testActivator(t, time.Minute)
1244	b.restoreFresh = true
1245	rt := testRoute("fresh.example.com")
1246	rt.Seed = `["seed"]`
1247
1248	release, err := a.beginRequest(context.Background(), rt)
1249	if err != nil {
1250	t.Fatalf("beginRequest: %v", err)
1251	}
1252	release()
1253
1254	if got := b.seedCount(); got != 1 {
1255	t.Fatalf("seed calls = %d, want 1 on a fresh boot with a seed", got)
1256	}
1257	ev := b.events()
1258	r, s, st := indexOf(ev, "restore"), indexOf(ev, "seed"), indexOf(ev, "start:instance")
1259	if r == -1 || s == -1 || st == -1 || !(r < s && s < st) {
1260	t.Fatalf("seed order wrong: %v (want restore < seed < start:instance — seeded before the first health probe)", ev)
1261	}
1262	b.mu.Lock()
1263	seededHost := b.seedRoutes[0].Host
1264	b.mu.Unlock()
1265	if seededHost != rt.Host {
1266	t.Fatalf("seeded host = %q, want %q", seededHost, rt.Host)
1267	}
1268	}
1269
1270	// A restore is never a first boot: the instance's data came back from the
1271	// bucket, and running the seed over it is exactly the corruption this fresh
1272	// flag exists to prevent.
1273	func TestActivatorSkipsSeedWhenRestored(t *testing.T) {
1274	a, b, _, _ := testActivator(t, time.Minute)
1275	b.restoreFresh = false
1276	rt := testRoute("restored.example.com")
1277	rt.Seed = `["seed"]`
1278
1279	release, err := a.beginRequest(context.Background(), rt)
1280	if err != nil {
1281	t.Fatalf("beginRequest: %v", err)
1282	}
1283	release()
1284
1285	if got := b.seedCount(); got != 0 {
1286	t.Fatalf("seed calls = %d on a restored instance, want 0", got)
1287	}
1288	}
1289
1290	// No seed declared: a fresh boot just boots (every route added before this
1291	// feature, and every app that creates its own schema).
1292	func TestActivatorSkipsSeedWhenRouteHasNone(t *testing.T) {
1293	a, b, _, _ := testActivator(t, time.Minute)
1294	b.restoreFresh = true
1295	rt := testRoute("noseed.example.com") // rt.Seed == ""
1296
1297	release, err := a.beginRequest(context.Background(), rt)
1298	if err != nil {
1299	t.Fatalf("beginRequest: %v", err)
1300	}
1301	release()
1302
1303	if got := b.seedCount(); got != 0 {
1304	t.Fatalf("seed calls = %d for a route with no seed, want 0", got)
1305	}
1306	}
1307
1308	// A failed seed fails the WHOLE wake — a half-seeded instance must not serve —
1309	// and gives the lease back exactly once, the Start-failure precedent. Nothing
1310	// is spawned, and the entry is gone so the next request retries from scratch.
1311	func TestActivatorSeedFailureFailsWakeAndReleasesLease(t *testing.T) {
1312	a, b, _, _ := testActivator(t, time.Minute)
1313	b.restoreFresh = true
1314	b.seedErr = errors.New("seed exited 1")
1315	rt := testRoute("badseed.example.com")
1316	rt.Seed = `["seed"]`
1317
1318	_, err := a.beginRequest(context.Background(), rt)
1319	if err == nil {
1320	t.Fatal("wake succeeded despite a failing seed")
1321	}
1322	if !strings.Contains(err.Error(), "seed "+rt.Host) {
1323	t.Fatalf("wake error = %q, want it to name the seed and the host", err)
1324	}
1325	if _, _, starts := b.counts(); starts != 0 {
1326	t.Fatalf("instance was spawned after a failed seed (starts=%d)", starts)
1327	}
1328	if got := b.releaseCount(); got != 1 {
1329	t.Fatalf("lease releases = %d after a failed seed, want exactly 1", got)
1330	}
1331	if a.entryCount() != 0 {
1332	t.Fatal("failed wake left an entry behind")
1333	}
1334	}
1335
1336	// TestActivatorSeedUnprovenLeavesTheLeaseToItsTTL pins design §6.2: a seed
1337	// whose cgroup never emptied may still have a SIGKILL survivor writing the
1338	// database, so doWake must not release the lease over it the way it does
1339	// for every other seed failure (design §6.3's ErrStillRunning gets the same
1340	// treatment in stop()).
1341	func TestActivatorSeedUnprovenLeavesTheLeaseToItsTTL(t *testing.T) {
1342	a, b, _, _ := testActivator(t, time.Minute)
1343	b.restoreFresh = true
1344	b.seedErr = fmt.Errorf("seed for badseed.example.com: %w: processes remain after a successful seed: still running", ErrSeedUnproven)
1345	rt := testRoute("badseed.example.com")
1346	rt.Seed = `["seed"]`
1347
1348	_, err := a.beginRequest(context.Background(), rt)
1349	if err == nil {
1350	t.Fatal("wake succeeded despite an unproven seed")
1351	}
1352	if !errors.Is(err, ErrSeedUnproven) {
1353	t.Fatalf("wake error = %v, want it to wrap ErrSeedUnproven", err)
1354	}
1355	if got := b.releaseCount(); got != 0 {
1356	t.Fatalf("lease releases = %d for an unproven seed, want exactly 0", got)
1357	}
1358	}
1359
1360	// Task 8, plan 3: a wake whose Restore refuses because an earlier seed's
1361	// processes are still not proven gone (the <db>.seed-unproven obligation)
1362	// treats it as the seed's own ErrSeedUnproven: nothing is seeded or
1363	// started, and the lease is left to its TTL.
1364	func TestActivatorRestoreRefusedForAnUnprovenSeedLeavesTheLeaseToItsTTL(t *testing.T) {
1365	a, b, _, _ := testActivator(t, time.Minute)
1366	b.restoreErr = fmt.Errorf("unproven.example.com: %w: an earlier seed's processes are not proven gone", ErrSeedUnproven)
1367	rt := testRoute("unproven.example.com")
1368	rt.Seed = `["seed"]`
1369
1370	_, err := a.beginRequest(context.Background(), rt)
1371	if !errors.Is(err, ErrSeedUnproven) {
1372	t.Fatalf("wake error = %v, want it to wrap ErrSeedUnproven", err)
1373	}
1374	if got := b.releaseCount(); got != 0 {
1375	t.Fatalf("lease releases = %d, want 0: something of the earlier seed may still write the database", got)
1376	}
1377	b.mu.Lock()
1378	seeds, starts := b.seeds, b.starts
1379	b.mu.Unlock()
1380	if seeds != 0 || starts != 0 {
1381	t.Fatalf("seeds=%d starts=%d after a refused restore, want neither", seeds, starts)
1382	}
1383	}
1384
1385	// execSeedBackend is fakeBackend with the real ExecBackend's Restore and
1386	// Seed, so a test can drive doWake through the seed obligation on disk.
1387	type execSeedBackend struct {
1388	*fakeBackend
1389	eb *ExecBackend
1390	}
1391
1392	func (x *execSeedBackend) Restore(ctx context.Context, rt registry.Route) (bool, error) {
1393	x.record("restore")
1394	return x.eb.Restore(ctx, rt)
1395	}
1396
1397	func (x *execSeedBackend) Seed(ctx context.Context, rt registry.Route) error {
1398	x.record("seed")
1399	return x.eb.Seed(ctx, rt)
1400	}
1401
1402	// Task 8 fix round 1, finding 8: the next wake after an unproven seed,
1403	// driven through doWake. The first wake's seed fails and its processes are
1404	// not proven gone: the lease stays and the obligation is on disk. The next
1405	// wake, while the proof still fails, is refused before any seed or start,
1406	// with the lease kept and the database untouched. Once the proof holds, the
1407	// wake removes the database, seeds again and starts.
1408	func TestTheNextWakeAfterAnUnprovenSeedWaitsForItsProof(t *testing.T) {
1409	const host = "next-wake.example.com"
1410	a, b, _, _ := testActivator(t, time.Minute)
1411	eb, _ := delegatedBackend(t, host)
1412	a.backend = &execSeedBackend{fakeBackend: b, eb: eb}
1413	runs := filepath.Join(t.TempDir(), "runs")
1414	stageSeedScript(t, eb, host, "v1", fmt.Sprintf(
1415	"echo run >> %q\nprintf half > \"$3\"\n[ \"$(wc -l < %q)\" -ge 2 ] && exit 0\nexit 1\n", runs, runs))
1416	swapRestoreReplica(t, func(context.Context, string, string, string, string) (bool, error) {
1417	return false, nil
1418	})
1419	rt := testRoute(host)
1420	rt.DBPath = ""
1421	rt.Seed = `["seed"]`
1422	db := filepath.Join(eb.DataDir, host+".db")
1423	seedRuns := func() int {
1424	raw, _ := os.ReadFile(runs)
1425	return strings.Count(string(raw), "run\n")
1426	}
1427
1428	if _, err := a.beginRequest(context.Background(), rt); !errors.Is(err, ErrSeedUnproven) {
1429	t.Fatalf("first wake = %v, want ErrSeedUnproven", err)
1430	}
1431	if _, err := os.Stat(db + seedUnprovenSuffix); err != nil {
1432	t.Fatalf("no obligation after the first wake: %v", err)
1433	}
1434
1435	// The proof still fails (a temp dir is no cgroupfs): refused.
1436	if _, err := a.beginRequest(context.Background(), rt); !errors.Is(err, ErrSeedUnproven) {
1437	t.Fatalf("second wake = %v, want ErrSeedUnproven", err)
1438	}
1439	if got := b.releaseCount(); got != 0 {
1440	t.Fatalf("lease releases = %d, want 0 while the seed is unproven", got)
1441	}
1442	if n, starts := seedRuns(), b.startCount(); n != 1 || starts != 0 {
1443	t.Fatalf("seed runs = %d, starts = %d after a refused wake; want 1 and 0", n, starts)
1444	}
1445	if got, _ := os.ReadFile(db); string(got) != "half" {
1446	t.Fatalf("database = %q after a refused wake, want it untouched", got)
1447	}
1448
1449	// The proof holds: the wake reseeds and starts.
1450	eb.cgroupKill = func(context.Context, registry.Route, string) error { return nil }
1451	if _, err := a.beginRequest(context.Background(), rt); err != nil {
1452	t.Fatalf("third wake = %v, want it to reseed and start", err)
1453	}
1454	if n, starts := seedRuns(), b.startCount(); n != 2 || starts != 1 {
1455	t.Fatalf("seed runs = %d, starts = %d; want the seed to have run again and the instance started", n, starts)
1456	}
1457	if _, err := os.Stat(db + seedUnprovenSuffix); !os.IsNotExist(err) {
1458	t.Fatalf("obligation still there (%v) after a proven wake", err)
1459	}
1460	}
1461
1462	// TestActivatorWakeRefusesFencedRowBeforeAcquire pins design §6.8: a fenced
1463	// row is refused at the top of doWake, before the lease is ever acquired,
1464	// not only inside Start after the lease is already held.
1465	func TestActivatorWakeRefusesFencedRowBeforeAcquire(t *testing.T) {
1466	a, b, _, _ := testActivator(t, time.Minute)
1467	const reason = "layout migration in flight"
1468	b.layoutFn = func(registry.Route) (Layout, string) { return LayoutFenced, reason }
1469	rt := testRoute("fenced.example.com")
1470
1471	_, err := a.beginRequest(context.Background(), rt)
1472	if err == nil {
1473	t.Fatal("wake succeeded against a fenced row")
1474	}
1475	if !strings.Contains(err.Error(), reason) {
1476	t.Fatalf("wake error = %q, want it to carry the fence reason %q", err, reason)
1477	}
1478	if acquires, _, starts := b.counts(); acquires != 0 || starts != 0 {
1479	t.Fatalf("fenced row reached the backend (acquires=%d starts=%d), want neither", acquires, starts)
1480	}
1481	}
1482
1483	// The other two doWake bail-outs release the lease exactly once too — the
1484	// property the seed step slots into the middle of.
1485	func TestActivatorWakeFailuresReleaseLeaseOnce(t *testing.T) {
1486	t.Parallel()
1487	for _, tc := range []struct {
1488	name string
1489	inject func(*fakeBackend)
1490	want string
1491	}{
1492	{"restore", func(b *fakeBackend) { b.restoreErr = errors.New("s3 down") }, "restore "},
1493	{"start", func(b *fakeBackend) { b.startErr = errors.New("never healthy") }, "start "},
1494	} {
1495	t.Run(tc.name, func(t *testing.T) {
1496	a, b, _, _ := testActivator(t, time.Minute)
1497	tc.inject(b)
1498	rt := testRoute(tc.name + "-fail.example.com")
1499
1500	_, err := a.beginRequest(context.Background(), rt)
1501	if err == nil {
1502	t.Fatalf("wake succeeded despite a failing %s", tc.name)
1503	}
1504	if !strings.Contains(err.Error(), tc.want+rt.Host) {
1505	t.Fatalf("wake error = %q, want it to name %q and the host", err, tc.want)
1506	}
1507	if got := b.releaseCount(); got != 1 {
1508	t.Fatalf("lease releases = %d after a failed %s, want exactly 1", got, tc.name)
1509	}
1510	if a.entryCount() != 0 {
1511	t.Fatal("failed wake left an entry behind")
1512	}
1513	})
1514	}
1515	}
1516
1517	// The singleflight hole behind flake #184: the wake INITIATOR must wait on
1518	// the entry it created, never re-read the map — a fast-FAILING wake deletes
1519	// its entry, and an initiator that looks the host up again finds nothing and
1520	// starts a second wake: two lease acquires, two restore attempts, doubled
1521	// error events, for one request. The window is one preemption wide (between
1522	// the initiator's unlock and its next lock), so this drives many failed
1523	// wakes under -race and contending goroutines and asserts the invariant
1524	// acquires == requests exactly.
1525	func TestActivatorFailedWakeIsSingleflight(t *testing.T) {
1526	t.Parallel()
1527	a, b, _, _ := testActivator(t, time.Minute)
1528	b.restoreErr = errors.New("s3 down")
1529	rt := testRoute("singleflight.example.com")
1530
1531	// Background lock churn widens the initiator's unlock→lock window the
1532	// way a loaded test suite does.
1533	stopChurn := make(chan struct{})
1534	var churn sync.WaitGroup
1535	for i := 0; i < 4; i++ {
1536	churn.Add(1)
1537	go func() {
1538	defer churn.Done()
1539	for {
1540	select {
1541	case <-stopChurn:
1542	return
1543	default:
1544	a.mu.Lock()
1545	_ = len(a.entries) //nolint:staticcheck // hold briefly
1546	a.mu.Unlock()
1547	}
1548	}
1549	}()
1550	}
1551
1552	const requests = 300
1553	for i := 0; i < requests; i++ {
1554	if _, err := a.beginRequest(context.Background(), rt); err == nil {
1555	t.Fatal("wake succeeded despite a failing restore")
1556	}
1557	}
1558	close(stopChurn)
1559	churn.Wait()
1560
1561	b.mu.Lock()
1562	acquires, releases := b.acquires, b.releases
1563	b.mu.Unlock()
1564	if acquires != requests || releases != requests {
1565	t.Fatalf("acquires=%d releases=%d for %d requests — a failed wake was not singleflight (#184)",
1566	acquires, releases, requests)
1567	}
1568	if a.entryCount() != 0 {
1569	t.Fatal("failed wakes left entries behind")
1570	}
1571	}
1572
1573	// A restore failure never reaches the seed: seeding a host whose data we
1574	// couldn't materialize is precisely the "half-seeded instance" case.
1575	func TestActivatorRestoreFailureSkipsSeed(t *testing.T) {
1576	a, b, _, _ := testActivator(t, time.Minute)
1577	b.restoreErr = errors.New("s3 down")
1578	rt := testRoute("restorefail.example.com")
1579	rt.Seed = `["seed"]`
1580
1581	if _, err := a.beginRequest(context.Background(), rt); err == nil {
1582	t.Fatal("wake succeeded despite a failing restore")
1583	}
1584	if got := b.seedCount(); got != 0 {
1585	t.Fatalf("seed calls = %d after a failed restore, want 0", got)
1586	}
1587	}
1588
1589	// After the idle window with nothing in flight, the sweeper hibernates the
1590	// instance: process stopped, replication closed (final checkpoint + lease
1591	// release), entry forgotten; the next request wakes it afresh.
1592	func TestActivatorIdleSleep(t *testing.T) {
1593	t.Parallel()
1594	a, b, clk, _ := testActivator(t, 10*time.Second)
1595
1596	release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
1597	if err != nil {
1598	t.Fatalf("beginRequest: %v", err)
1599	}
1600	release()
1601
1602	// Not yet idle: nothing happens.
1603	clk.advance(9 * time.Second)
1604	a.sweepOnce()
1605	if a.entryCount() != 1 {
1606	t.Fatal("instance hibernated before its idle window elapsed")
1607	}
1608
1609	clk.advance(2 * time.Second)
1610	a.sweepOnce()
1611	waitFor(t, "hibernation", func() bool { return a.entryCount() == 0 })
1612	if !b.lastProc().isStopped() {
1613	t.Fatal("idle instance's process was not stopped")
1614	}
1615	if !b.lastRep().isClosed() {
1616	t.Fatal("idle instance's replication was not closed (no final checkpoint/lease release)")
1617	}
1618
1619	// A new request wakes it again — a second full wake.
1620	release, err = a.beginRequest(context.Background(), testRoute("one.example.com"))
1621	if err != nil {
1622	t.Fatalf("beginRequest after sleep: %v", err)
1623	}
1624	release()
1625	if _, _, starts := b.counts(); starts != 2 {
1626	t.Fatalf("starts=%d after sleep+rewake, want 2", starts)
1627	}
1628	}
1629
1630	// An in-flight request (an open WebSocket, in production) holds the instance
1631	// awake past the idle window; only after it finishes does the idle clock run.
1632	func TestActivatorInflightKeepsAwake(t *testing.T) {
1633	t.Parallel()
1634	a, b, clk, _ := testActivator(t, 10*time.Second)
1635
1636	release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
1637	if err != nil {
1638	t.Fatalf("beginRequest: %v", err)
1639	}
1640
1641	clk.advance(time.Hour)
1642	a.sweepOnce()
1643	if a.entryCount() != 1 || b.lastProc().isStopped() {
1644	t.Fatal("instance with an in-flight request was hibernated")
1645	}
1646
1647	release() // stamps lastActive = now
1648	a.sweepOnce()
1649	if a.entryCount() != 1 {
1650	t.Fatal("instance hibernated immediately after its last request finished")
1651	}
1652	clk.advance(11 * time.Second)
1653	a.sweepOnce()
1654	waitFor(t, "hibernation", func() bool { return a.entryCount() == 0 })
1655	}
1656
1657	// A ship repoints live/<host> under an awake child. A running process never
1658	// re-reads its symlink, so idle-hibernate is the only thing that converges an
1659	// instance — and an always-awake host (every sidecar host) never idles.
1660	// The sweep closes that gap: it notices the drift and, after a grace window and
1661	// rate-limited to one per tick, gracefully cycles the child so its next wake
1662	// execs the new binary.
1663	func TestActivatorConvergesOnSymlinkRepoint(t *testing.T) {
1664	t.Parallel()
1665	a, b, clk, _ := testActivator(t, time.Hour) // idle far away — isolate the convergence path
1666	var ver atomic.Value // the live/<host> symlink target; swapped to model a ship
1667	ver.Store("v1")
1668	// Model one symlink: the child execs whatever it points at now (execVerFn,
1669	// frozen into spawnVersion at wake), and the sweep reads the current target
1670	// (liveVer). A "ship" swaps ver; the running child keeps its frozen version.
1671	a.liveVer = func(string) (string, error) { return ver.Load().(string), nil }
1672	b.execVerFn = func() string { return ver.Load().(string) }
1673
1674	release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
1675	if err != nil {
1676	t.Fatalf("beginRequest: %v", err)
1677	}
1678	release()
1679
1680	// Symlink unchanged: never cycles, however long it stays awake.
1681	clk.advance(3 * ConvergeGrace)
1682	a.sweepOnce()
1683	if a.entryCount() != 1 {
1684	t.Fatal("cycled an instance whose symlink never moved")
1685	}
1686
1687	// Ship: repoint the symlink. The first sweep after only starts the grace
1688	// clock — it must not cycle yet, so a move-then-rollback never drops a live
1689	// connection.
1690	ver.Store("v2")
1691	a.sweepOnce()
1692	if a.entryCount() != 1 {
1693	t.Fatal("cycled inside the grace window — a transient repoint would drop a live connection")
1694	}
1695
1696	// Past the grace: gracefully cycle (stop → final checkpoint → lease release →
1697	// asleep). The reconnecting client/sidecar would wake it onto v2.
1698	clk.advance(ConvergeGrace + time.Second)
1699	a.sweepOnce()
1700	waitFor(t, "convergence cycle", func() bool { return a.entryCount() == 0 })
1701	if !b.lastProc().isStopped() {
1702	t.Fatal("stale instance's process was not stopped")
1703	}
1704	if !b.lastRep().isClosed() {
1705	t.Fatal("convergence cycle skipped the final checkpoint / lease release")
1706	}
1707
1708	// Re-wake against the now-current symlink: spawnVersion tracks v2, so it
1709	// never cycles again — no thrash.
1710	release, err = a.beginRequest(context.Background(), testRoute("one.example.com"))
1711	if err != nil {
1712	t.Fatalf("re-wake after convergence: %v", err)
1713	}
1714	release()
1715	clk.advance(3 * ConvergeGrace)
1716	a.sweepOnce()
1717	if a.entryCount() != 1 {
1718	t.Fatal("cycled again after already converging onto the current build")
1719	}
1720	}
1721
1722	// Two awake children go stale at once (a fleet-wide ship). One sweep cycles at
1723	// most one — a staggered convergence, never a simultaneous blip of every space.
1724	func TestActivatorConvergenceRateLimitsPerSweep(t *testing.T) {
1725	t.Parallel()
1726	a, b, clk, _ := testActivator(t, time.Hour)
1727	var ver atomic.Value
1728	ver.Store("v1")
1729	a.liveVer = func(string) (string, error) { return ver.Load().(string), nil }
1730	b.execVerFn = func() string { return ver.Load().(string) }
1731
1732	for _, h := range []string{"one.example.com", "two.example.com"} {
1733	release, err := a.beginRequest(context.Background(), testRoute(h))
1734	if err != nil {
1735	t.Fatalf("beginRequest %s: %v", h, err)
1736	}
1737	release()
1738	}
1739	if a.entryCount() != 2 {
1740	t.Fatalf("want 2 awake instances, got %d", a.entryCount())
1741	}
1742
1743	ver.Store("v2") // ship under both
1744	a.sweepOnce() // starts the grace clock for both
1745	clk.advance(ConvergeGrace + time.Second)
1746
1747	a.sweepOnce() // both are past grace, but only one may cycle this tick
1748	waitFor(t, "first convergence cycle", func() bool { return a.entryCount() == 1 })
1749	// A moment later, the sweep takes the second.
1750	a.sweepOnce()
1751	waitFor(t, "second convergence cycle", func() bool { return a.entryCount() == 0 })
1752	}
1753
1754	// The feature's whole premise: converge a host that NEVER reaches inflight==0
1755	// (an open WebSocket, or a sidecar's persistent connection), which
1756	// idle-hibernate can never touch. Unlike idle-hibernate, convergence fires with
1757	// a request in flight — the graceful stop drains it exactly as a restart would.
1758	func TestActivatorConvergesWithRequestInFlight(t *testing.T) {
1759	t.Parallel()
1760	a, b, clk, _ := testActivator(t, time.Hour)
1761	var ver atomic.Value
1762	ver.Store("v1")
1763	a.liveVer = func(string) (string, error) { return ver.Load().(string), nil }
1764	b.execVerFn = func() string { return ver.Load().(string) }
1765
1766	// Hold the request open for the whole test — inflight stays 1, so the idle
1767	// path (which needs inflight==0) can never converge this host.
1768	release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
1769	if err != nil {
1770	t.Fatalf("beginRequest: %v", err)
1771	}
1772	defer release()
1773
1774	ver.Store("v2") // ship under the busy host
1775	a.sweepOnce() // grace starts; must not cycle a busy host mid-grace
1776	if a.entryCount() != 1 {
1777	t.Fatal("cycled inside the grace window")
1778	}
1779	clk.advance(ConvergeGrace + time.Second)
1780	a.sweepOnce()
1781	// Cycles despite the in-flight request — idle-hibernate never would.
1782	waitFor(t, "busy-host convergence", func() bool { return a.entryCount() == 0 })
1783	if !b.lastProc().isStopped() {
1784	t.Fatal("busy stale instance's process was not stopped")
1785	}
1786	}
1787
1788	// ConvergeDecision's state machine, exhaustively: the grace clock starts on
1789	// first drift, survives a rollback correctly, and a read error never clears
1790	// accrued grace (it means "unknown", not "converged").
1791	func TestConvergeDecision(t *testing.T) {
1792	base := time.Unix(1_700_000_000, 0)
1793	set := base.Add(-40 * time.Second) // a grace clock that started 40s ago
1794	errBoom := errors.New("readlink boom")
1795	cases := []struct {
1796	name string
1797	cur string
1798	readErr error
1799	spawnVer string
1800	staleSince time.Time
1801	wantCycle bool
1802	wantStale time.Time
1803	}{
1804	{"matches — clears grace", "v1", nil, "v1", set, false, time.Time{}},
1805	{"empty cur — no data, clears", "", nil, "v1", set, false, time.Time{}},
1806	{"empty spawnVer — unknown, clears", "v2", nil, "", set, false, time.Time{}},
1807	{"first drift — starts grace, no cycle", "v2", nil, "v1", time.Time{}, false, base},
1808	{"drift within grace — holds, no cycle", "v2", nil, "v1", base.Add(-30 * time.Second), false, base.Add(-30 * time.Second)},
1809	{"drift past grace — cycles", "v2", nil, "v1", base.Add(-ConvergeGrace - time.Second), true, base.Add(-ConvergeGrace - time.Second)},
1810	{"read error — preserves accrued grace", "", errBoom, "v1", set, false, set},
1811	{"read error before any drift — stays zero", "", errBoom, "v1", time.Time{}, false, time.Time{}},
1812	{"rollback to spawn — clears grace", "v1", nil, "v1", base.Add(-ConvergeGrace - time.Second), false, time.Time{}},
1813	}
1814	for _, c := range cases {
1815	t.Run(c.name, func(t *testing.T) {
1816	cycle, next := ConvergeDecision(c.cur, c.readErr, c.spawnVer, c.staleSince, base, ConvergeGrace)
1817	if cycle != c.wantCycle {
1818	t.Errorf("cycle = %v, want %v", cycle, c.wantCycle)
1819	}
1820	if !next.Equal(c.wantStale) {
1821	t.Errorf("nextStale = %v, want %v", next, c.wantStale)
1822	}
1823	})
1824	}
1825	}
1826
1827	// A lost lease (failed renewal) stands the instance down immediately — no
1828	// waiting for idle: someone else may own the S3 prefix now.
1829	func TestActivatorLeaseLostStandsDown(t *testing.T) {
1830	t.Parallel()
1831	a, b, _, _ := testActivator(t, time.Hour)
1832
1833	release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
1834	if err != nil {
1835	t.Fatalf("beginRequest: %v", err)
1836	}
1837	release()
1838
1839	b.lastRep().setErr(errors.New("lease renewal failed"))
1840	a.sweepOnce() // idle window is an hour away; the lost lease alone must trigger the stop
1841	waitFor(t, "stand-down", func() bool { return a.entryCount() == 0 })
1842	if !b.lastProc().isStopped() {
1843	t.Fatal("lease-lost instance's process was not stopped")
1844	}
1845	if !b.lastRep().isClosed() {
1846	t.Fatal("lease-lost instance's replication was not closed")
1847	}
1848	}
1849
1850	// A dead child (crashed instance) is recycled by the next request: the stale
1851	// entry is stopped and a fresh wake happens in the same beginRequest call.
1852	func TestActivatorDeadChildRewakesOnRequest(t *testing.T) {
1853	a, b, _, _ := testActivator(t, time.Hour)
1854
1855	release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
1856	if err != nil {
1857	t.Fatalf("beginRequest: %v", err)
1858	}
1859	release()
1860
1861	p := b.lastProc()
1862	p.mu.Lock()
1863	p.dead = true
1864	p.mu.Unlock()
1865
1866	release, err = a.beginRequest(context.Background(), testRoute("one.example.com"))
1867	if err != nil {
1868	t.Fatalf("beginRequest after crash: %v", err)
1869	}
1870	release()
1871	if _, _, starts := b.counts(); starts != 2 {
1872	t.Fatalf("starts=%d after crash+request, want 2 (recycle + rewake)", starts)
1873	}
1874	}
1875
1876	// A request that arrives mid-stop waits for the stop, then wakes fresh.
1877	func TestActivatorRequestDuringStopRewakes(t *testing.T) {
1878	t.Parallel()
1879	a, b, clk, _ := testActivator(t, 10*time.Second)
1880
1881	release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
1882	if err != nil {
1883	t.Fatalf("beginRequest: %v", err)
1884	}
1885	release()
1886
1887	// Hold the stop open so the entry sits in stopping.
1888	gate := make(chan struct{})
1889	p := b.lastProc()
1890	p.mu.Lock()
1891	p.stopGate = gate
1892	p.mu.Unlock()
1893
1894	clk.advance(time.Minute)
1895	a.sweepOnce()
1896
1897	got := make(chan error, 1)
1898	go func() {
1899	release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
1900	if err == nil {
1901	release()
1902	}
1903	got <- err
1904	}()
1905	time.Sleep(50 * time.Millisecond) // the request should now be parked on e.stopped
1906	close(gate)
1907	if err := <-got; err != nil {
1908	t.Fatalf("request during stop: %v", err)
1909	}
1910	if _, _, starts := b.counts(); starts != 2 {
1911	t.Fatalf("starts=%d, want 2 (sleep then rewake)", starts)
1912	}
1913	}
1914
1915	// A wake that exceeds the bound turns queued requests away with errWakeTimeout
1916	// (503 at the edge) rather than piling connections up behind it.
1917	func TestActivatorWakeTimeout(t *testing.T) {
1918	a, b, _, _ := testActivator(t, time.Minute)
1919	old := wakeWaitBound
1920	wakeWaitBound = 100 * time.Millisecond
1921	t.Cleanup(func() { wakeWaitBound = old })
1922
1923	gate := make(chan struct{})
1924	b.startGate = gate
1925	t.Cleanup(func() { close(gate) }) // let the stuck wake finish at test end
1926
1927	_, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
1928	if !errors.Is(err, errWakeTimeout) {
1929	t.Fatalf("got %v, want errWakeTimeout", err)
1930	}
1931	}
1932
1933	// Lease contention surfaces through the edge as 503 + Retry-After, and a
1934	// failed wake leaves no entry behind (the next request retries the claim).
1935	func TestActivatorLeaseContention503(t *testing.T) {
1936	a, b, _, reg := testActivator(t, time.Minute)
1937	b.acquireErr = &errLeaseHeld{holder: "old-incarnation", retryAfter: 7 * time.Second}
1938
1939	if err := reg.Upsert(registry.Route{Host: "held.example.com", Kind: "instance", Socket: "/tmp/held.sock", Hibernate: true}); err != nil {
1940	t.Fatalf("upsert: %v", err)
1941	}
1942	rt, err := newTestRouter(reg, a)
1943	if err != nil {
1944	t.Fatalf("new router: %v", err)
1945	}
1946	front := httptest.NewServer(rt)
1947	t.Cleanup(front.Close)
1948
1949	req, _ := http.NewRequest("GET", front.URL+"/", nil)
1950	req.Host = "held.example.com"
1951	resp, err := http.DefaultClient.Do(req)
1952	if err != nil {
1953	t.Fatalf("request: %v", err)
1954	}
1955	defer resp.Body.Close()
1956	if resp.StatusCode != http.StatusServiceUnavailable {
1957	t.Fatalf("status %d, want 503", resp.StatusCode)
1958	}
1959	if ra := resp.Header.Get("Retry-After"); ra != "8" { // 7s truncated + 1
1960	t.Fatalf("Retry-After %q, want 8", ra)
1961	}
1962	if a.entryCount() != 0 {
1963	t.Fatal("failed wake left an entry behind")
1964	}
1965	}
1966
1967	// Per-host gating (the opt-in): with the activator CONFIGURED, only a
1968	// hibernate-flagged instance goes through the wake path; an unflagged one is
1969	// proxied straight, exactly as if there were no activator at all — the
1970	// deploy-safety property that lets the bucket env ship fleet-wide before any
1971	// host is flagged.
1972	func TestRouterPerHostHibernateGating(t *testing.T) {
1973	requireUnixSockets(t)
1974	a, b, _, reg := testActivator(t, time.Minute)
1975
1976	// Two live backends playing the instances. The flagged host's listener
1977	// stands in for the child a wake would have spawned (the fake backend's
1978	// Start() spawns nothing).
1979	bind := func(name string) string {
1980	sock := tmpSock(t)
1981	ln, err := net.Listen("unix", sock)
1982	if err != nil {
1983	t.Fatalf("listen unix: %v", err)
1984	}
1985	srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1986	io.WriteString(w, name)
1987	})}
1988	go srv.Serve(ln)
1989	t.Cleanup(func() { srv.Close() })
1990	return sock
1991	}
1992	if err := reg.Upsert(registry.Route{Host: "hib.example.com", Kind: "instance", Socket: bind("hib"), Hibernate: true}); err != nil {
1993	t.Fatal(err)
1994	}
1995	if err := reg.Upsert(registry.Route{Host: "plain.example.com", Kind: "instance", Socket: bind("plain")}); err != nil {
1996	t.Fatal(err)
1997	}
1998
1999	rt, err := newTestRouter(reg, a)
2000	if err != nil {
2001	t.Fatalf("new router: %v", err)
2002	}
2003	front := httptest.NewServer(rt)
2004	t.Cleanup(front.Close)
2005
2006	// The unflagged host: proxied straight, the activator never touched.
2007	if body, code := proxyGet(t, front.URL, "plain.example.com"); code != 200 || body != "plain" {
2008	t.Fatalf("unflagged host: got %d %q, want 200 plain", code, body)
2009	}
2010	if acquires, _, starts := b.counts(); acquires != 0 || starts != 0 {
2011	t.Fatalf("unflagged instance touched the activator (acquires=%d starts=%d)", acquires, starts)
2012	}
2013
2014	// The flagged host: served through a full wake — lease, restore, spawn.
2015	if body, code := proxyGet(t, front.URL, "hib.example.com"); code != 200 || body != "hib" {
2016	t.Fatalf("flagged host: got %d %q, want 200 hib", code, body)
2017	}
2018	if acquires, restores, starts := b.counts(); acquires != 1 || restores != 1 || starts != 1 {
2019	t.Fatalf("flagged host skipped the wake path: acquires=%d restores=%d starts=%d (want 1 each)", acquires, restores, starts)
2020	}
2021	}
2022
2023	// The activator's own gate: handed an unflagged (or non-instance) route it
2024	// declines — returns false, writes nothing, touches no backend — so the edge
2025	// proxies straight. This is the bool half of the edge.Activator contract.
2026	func TestServeHTTPUnmanagedRouteReturnsFalse(t *testing.T) {
2027	a, b, _, _ := testActivator(t, time.Minute)
2028	noop := http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
2029
2030	rec := httptest.NewRecorder()
2031	req := httptest.NewRequest("GET", "http://plain.example.com/", nil)
2032	unflagged := registry.Route{Host: "plain.example.com", Kind: "instance", Socket: "/tmp/x.sock"}
2033	if a.ServeHTTP(rec, req, unflagged, noop) {
2034	t.Fatal("unflagged instance route was handled by the activator")
2035	}
2036	svc := registry.Route{Host: "svc.example.com", Kind: "service", Socket: "/tmp/s.sock", Hibernate: true}
2037	if a.ServeHTTP(rec, req, svc, noop) {
2038	t.Fatal("service route was handled by the activator")
2039	}
2040	if rec.Body.Len() != 0 {
2041	t.Fatalf("declined routes wrote a response: %q", rec.Body.String())
2042	}
2043	if acquires, _, starts := b.counts(); acquires != 0 || starts != 0 {
2044	t.Fatalf("declined routes touched the backend (acquires=%d starts=%d)", acquires, starts)
2045	}
2046	}
2047
2048	// A scheduled tick must not move lastActive: it is the platform's own clock
2049	// poking the instance, not a visitor. A tick that counted would make any
2050	// schedule tighter than the idle window a permanent keep-awake.
2051	func TestServeScheduledDoesNotStampActivity(t *testing.T) {
2052	const host = "tick.bab.oncarlos.com"
2053	a, _, clk, _ := testActivator(t, time.Hour)
2054	rt := testRoute(host)
2055	arrangeAwake(t, a, rt) // a real beginRequest round trip, then release
2056
2057	a.mu.Lock()
2058	before := a.entries[rt.Host].lastActive
2059	a.mu.Unlock()
2060
2061	clk.advance(time.Minute)
2062
2063	rec := httptest.NewRecorder()
2064	req := httptest.NewRequest("POST", "http://carlos/jobs/sweep", nil)
2065	if !a.ServeScheduled(rec, req, rt, okHandler()) {
2066	t.Fatal("ServeScheduled returned false for a managed route")
2067	}
2068
2069	a.mu.Lock()
2070	after := a.entries[rt.Host].lastActive
2071	inflight := a.entries[rt.Host].inflight
2072	a.mu.Unlock()
2073
2074	if !after.Equal(before) {
2075	t.Errorf("lastActive moved %v -> %v; a scheduled tick must not count as activity", before, after)
2076	}
2077	if inflight != 0 {
2078	t.Errorf("inflight = %d after delivery, want 0 (the hold is released even though the stamp is not written)", inflight)
2079	}
2080	}
2081
2082	// The other half of the same coin, asserted against the first so the two
2083	// paths can never quietly become one again — which is the bug this fixes.
2084	func TestServeHTTPStillStampsActivity(t *testing.T) {
2085	const host = "visitor.bab.oncarlos.com"
2086	a, _, clk, _ := testActivator(t, time.Hour)
2087	rt := testRoute(host)
2088	arrangeAwake(t, a, rt)
2089
2090	a.mu.Lock()
2091	before := a.entries[rt.Host].lastActive
2092	a.mu.Unlock()
2093
2094	clk.advance(time.Minute)
2095
2096	rec := httptest.NewRecorder()
2097	req := httptest.NewRequest("GET", "http://carlos/", nil)
2098	if !a.ServeHTTP(rec, req, rt, okHandler()) {
2099	t.Fatal("ServeHTTP returned false for a managed route")
2100	}
2101
2102	a.mu.Lock()
2103	after := a.entries[rt.Host].lastActive
2104	a.mu.Unlock()
2105
2106	if !after.After(before) {
2107	t.Errorf("lastActive did not move on a visitor request (%v -> %v)", before, after)
2108	}
2109	}
2110
2111	// A tick is a wake source and must stay one: an instance asleep when its
2112	// timer comes due has to be woken by the delivery itself.
2113	func TestServeScheduledWakesASleepingInstance(t *testing.T) {
2114	const host = "asleep.bab.oncarlos.com"
2115	a, _, _, _ := testActivator(t, time.Hour) // no entries yet: the host is asleep
2116	rt := testRoute(host)
2117
2118	rec := httptest.NewRecorder()
2119	req := httptest.NewRequest("POST", "http://carlos/jobs/sweep", nil)
2120	if !a.ServeScheduled(rec, req, rt, okHandler()) {
2121	t.Fatal("ServeScheduled returned false for a managed route")
2122	}
2123	if rec.Code != http.StatusOK {
2124	t.Fatalf("status %d, want 200 — the tick must wake the instance, not be refused", rec.Code)
2125	}
2126	a.mu.Lock()
2127	e := a.entries[rt.Host]
2128	a.mu.Unlock()
2129	if e == nil || e.state != actAwake {
2130	t.Fatal("instance is not awake after a scheduled delivery")
2131	}
2132	}
2133
2134	// A tick never wakes a route the box has stopped serving (G12): the move
2135	// cut sets "moving", and a tick that woke the host there would restore it
2136	// and let it write behind the move's final mirror round. This gate is the
2137	// half that survives an agent restart, when the in-memory fence is gone.
2138	func TestServeScheduledRefusesARouteThatIsNotReady(t *testing.T) {
2139	const host = "moving.bab.oncarlos.com"
2140	a, _, _, _ := testActivator(t, time.Hour)
2141	rt := testRoute(host)
2142	rt.Status = "moving"
2143
2144	rec := httptest.NewRecorder()
2145	req := httptest.NewRequest("POST", "http://carlos/jobs/sweep", nil)
2146	if !a.ServeScheduled(rec, req, rt, okHandler()) {
2147	t.Fatal("ServeScheduled returned false: the runner would deliver to the socket itself")
2148	}
2149	if rec.Code != http.StatusServiceUnavailable {
2150	t.Fatalf("status %d, want 503", rec.Code)
2151	}
2152	a.mu.Lock()
2153	e := a.entries[rt.Host]
2154	a.mu.Unlock()
2155	if e != nil {
2156	t.Fatal("a tick woke a moving route")
2157	}
2158	}
2159
2160	// LiftFence reopens only the fence it names: a stale generation leaves a
2161	// newer fence closed, and the right one lets the next tick wake the host.
2162	func TestLiftFenceReopensOnlyItsOwnGeneration(t *testing.T) {
2163	const host = "fenced.bab.oncarlos.com"
2164	a, _, _, _ := testActivator(t, time.Hour)
2165	rt := testRoute(host)
2166	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
2167	defer cancel()
2168	if _, err := a.FenceAndStop(ctx, host, 7); err != nil && !errors.Is(err, ErrStopUnproven) {
2169	t.Fatalf("FenceAndStop = %v", err)
2170	}
2171	if a.LiftFence(host, 6) || a.LiftFence(host, 0) {
2172	t.Fatal("a stale generation lifted the fence")
2173	}
2174	rec := httptest.NewRecorder()
2175	a.ServeScheduled(rec, httptest.NewRequest("POST", "http://carlos/jobs/sweep", nil), rt, okHandler())
2176	if rec.Code != http.StatusServiceUnavailable {
2177	t.Fatalf("fenced tick status %d, want 503", rec.Code)
2178	}
2179	if !a.LiftFence(host, 7) {
2180	t.Fatal("the fence's own generation did not lift it")
2181	}
2182	rec = httptest.NewRecorder()
2183	a.ServeScheduled(rec, httptest.NewRequest("POST", "http://carlos/jobs/sweep", nil), rt, okHandler())
2184	if rec.Code != http.StatusOK {
2185	t.Fatalf("tick after the lift: status %d, want 200", rec.Code)
2186	}
2187	}
2188
2189	// The guard: a host whose next platform tick is nearer than the idle window
2190	// stays resident. Sleeping would buy a few minutes of absence at the price of
2191	// a full checkpoint-and-restore round trip, and then an immediate wake.
2192	func TestSweepKeepsHostAwakeWhenATickIsImminent(t *testing.T) {
2193	const host = "imminent.bab.oncarlos.com"
2194	a, _, clk, reg := testActivator(t, 15*time.Minute)
2195	rt := testRoute(host)
2196	arrangeAwake(t, a, rt)
2197
2198	// Arm 21 minutes out, then advance the clock 16 minutes: at sweep time
2199	// the tick is 5 minutes in the FUTURE — genuinely inside the window, not
2200	// merely overdue (that boundary is TestSweepKeepsHostAwakeForAnOverdueTick's
2201	// job). A guard that only checked next <= now would wrongly hibernate here.
2202	armTimer(t, reg, host, clk.now().Add(21*time.Minute))
2203	clk.advance(16 * time.Minute) // lastActive is now well past the window
2204
2205	a.sweepOnce()
2206
2207	a.mu.Lock()
2208	e := a.entries[rt.Host]
2209	a.mu.Unlock()
2210	if e == nil || e.state != actAwake {
2211	t.Fatal("host hibernated with a tick due inside the idle window")
2212	}
2213	}
2214
2215	// Beyond the window, the sleep is worth taking.
2216	func TestSweepHibernatesWhenTheNextTickIsFarOff(t *testing.T) {
2217	t.Parallel()
2218	const host = "faroff.bab.oncarlos.com"
2219	a, _, clk, reg := testActivator(t, 15*time.Minute)
2220	rt := testRoute(host)
2221	arrangeAwake(t, a, rt)
2222
2223	// Next tick in an hour: well beyond the window.
2224	armTimer(t, reg, host, clk.now().Add(time.Hour))
2225	clk.advance(16 * time.Minute)
2226
2227	a.sweepOnce()
2228
2229	waitFor(t, "the host to hibernate with its next tick an hour away", func() bool {
2230	a.mu.Lock()
2231	defer a.mu.Unlock()
2232	return a.entries[rt.Host] == nil
2233	})
2234	}
2235
2236	// A host with no timer rows is untouched by the guard — the overwhelming
2237	// majority of instances, and the case that must not regress.
2238	func TestSweepHibernatesAHostWithNoTimers(t *testing.T) {
2239	t.Parallel()
2240	const host = "untimed.bab.oncarlos.com"
2241	a, _, clk, _ := testActivator(t, 15*time.Minute) // no timer rows at all
2242	rt := testRoute(host)
2243	arrangeAwake(t, a, rt)
2244	clk.advance(16 * time.Minute)
2245
2246	a.sweepOnce()
2247
2248	waitFor(t, "a host with no timers to hibernate as it always did", func() bool {
2249	a.mu.Lock()
2250	defer a.mu.Unlock()
2251	return a.entries[rt.Host] == nil
2252	})
2253	}
2254
2255	// An overdue row (next_at already in the past — the runner has not got to it
2256	// yet) is the most imminent tick there is, so it holds the host too.
2257	func TestSweepKeepsHostAwakeForAnOverdueTick(t *testing.T) {
2258	const host = "overdue.bab.oncarlos.com"
2259	a, _, clk, reg := testActivator(t, 15*time.Minute)
2260	rt := testRoute(host)
2261	arrangeAwake(t, a, rt)
2262
2263	// next_at already behind us: the runner will fire it within 15s.
2264	armTimer(t, reg, host, clk.now().Add(-time.Minute))
2265	clk.advance(16 * time.Minute)
2266
2267	a.sweepOnce()
2268
2269	a.mu.Lock()
2270	e := a.entries[rt.Host]
2271	a.mu.Unlock()
2272	if e == nil || e.state != actAwake {
2273	t.Fatal("host hibernated with an overdue tick pending")
2274	}
2275	}
2276
2277	// A host held awake by the guard never reaches a "next wake" to converge on
2278	// — the same gap the always-awake sidecar case closes. The guard's continue
2279	// must fall through to the convergence check, not skip it, or a `carlos
2280	// ship` behind a tick-held host is served stale indefinitely.
2281	func TestSweepHeldForTickStillConverges(t *testing.T) {
2282	t.Parallel()
2283	const host = "tickheld.bab.oncarlos.com"
2284	a, b, clk, reg := testActivator(t, 15*time.Minute)
2285	var ver atomic.Value // the live/<host> symlink target; swapped to model a ship
2286	ver.Store("v1")
2287	a.liveVer = func(string) (string, error) { return ver.Load().(string), nil }
2288	b.execVerFn = func() string { return ver.Load().(string) }
2289
2290	rt := testRoute(host)
2291	arrangeAwake(t, a, rt)
2292
2293	// A tight schedule: next tick always inside the window, so the guard
2294	// holds the host awake on every sweep from here on (an overdue row is
2295	// still "inside the window" — more so, if anything).
2296	armTimer(t, reg, host, clk.now().Add(5*time.Minute))
2297	clk.advance(16 * time.Minute) // idle timeout reached
2298
2299	a.sweepOnce()
2300	a.mu.Lock()
2301	e := a.entries[rt.Host]
2302	a.mu.Unlock()
2303	if e == nil || e.state != actAwake {
2304	t.Fatal("host hibernated despite an imminent tick — guard not exercised")
2305	}
2306
2307	// Ship: repoint the symlink while the guard still holds the host awake.
2308	ver.Store("v2")
2309	a.sweepOnce() // starts the grace clock only
2310	if a.entryCount() != 1 {
2311	t.Fatal("cycled inside the grace window")
2312	}
2313
2314	// Past the grace window: the guard still holds (the tick is now even
2315	// more overdue), but convergence must still fire — that's the fix.
2316	clk.advance(ConvergeGrace + time.Second)
2317	a.sweepOnce()
2318	waitFor(t, "a tick-held host to still converge onto the new build", func() bool {
2319	return a.entryCount() == 0
2320	})
2321	if !b.lastProc().isStopped() {
2322	t.Fatal("stale instance's process was not stopped")
2323	}
2324	}
2325
2326	// A reload that finds an awake host no longer hibernate-flagged retires its
2327	// child — stopped, final-checkpointed (replication closed), entry forgotten —
2328	// the edge-process half of flipping hibernation off. And once the flag is
2329	// off, a recorded next_due no longer wakes it: the due index only acts on
2330	// hibernating hosts, so a systemd-managed instance can never be spawned
2331	// against its own unit.
2332	func TestActivatorReloadRetiresUnflaggedHost(t *testing.T) {
2333	t.Parallel()
2334	a, b, clk, reg := testActivator(t, time.Hour)
2335	route := testRoute("flip.example.com")
2336	if err := reg.Upsert(route); err != nil {
2337	t.Fatal(err)
2338	}
2339	rt, err := newTestRouter(reg, a)
2340	if err != nil {
2341	t.Fatalf("new router: %v", err)
2342	}
2343
2344	release, err := a.beginRequest(context.Background(), route)
2345	if err != nil {
2346	t.Fatalf("beginRequest: %v", err)
2347	}
2348	release()
2349	// The retire's sleep re-records next_due AFTER the flag is cleared; an
2350	// already-past value must stay inert under the flag filter.
2351	b.setDue(clk.now().Add(-5 * time.Second).Unix())
2352
2353	if err := reg.SetHibernate("flip.example.com", false); err != nil {
2354	t.Fatal(err)
2355	}
2356	if err := rt.reload(); err != nil {
2357	t.Fatalf("reload: %v", err)
2358	}
2359	waitFor(t, "retire", func() bool { return a.entryCount() == 0 })
2360	if !b.lastProc().isStopped() {
2361	t.Fatal("retired child was not stopped")
2362	}
2363	if !b.lastRep().isClosed() {
2364	t.Fatal("retired child's replication was not closed (no final checkpoint/lease release)")
2365	}
2366
2367	// The due sweeper must not resurrect a host that's back under systemd.
2368	a.sweepOnce()
2369	if _, _, starts := b.counts(); starts != 1 {
2370	t.Fatalf("starts=%d after retire, want 1 (no due wake for a systemd-managed host)", starts)
2371	}
2372	}
2373
2374	// A retire that lands mid-wake can't stop the child yet — it finishes the job
2375	// once the wake resolves, so a flip-off during the small wake window still
2376	// releases cleanly.
2377	func TestActivatorRetireDuringWake(t *testing.T) {
2378	t.Parallel()
2379	a, b, clk, reg := testActivator(t, time.Hour)
2380	route := testRoute("mid.example.com")
2381	if err := reg.Upsert(route); err != nil {
2382	t.Fatal(err)
2383	}
2384	rt, err := newTestRouter(reg, a)
2385	if err != nil {
2386	t.Fatalf("new router: %v", err)
2387	}
2388
2389	// Start a request-less (due-time) wake and hold it in the waking state.
2390	b.startGate = make(chan struct{})
2391	if err := reg.SetNextDue("mid.example.com", clk.now().Add(-time.Second).Unix()); err != nil {
2392	t.Fatal(err)
2393	}
2394	a.sweepOnce()
2395	waitFor(t, "the wake to begin", func() bool { _, _, s := b.counts(); return s == 1 })
2396
2397	// Flip off mid-wake: the reload's retire must defer, not deadlock.
2398	if err := reg.SetHibernate("mid.example.com", false); err != nil {
2399	t.Fatal(err)
2400	}
2401	if err := rt.reload(); err != nil {
2402	t.Fatalf("reload: %v", err)
2403	}
2404	close(b.startGate)
2405
2406	waitFor(t, "post-wake retire", func() bool { return a.entryCount() == 0 })
2407	if !b.lastProc().isStopped() {
2408	t.Fatal("child woken mid-retire was not stopped")
2409	}
2410	if !b.lastRep().isClosed() {
2411	t.Fatal("child woken mid-retire left replication open (lease never released)")
2412	}
2413	}
2414
2415	// With hibernation on, sibling services (kind "service") are untouched: the
2416	// edge proxies them directly, no wake, no activity tracking.
2417	func TestActivatorServiceRoutesBypass(t *testing.T) {
2418	t.Parallel()
2419	requireUnixSockets(t)
2420	sock := tmpSock(t)
2421	ln, err := net.Listen("unix", sock)
2422	if err != nil {
2423	t.Fatalf("listen unix: %v", err)
2424	}
2425	backend := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
2426	io.WriteString(w, "service ok")
2427	})}
2428	go backend.Serve(ln)
2429	t.Cleanup(func() { backend.Close() })
2430
2431	a, b, _, _ := testActivator(t, time.Minute)
2432	reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db"))
2433	if err != nil {
2434	t.Fatalf("open registry: %v", err)
2435	}
2436	if err := reg.Upsert(registry.Route{Host: "svc.example.com", Kind: "service", Socket: sock}); err != nil {
2437	t.Fatalf("upsert: %v", err)
2438	}
2439	rt, err := newTestRouter(reg, a)
2440	if err != nil {
2441	t.Fatalf("new router: %v", err)
2442	}
2443	front := httptest.NewServer(rt)
2444	t.Cleanup(front.Close)
2445
2446	req, _ := http.NewRequest("GET", front.URL+"/", nil)
2447	req.Host = "svc.example.com"
2448	resp, err := http.DefaultClient.Do(req)
2449	if err != nil {
2450	t.Fatalf("request: %v", err)
2451	}
2452	body, _ := io.ReadAll(resp.Body)
2453	resp.Body.Close()
2454	if resp.StatusCode != 200 || string(body) != "service ok" {
2455	t.Fatalf("got %d %q, want 200 service ok", resp.StatusCode, body)
2456	}
2457	if acquires, _, _ := b.counts(); acquires != 0 {
2458	t.Fatal("a service route touched the activator")
2459	}
2460	}
2461
2462	// Shutdown checkpoints and stops every awake child before the edge exits.
2463	func TestActivatorShutdownStopsChildren(t *testing.T) {
2464	a, b, _, _ := testActivator(t, time.Hour)
2465	for _, h := range []string{"one.example.com", "two.example.com"} {
2466	release, err := a.beginRequest(context.Background(), testRoute(h))
2467	if err != nil {
2468	t.Fatalf("beginRequest %s: %v", h, err)
2469	}
2470	release()
2471	}
2472
2473	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
2474	defer cancel()
2475	a.Shutdown(ctx)
2476
2477	if a.entryCount() != 0 {
2478	t.Fatal("shutdown left entries behind")
2479	}
2480	b.mu.Lock()
2481	defer b.mu.Unlock()
2482	for i, p := range b.procs {
2483	if !p.stopped {
2484	t.Fatalf("proc %d not stopped on shutdown", i)
2485	}
2486	}
2487	for i, r := range b.reps {
2488	if !r.closed {
2489	t.Fatalf("replication %d not closed on shutdown", i)
2490	}
2491	}
2492	}
2493
2494	// Bounce stops a running instance so its next wake respawns with fresh
2495	// env (config delivery). It must be a no-op for unknown hosts and for
2496	// entries not currently awake — never a spurious wake, never a panic.
2497	func TestBounceStopsAwakeInstanceOnly(t *testing.T) {
2498	a := newTestServiceAwake(t, "b.example.com")
2499	if !a.Bounce("b.example.com") {
2500	t.Fatal("Bounce(awake host) = false, want true")
2501	}
2502	a.mu.Lock()
2503	st := a.entries["b.example.com"].state
2504	a.mu.Unlock()
2505	if st != actStopping {
2506	t.Fatalf("state after Bounce = %v, want actStopping", st)
2507	}
2508	if a.Bounce("b.example.com") {
2509	t.Fatal("Bounce(already stopping) = true, want false")
2510	}
2511	if a.Bounce("never-woken.example.com") {
2512	t.Fatal("Bounce(unknown host) = true, want false")
2513	}
2514	}
2515
2516	// Owns is the external-socket advisory's predicate (issue #17): a live entry
2517	// in ANY state means whatever serves the host's socket is this activator's own
2518	// child, so a host draining its SIGTERM must still read as owned — otherwise
2519	// the advisory calls the activator's own children strays.
2520	func TestOwnsCoversEntryLifetimeAnyState(t *testing.T) {
2521	t.Parallel()
2522	a, b, clk, _ := testActivator(t, 10*time.Second)
2523	host := "own.example.com"
2524	if a.Owns(host) {
2525	t.Fatal("Owns(host) = true before any request, want false")
2526	}
2527
2528	release, err := a.beginRequest(context.Background(), testRoute(host))
2529	if err != nil {
2530	t.Fatalf("beginRequest: %v", err)
2531	}
2532	release()
2533	if !a.entryAwake(host) || !a.Owns(host) {
2534	t.Fatalf("awake: entryAwake=%v Owns=%v, want both true", a.entryAwake(host), a.Owns(host))
2535	}
2536
2537	// Hold the stop open so the entry sits in stopping.
2538	gate := make(chan struct{})
2539	p := b.lastProc()
2540	p.mu.Lock()
2541	p.stopGate = gate
2542	p.mu.Unlock()
2543	clk.advance(time.Minute)
2544	a.sweepOnce()
2545	if a.entryAwake(host) {
2546	t.Fatal("entry still awake after the idle sweep, want stopping")
2547	}
2548	if !a.Owns(host) {
2549	t.Error("Owns(stopping host) = false, want true")
2550	}
2551	close(gate)
2552
2553	waitFor(t, "hibernation", func() bool { return a.entryCount() == 0 })
2554	if a.Owns(host) {
2555	t.Error("Owns(host) = true after the entry was retired, want false")
2556	}
2557	if a.Owns("never-woken.example.com") {
2558	t.Error("Owns(unknown host) = true, want false")
2559	}
2560	}
2561
2562	// --- due-time index --------------------------------------------------------------
2563
2564	// entryAwake reports whether host has a live entry in the awake state — how the
2565	// due tests observe an async due-time wake resolving.
2566	// entryRespawning: whether a converge respawn currently owns this entry's
2567	// child — the interlock the sweeper honours.
2568	func (a *Service) entryRespawning(host string) bool {
2569	a.mu.Lock()
2570	defer a.mu.Unlock()
2571	e := a.entries[host]
2572	return e != nil && e.workerRespawning
2573	}
2574
2575	// entrySpawnVersion: the build the host's live child was exec'd from, "" if
2576	// there is no entry — how a test sees that a respawn actually landed on the
2577	// new binary rather than merely happening.
2578	func (a *Service) entrySpawnVersion(host string) string {
2579	a.mu.Lock()
2580	defer a.mu.Unlock()
2581	if e := a.entries[host]; e != nil {
2582	return e.spawnVersion
2583	}
2584	return ""
2585	}
2586
2587	func (a *Service) entryAwake(host string) bool {
2588	a.mu.Lock()
2589	defer a.mu.Unlock()
2590	e := a.entries[host]
2591	return e != nil && e.state == actAwake
2592	}
2593
2594	// The full self-maintaining cycle: sleep records the instance's next due time
2595	// in the registry; the sweeper wakes the host (no request) when it arrives; the
2596	// following sleep re-records — and once nothing is scheduled, no more due wakes.
2597	func TestActivatorDueTimeWakeCycle(t *testing.T) {
2598	t.Parallel()
2599	a, b, clk, reg := testActivator(t, 10*time.Second)
2600	route := testRoute("due.example.com")
2601	if err := reg.Upsert(route); err != nil {
2602	t.Fatalf("upsert: %v", err)
2603	}
2604
2605	// Wake by request; the instance has a delivery scheduled 30s out.
2606	dueAt := clk.now().Add(30 * time.Second).Unix()
2607	b.setDue(dueAt)
2608	release, err := a.beginRequest(context.Background(), route)
2609	if err != nil {
2610	t.Fatalf("beginRequest: %v", err)
2611	}
2612	release()
2613
2614	// Idle out: the sleep probes the instance and records its due time.
2615	clk.advance(11 * time.Second)
2616	a.sweepOnce()
2617	waitFor(t, "hibernation", func() bool { return a.entryCount() == 0 })
2618	if rt, err := reg.Get(route.Host); err != nil || rt == nil || rt.NextDue != dueAt {
2619	t.Fatalf("registry next_due = %+v, %v; want %d", rt, err, dueAt)
2620	}
2621
2622	// Before the due time: sweeps leave it asleep.
2623	a.sweepOnce()
2624	if _, _, starts := b.counts(); starts != 1 {
2625	t.Fatalf("woke before its due time (starts=%d)", starts)
2626	}
2627
2628	// At the due time: the sweeper wakes it with no client request.
2629	clk.advance(20 * time.Second) // now = wake+31s ≥ due
2630	a.sweepOnce()
2631	waitFor(t, "due-time wake", func() bool { return a.entryAwake(route.Host) })
2632	if _, _, starts := b.counts(); starts != 2 {
2633	t.Fatalf("starts=%d after due wake, want 2", starts)
2634	}
2635
2636	// The woken instance stays up for the normal idle window, then sleeps and
2637	// re-records — nothing scheduled now, so the index clears and stays quiet.
2638	b.setDue(0)
2639	clk.advance(11 * time.Second)
2640	a.sweepOnce()
2641	waitFor(t, "re-hibernation", func() bool { return a.entryCount() == 0 })
2642	if rt, _ := reg.Get(route.Host); rt.NextDue != 0 {
2643	t.Fatalf("next_due = %d after empty-schedule sleep, want 0", rt.NextDue)
2644	}
2645	clk.advance(time.Hour)
2646	a.sweepOnce()
2647	if _, _, starts := b.counts(); starts != 2 {
2648	t.Fatalf("starts=%d with nothing scheduled, want 2 (no phantom due wake)", starts)
2649	}
2650	}
2651
2652	// A due time already in the past at sleep time (a delivery that raced the stop)
2653	// is recorded as-is: the sweeper wakes the host on its next tick — once — and
2654	// there is no ping-pong, because a woken instance can't sleep again until a
2655	// full idle window passes.
2656	func TestActivatorDuePastWakesOnceNoPingPong(t *testing.T) {
2657	t.Parallel()
2658	a, b, clk, reg := testActivator(t, 10*time.Second)
2659	route := testRoute("late.example.com")
2660	if err := reg.Upsert(route); err != nil {
2661	t.Fatalf("upsert: %v", err)
2662	}
2663	b.setDue(clk.now().Add(-5 * time.Second).Unix()) // overdue before it even sleeps
2664
2665	release, err := a.beginRequest(context.Background(), route)
2666	if err != nil {
2667	t.Fatalf("beginRequest: %v", err)
2668	}
2669	release()
2670	clk.advance(11 * time.Second)
2671	a.sweepOnce()
2672	waitFor(t, "hibernation", func() bool { return a.entryCount() == 0 })
2673
2674	// Next tick: woken for the overdue delivery.
2675	a.sweepOnce()
2676	waitFor(t, "overdue wake", func() bool { return a.entryAwake(route.Host) })
2677
2678	// No thrash: with the wake fresh (clock unmoved), repeated sweeps neither
2679	// re-sleep it nor double-wake it, even though next_due is still in the past.
2680	for i := 0; i < 5; i++ {
2681	a.sweepOnce()
2682	}
2683	if a.entryCount() != 1 || !a.entryAwake(route.Host) {
2684	t.Fatal("due-woken instance did not stay awake through its idle window")
2685	}
2686	if _, _, starts := b.counts(); starts != 2 {
2687	t.Fatalf("starts=%d, want 2 (request wake + one due wake)", starts)
2688	}
2689	}
2690
2691	// A failed next-due probe (dead socket, missing admin token, …) records 0 and
2692	// the sleep proceeds — the instance degrades to "sleeps until a request",
2693	// never a crash and never a stuck stop.
2694	func TestActivatorDueProbeFailureRecordsZero(t *testing.T) {
2695	t.Parallel()
2696	a, b, clk, reg := testActivator(t, 10*time.Second)
2697	route := testRoute("deaf.example.com")
2698	if err := reg.Upsert(route); err != nil {
2699	t.Fatalf("upsert: %v", err)
2700	}
2701	b.dueErr = errors.New("dial unix: no such file")
2702
2703	release, err := a.beginRequest(context.Background(), route)
2704	if err != nil {
2705	t.Fatalf("beginRequest: %v", err)
2706	}
2707	release()
2708	clk.advance(11 * time.Second)
2709	a.sweepOnce()
2710	waitFor(t, "hibernation despite probe failure", func() bool { return a.entryCount() == 0 })
2711
2712	if rt, _ := reg.Get(route.Host); rt.NextDue != 0 {
2713	t.Fatalf("next_due = %d after failed probe, want 0", rt.NextDue)
2714	}
2715	clk.advance(time.Hour)
2716	a.sweepOnce()
2717	if _, _, starts := b.counts(); starts != 1 {
2718	t.Fatalf("starts=%d, want 1 (a failed probe must not invent a wake)", starts)
2719	}
2720	}
2721
2722	// The index lives in the registry, so a due time recorded before an edge
2723	// restart still fires: a brand-new activator over the same registry — no
2724	// in-memory state carried over — wakes the host when its time arrives.
2725	func TestActivatorDueSurvivesRestart(t *testing.T) {
2726	t.Parallel()
2727	a1, b1, clk, reg := testActivator(t, 10*time.Second)
2728	route := testRoute("reborn.example.com")
2729	if err := reg.Upsert(route); err != nil {
2730	t.Fatalf("upsert: %v", err)
2731	}
2732	b1.setDue(clk.now().Add(30 * time.Second).Unix())
2733
2734	release, err := a1.beginRequest(context.Background(), route)
2735	if err != nil {
2736	t.Fatalf("beginRequest: %v", err)
2737	}
2738	release()
2739	clk.advance(11 * time.Second)
2740	a1.sweepOnce()
2741	waitFor(t, "hibernation", func() bool { return a1.entryCount() == 0 })
2742
2743	// "Restart": a fresh activator, same registry.
2744	b2 := &fakeBackend{}
2745	a2 := New(Config{Bucket: "test-bucket", Idle: 10 * time.Second}, b2, reg)
2746	a2.now = clk.now
2747
2748	clk.advance(25 * time.Second) // past the due time
2749	a2.sweepOnce()
2750	waitFor(t, "post-restart due wake", func() bool { return a2.entryAwake(route.Host) })
2751	if _, _, starts := b2.counts(); starts != 1 {
2752	t.Fatalf("starts=%d on the new activator, want 1", starts)
2753	}
2754	}
2755
2756	// --- sidecar co-spawn ----------------------------------------------------------
2757
2758	// indexOf: position of ev in the backend's order log (-1 = absent).
2759	func indexOf(events []string, ev string) int {
2760	for i, e := range events {
2761	if e == ev {
2762	return i
2763	}
2764	}
2765	return -1
2766	}
2767
2768	// A host with a sidecar staged gets it co-spawned on wake, strictly AFTER the
2769	// instance process is up — the sidecar dials the instance, so spawning it
2770	// first would just burn its reconnect backoff. (Every other test in this file
2771	// runs with sidecarStaged=false and asserts nothing sidecar-ish happens, which
2772	// is the "host without a sidecar behaves exactly as today" half.)
2773	func TestActivatorCoSpawnsSidecarAfterInstance(t *testing.T) {
2774	a, b, _, _ := testActivator(t, time.Minute)
2775	b.sidecarStaged = true
2776
2777	release, err := a.beginRequest(context.Background(), testRoute("bots.example.com"))
2778	if err != nil {
2779	t.Fatalf("beginRequest: %v", err)
2780	}
2781	release()
2782
2783	if got := b.sidecarStartCount(); got != 1 {
2784	t.Fatalf("sidecar starts = %d, want 1", got)
2785	}
2786	ev := b.events()
2787	if i, s := indexOf(ev, "start:instance"), indexOf(ev, "start:sidecar"); i == -1 || s == -1 || s < i {
2788	t.Fatalf("sidecar did not start strictly after the instance: %v", ev)
2789	}
2790	}
2791
2792	// A sidecar that fails to spawn must not fail the wake — members > bots: the
2793	// host serves without its sidecar, and the sweeper keeps retrying with backoff
2794	// until a spawn sticks.
2795	func TestActivatorSidecarFailureDoesNotFailWake(t *testing.T) {
2796	t.Parallel()
2797	a, b, clk, _ := testActivator(t, time.Hour)
2798	b.sidecarStaged = true
2799	b.setSidecarErr(errors.New("exec: live symlink gone"))
2800	route := testRoute("botless.example.com")
2801
2802	release, err := a.beginRequest(context.Background(), route)
2803	if err != nil {
2804	t.Fatalf("beginRequest with a failing sidecar: %v", err)
2805	}
2806	release()
2807	if !a.entryAwake(route.Host) {
2808	t.Fatal("instance not awake after a sidecar-only failure")
2809	}
2810	if got := b.sidecarStartCount(); got != 1 {
2811	t.Fatalf("sidecar attempts = %d, want 1 (the wake's)", got)
2812	}
2813
2814	// The sweeper retries: attempt 2 immediately (first retry), then gated by
2815	// the doubling backoff; once the fault clears, a retry lands the sidecar.
2816	// The restart call itself now runs off a.mu (design §6.7), so a fresh
2817	// attempt lands asynchronously. waitFor gives it a moment rather than
2818	// asserting on sweepOnce's own (now much shorter) return.
2819	a.sweepOnce()
2820	waitFor(t, "sidecar restart attempt 2 to land", func() bool { return b.sidecarStartCount() == 2 && a.sidecarSettled(route.Host) })
2821	if got := b.sidecarStartCount(); got != 2 {
2822	t.Fatalf("sidecar attempts after first sweep = %d, want 2", got)
2823	}
2824	a.sweepOnce() // still inside the 2s gate — no attempt
2825	if got := b.sidecarStartCount(); got != 2 {
2826	t.Fatalf("backoff gate ignored: attempts = %d, want 2", got)
2827	}
2828	b.setSidecarErr(nil)
2829	clk.advance(3 * time.Second)
2830	a.sweepOnce()
2831	waitFor(t, "sidecar restart attempt 3 to land", func() bool { return b.sidecarStartCount() == 3 && a.sidecarSettled(route.Host) })
2832	if got := b.sidecarStartCount(); got != 3 {
2833	t.Fatalf("sidecar attempts after fault cleared = %d, want 3", got)
2834	}
2835	if b.lastSidecar().isStopped() || b.lastSidecar().Exited() {
2836	t.Fatal("recovered sidecar isn't running")
2837	}
2838	}
2839
2840	// Co-stop ordering at sleep: the sidecar is stopped FIRST — before the
2841	// next-due probe (so bots can't shift the schedule being recorded) and before
2842	// the instance's own stop (so the sidecar never outlives its instance).
2843	func TestActivatorCoStopOrdering(t *testing.T) {
2844	t.Parallel()
2845	a, b, clk, _ := testActivator(t, 10*time.Second)
2846	b.sidecarStaged = true
2847
2848	release, err := a.beginRequest(context.Background(), testRoute("bots.example.com"))
2849	if err != nil {
2850	t.Fatalf("beginRequest: %v", err)
2851	}
2852	release()
2853
2854	clk.advance(11 * time.Second)
2855	a.sweepOnce()
2856	waitFor(t, "hibernation", func() bool { return a.entryCount() == 0 })
2857
2858	if !b.lastSidecar().isStopped() {
2859	t.Fatal("sidecar not stopped at sleep")
2860	}
2861	ev := b.events()
2862	sc, due, inst := indexOf(ev, "stop:sidecar"), indexOf(ev, "nextDue"), indexOf(ev, "stop:instance")
2863	if sc == -1 || due == -1 || inst == -1 || !(sc < due && due < inst) {
2864	t.Fatalf("co-stop order wrong: %v (want stop:sidecar < nextDue < stop:instance)", ev)
2865	}
2866	}
2867
2868	// Orphan prevention: whatever takes the instance down — here a crashed child,
2869	// but a lost lease and a retire ride the identical stop path — takes the
2870	// sidecar with it. An orphaned sidecar would reconnect-loop against the edge
2871	// forever (503s at best; a wake source if the marker rule ever broke).
2872	func TestActivatorInstanceDeathStopsSidecar(t *testing.T) {
2873	t.Parallel()
2874	a, b, _, _ := testActivator(t, time.Hour)
2875	b.sidecarStaged = true
2876
2877	release, err := a.beginRequest(context.Background(), testRoute("bots.example.com"))
2878	if err != nil {
2879	t.Fatalf("beginRequest: %v", err)
2880	}
2881	release()
2882
2883	p := b.lastProc()
2884	p.mu.Lock()
2885	p.dead = true
2886	p.mu.Unlock()
2887	a.sweepOnce()
2888	waitFor(t, "stand-down", func() bool { return a.entryCount() == 0 })
2889	if !b.lastSidecar().isStopped() {
2890	t.Fatal("instance died but its sidecar was left running (orphan)")
2891	}
2892	}
2893
2894	// Sidecar death while the instance stays awake: restarted with doubling,
2895	// capped backoff — Restart=always semantics, sweep-driven — and a run that
2896	// held for sidecarHealthyRun earns a fresh backoff.
2897	func TestActivatorSidecarRestartBackoff(t *testing.T) {
2898	t.Parallel()
2899	a, b, clk, _ := testActivator(t, time.Hour)
2900	b.sidecarStaged = true
2901
2902	release, err := a.beginRequest(context.Background(), testRoute("bots.example.com"))
2903	if err != nil {
2904	t.Fatalf("beginRequest: %v", err)
2905	}
2906	release()
2907	kill := func() {
2908	sc := b.lastSidecar()
2909	sc.mu.Lock()
2910	sc.dead = true
2911	sc.mu.Unlock()
2912	}
2913
2914	// The restart call itself runs off a.mu (design §6.7): each waitFor below
2915	// gives the previous attempt's goroutine a moment to land, both the
2916	// count and the entry's sidecarStarting flag, before the next kill/sweep,
2917	// so a slow scheduler can't make this sequence's next sweepOnce see the
2918	// prior attempt still "in flight" and skip its own.
2919	host := "bots.example.com"
2920
2921	// T0: first crash — restarted on the next sweep (attempt 2), gate 2s.
2922	kill()
2923	a.sweepOnce()
2924	waitFor(t, "restart 2 to land", func() bool { return b.sidecarStartCount() == 2 && a.sidecarSettled(host) })
2925	if got := b.sidecarStartCount(); got != 2 {
2926	t.Fatalf("first restart: attempts = %d, want 2", got)
2927	}
2928	// Crash-loop: still inside the 2s gate — no restart.
2929	kill()
2930	a.sweepOnce()
2931	if got := b.sidecarStartCount(); got != 2 {
2932	t.Fatalf("gate ignored: attempts = %d, want 2", got)
2933	}
2934	// T3: past the 2s gate — attempt 3, gate doubles to 4s.
2935	clk.advance(3 * time.Second)
2936	a.sweepOnce()
2937	waitFor(t, "restart 3 to land", func() bool { return b.sidecarStartCount() == 3 && a.sidecarSettled(host) })
2938	if got := b.sidecarStartCount(); got != 3 {
2939	t.Fatalf("second restart: attempts = %d, want 3", got)
2940	}
2941	// T6: 3s later is inside the 4s gate — the doubling is real.
2942	kill()
2943	clk.advance(3 * time.Second)
2944	a.sweepOnce()
2945	if got := b.sidecarStartCount(); got != 3 {
2946	t.Fatalf("doubled gate ignored: attempts = %d, want 3", got)
2947	}
2948	// T8: past it — attempt 4 (gate now 8s).
2949	clk.advance(2 * time.Second)
2950	a.sweepOnce()
2951	waitFor(t, "restart 4 to land", func() bool { return b.sidecarStartCount() == 4 && a.sidecarSettled(host) })
2952	if got := b.sidecarStartCount(); got != 4 {
2953	t.Fatalf("third restart: attempts = %d, want 4", got)
2954	}
2955	// A healthy run (≥ sidecarHealthyRun) resets the ladder: the next crash
2956	// restarts immediately and its gate is back to the 2s base…
2957	clk.advance(2 * time.Minute)
2958	kill()
2959	a.sweepOnce()
2960	waitFor(t, "restart 5 to land", func() bool { return b.sidecarStartCount() == 5 && a.sidecarSettled(host) })
2961	if got := b.sidecarStartCount(); got != 5 {
2962	t.Fatalf("post-healthy restart: attempts = %d, want 5", got)
2963	}
2964	// …which a 3s advance clears (16s — the unreset ladder — would not).
2965	kill()
2966	clk.advance(3 * time.Second)
2967	a.sweepOnce()
2968	waitFor(t, "restart 6 to land", func() bool { return b.sidecarStartCount() == 6 && a.sidecarSettled(host) })
2969	if got := b.sidecarStartCount(); got != 6 {
2970	t.Fatalf("backoff not reset after a healthy run: attempts = %d, want 6", got)
2971	}
2972	}
2973
2974	// The activity question: the co-spawned sidecar dials its instance's public
2975	// URL through this very edge (the box pins its hosted names to loopback), so
2976	// without the marker its keepalives would be member activity and the pair
2977	// could never sleep. Full front-door check: marked requests are served while
2978	// awake — with the marker stripped before the backend — but stamp nothing
2979	// (the host hibernates on schedule beneath them), and once asleep they get
2980	// 503 and wake nothing. An outsider spoofing the marker gets exactly that
2981	// same treatment: it can't wake the host and can't suppress anyone else's
2982	// stamps (stamps are per-request), so the marker is not a DoS lever in either
2983	// direction.
2984	func TestActivatorSidecarRequestsInvisibleToActivity(t *testing.T) {
2985	t.Parallel()
2986	requireUnixSockets(t)
2987	a, b, clk, reg := testActivator(t, 10*time.Second)
2988
2989	// A live backend listener stands in for the child a wake would have
2990	// spawned (the fake backend's Start() spawns nothing), recording whether
2991	// the sidecar marker ever leaks through the proxy.
2992	sock := tmpSock(t)
2993	ln, err := net.Listen("unix", sock)
2994	if err != nil {
2995	t.Fatalf("listen unix: %v", err)
2996	}
2997	var leaked atomic.Bool
2998	srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
2999	if r.Header.Get(SidecarHeader) != "" {
3000	leaked.Store(true)
3001	}
3002	io.WriteString(w, "ok")
3003	})}
3004	go srv.Serve(ln)
3005	t.Cleanup(func() { srv.Close() })
3006
3007	const host = "bots.example.com"
3008	if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", Socket: sock, Hibernate: true}); err != nil {
3009	t.Fatal(err)
3010	}
3011	rt, err := newTestRouter(reg, a)
3012	if err != nil {
3013	t.Fatalf("new router: %v", err)
3014	}
3015	front := httptest.NewServer(rt)
3016	t.Cleanup(front.Close)
3017
3018	markedGet := func() int {
3019	req, _ := http.NewRequest("GET", front.URL+"/", nil)
3020	req.Host = host
3021	req.Header.Set(SidecarHeader, "1")
3022	resp, err := http.DefaultClient.Do(req)
3023	if err != nil {
3024	t.Fatalf("marked request: %v", err)
3025	}
3026	defer resp.Body.Close()
3027	io.Copy(io.Discard, resp.Body)
3028	return resp.StatusCode
3029	}
3030
3031	// A member request wakes the host.
3032	if body, code := proxyGet(t, front.URL, host); code != 200 || body != "ok" {
3033	t.Fatalf("member wake: got %d %q", code, body)
3034	}
3035	// Marked requests are served while awake…
3036	if code := markedGet(); code != 200 {
3037	t.Fatalf("marked request while awake: %d, want 200", code)
3038	}
3039	// …but keep stamping nothing: sidecar traffic riding through the whole
3040	// idle window must not delay the sleep.
3041	clk.advance(6 * time.Second)
3042	if code := markedGet(); code != 200 {
3043	t.Fatalf("marked request mid-window: %d, want 200", code)
3044	}
3045	clk.advance(5 * time.Second) // 11s since the member request
3046	a.sweepOnce()
3047	waitFor(t, "hibernation beneath sidecar traffic", func() bool { return a.entryCount() == 0 })
3048
3049	// Asleep: marked requests are refused and wake NOTHING — this is also the
3050	// outsider-spoof case, and it's harmless.
3051	if code := markedGet(); code != 503 {
3052	t.Fatalf("marked request while asleep: %d, want 503", code)
3053	}
3054	if a.entryCount() != 0 {
3055	t.Fatal("a marked request created an entry (a wake) for a sleeping host")
3056	}
3057	if _, _, starts := b.counts(); starts != 1 {
3058	t.Fatalf("starts = %d after marked requests, want 1 (no sidecar-triggered wake)", starts)
3059	}
3060	// An unmarked request still wakes it, and the marker never reached the
3061	// backend on any request.
3062	if _, code := proxyGet(t, front.URL, host); code != 200 {
3063	t.Fatalf("member re-wake: %d, want 200", code)
3064	}
3065	if _, _, starts := b.counts(); starts != 2 {
3066	t.Fatalf("starts = %d after member re-wake, want 2", starts)
3067	}
3068	if leaked.Load() {
3069	t.Fatal("the sidecar marker leaked through the proxy to the backend")
3070	}
3071	}
3072
3073	// --- standalone sidecars (workers) -------------------------------------------
3074	//
3075	// A worker is a member of the same state machine with the request path removed:
3076	// nothing can dial it, so every wake arrives from the due-time sweep, an
3077	// operator command, or the agent's boot pass, and its only sign of life is
3078	// contact over the agent socket. These tests pin exactly the places where its
3079	// lifecycle DIVERGES from an instance's — the spawn call, the absent next-due
3080	// probe, the two ceilings — and the places where it must not (lease release,
3081	// DB parking, singleflight).
3082
3083	func TestWorkerWakeUsesStartWorkerAndNoProbe(t *testing.T) {
3084	t.Parallel()
3085	a, b, _, _ := testActivator(t, 10*time.Second)
3086	rt := testWorkerRoute("bot.x.internal")
3087	if err := a.WakeWorker(rt); err != nil {
3088	t.Fatalf("WakeWorker: %v", err)
3089	}
3090	if got := b.events(); !contains(got, "startWorker:bot.x.internal") {
3091	t.Fatalf("events %v — a worker must spawn through StartWorker, never Start", got)
3092	}
3093	if b.startCount() != 0 {
3094	t.Fatal("Start (the socket path, with its health probe) must never be called for a worker")
3095	}
3096	}
3097
3098	// The wake hands the child its agent-socket coordinates and the per-wake
3099	// token — the child's only way to reach back, and the activator's only way to
3100	// authorize it. WorkerToken must agree with what the child was told, or every
3101	// call the sidecar makes is a 401.
3102	func TestWorkerWakeThreadsTokenAndSocketEnv(t *testing.T) {
3103	t.Parallel()
3104	a, b, _, _ := testActivator(t, time.Hour)
3105	if err := a.WakeWorker(testWorkerRoute("bot.x.internal")); err != nil {
3106	t.Fatalf("WakeWorker: %v", err)
3107	}
3108	tok := a.WorkerToken("bot.x.internal")
3109	if tok == "" {
3110	t.Fatal("WorkerToken is empty for an awake worker")
3111	}
3112	env := b.lastWorkerEnv()
3113	if env["CARLOS_SIDECAR_TOKEN"] != tok {
3114	t.Fatalf("child got token %q but WorkerToken reports %q — the socket would refuse its own child",
3115	env["CARLOS_SIDECAR_TOKEN"], tok)
3116	}
3117	if env["CARLOS_SIDECAR_HOST"] != "bot.x.internal" || env["CARLOS_SIDECAR_NAME"] != "bot" {
3118	t.Fatalf("child env = %v, want the host and its first label", env)
3119	}
3120	if env["CARLOS_AGENT_SOCKET"] == "" {
3121	t.Fatalf("child env = %v, want the agent socket path (the child dials, never the platform)", env)
3122	}
3123	if a.WorkerToken("nobody.x.internal") != "" {
3124	t.Fatal("WorkerToken must be empty for a host with no live worker")
3125	}
3126	}
3127
3128	func TestWorkerSleepsWhenIdleAndRecordsNextDue(t *testing.T) {
3129	t.Parallel()
3130	a, b, clk, reg := testActivator(t, 10*time.Second)
3131	rt := testWorkerRoute("bot.x.internal")
3132	if err := reg.Upsert(rt); err != nil {
3133	t.Fatalf("upsert: %v", err)
3134	}
3135	if err := a.WakeWorker(rt); err != nil {
3136	t.Fatalf("WakeWorker: %v", err)
3137	}
3138	if err := a.SetWorkerNextDue("bot.x.internal", 4242); err != nil {
3139	t.Fatalf("SetWorkerNextDue: %v", err)
3140	}
3141
3142	clk.advance(11 * time.Second)
3143	a.sweepOnce()
3144	waitFor(t, "worker hibernation", func() bool { return a.entryCount() == 0 })
3145
3146	got, err := reg.Get("bot.x.internal")
3147	if err != nil {
3148	t.Fatalf("get: %v", err)
3149	}
3150	if got.NextDue != 4242 {
3151	t.Fatalf("next_due = %d, want 4242 — a worker's next-due arrives over the agent socket while it is ALIVE, not from a probe of a process we are about to kill", got.NextDue)
3152	}
3153	if b.dueProbes() != 0 {
3154	t.Fatal("a worker must never be probed for next-due: it has no socket to answer on")
3155	}
3156	if !b.lastRep().isClosed() {
3157	t.Fatal("the lease must be released and the DB parked on a worker sleep, exactly as for an instance")
3158	}
3159	if !b.workers[len(b.workers)-1].isStopped() {
3160	t.Fatal("the worker child was not stopped")
3161	}
3162	}
3163
3164	func TestWorkerContactDefersSleep(t *testing.T) {
3165	t.Parallel()
3166	a, _, clk, _ := testActivator(t, 10*time.Second)
3167	if err := a.WakeWorker(testWorkerRoute("bot.x.internal")); err != nil {
3168	t.Fatalf("WakeWorker: %v", err)
3169	}
3170	clk.advance(9 * time.Second)
3171	a.NoteWorkerContact("bot.x.internal")
3172	clk.advance(9 * time.Second)
3173	a.sweepOnce()
3174	// entryAwake, not entryCount: a condemned entry still counts until its
3175	// stop goroutine finishes, so a count here passes either way.
3176	if !a.entryAwake("bot.x.internal") {
3177	t.Fatal("agent-socket contact must defer the idle clock — it is a worker's only sign of life")
3178	}
3179	}
3180
3181	// The row's own idle window wins over the platform default: the ceilings ride
3182	// on the route (Task 1) precisely so the sweeper never needs a bucket read to
3183	// decide a worker has sat idle long enough.
3184	func TestWorkerRouteIdleWindowOverridesPlatformDefault(t *testing.T) {
3185	t.Parallel()
3186	a, _, clk, _ := testActivator(t, time.Hour) // the platform default would never fire
3187	rt := testWorkerRoute("bot.x.internal")
3188	rt.IdleSeconds = 30
3189	if err := a.WakeWorker(rt); err != nil {
3190	t.Fatalf("WakeWorker: %v", err)
3191	}
3192	clk.advance(20 * time.Second)
3193	a.sweepOnce()
3194	if !a.entryAwake("bot.x.internal") {
3195	t.Fatal("worker slept before its own idle window elapsed")
3196	}
3197	clk.advance(11 * time.Second)
3198	a.sweepOnce()
3199	waitFor(t, "row-scoped idle sleep", func() bool { return a.entryCount() == 0 })
3200	}
3201
3202	func TestWorkerMaxRunStopsALoopingWorker(t *testing.T) {
3203	t.Parallel()
3204	a, _, clk, _ := testActivator(t, time.Hour) // idle window never fires
3205	rt := testWorkerRoute("bot.x.internal")
3206	rt.MaxRunSeconds = 60
3207	if err := a.WakeWorker(rt); err != nil {
3208	t.Fatalf("WakeWorker: %v", err)
3209	}
3210	for i := 0; i < 10; i++ { // a worker that keeps talking but never idles
3211	clk.advance(10 * time.Second)
3212	a.NoteWorkerContact("bot.x.internal")
3213	}
3214	a.sweepOnce()
3215	waitFor(t, "max_run stop", func() bool { return a.entryCount() == 0 })
3216	}
3217
3218	// No ceiling declared (0) means no ceiling — an always-on worker must not be
3219	// stopped by a default nobody asked for.
3220	func TestWorkerWithoutMaxRunKeepsRunning(t *testing.T) {
3221	t.Parallel()
3222	a, _, clk, _ := testActivator(t, time.Hour)
3223	if err := a.WakeWorker(testWorkerRoute("bot.x.internal")); err != nil {
3224	t.Fatalf("WakeWorker: %v", err)
3225	}
3226	for i := 0; i < 10; i++ {
3227	clk.advance(time.Hour)
3228	a.NoteWorkerContact("bot.x.internal")
3229	a.sweepOnce()
3230	}
3231	if a.entryCount() != 1 {
3232	t.Fatal("a worker with max_run 0 was stopped by a ceiling it never declared")
3233	}
3234	}
3235
3236	func TestSleepWorkerIsImmediate(t *testing.T) {
3237	t.Parallel()
3238	a, _, _, _ := testActivator(t, time.Hour)
3239	if err := a.WakeWorker(testWorkerRoute("bot.x.internal")); err != nil {
3240	t.Fatalf("WakeWorker: %v", err)
3241	}
3242	a.SleepWorker("bot.x.internal")
3243	waitFor(t, "idle-requested stop", func() bool { return a.entryCount() == 0 })
3244	}
3245
3246	func TestWakeWorkerIsIdempotent(t *testing.T) {
3247	t.Parallel()
3248	a, b, _, _ := testActivator(t, time.Hour)
3249	rt := testWorkerRoute("bot.x.internal")
3250	for i := 0; i < 3; i++ {
3251	if err := a.WakeWorker(rt); err != nil {
3252	t.Fatalf("WakeWorker #%d: %v", i, err)
3253	}
3254	}
3255	if n := b.workerStarts(); n != 1 {
3256	t.Fatalf("workerStarts = %d, want 1 — a wake for an already-awake worker is a no-op, not a second child", n)
3257	}
3258	}
3259
3260	// An already-awake worker whose lease is gone is BROKEN, not up. WakeWorker
3261	// must stand it down and wake it afresh — reporting success to a boot pass or
3262	// an operator for a worker that cannot write is the one answer that makes a
3263	// dead sidecar look healthy.
3264	func TestWakeWorkerStandsDownALostLeaseAndRewakes(t *testing.T) {
3265	t.Parallel()
3266	a, b, _, _ := testActivator(t, time.Hour)
3267	rt := testWorkerRoute("bot.x.internal")
3268	if err := a.WakeWorker(rt); err != nil {
3269	t.Fatalf("WakeWorker: %v", err)
3270	}
3271	b.lastRep().setErr(errors.New("lease renewal failed"))
3272
3273	if err := a.WakeWorker(rt); err != nil {
3274	t.Fatalf("second WakeWorker: %v", err)
3275	}
3276	if n := b.workerStarts(); n != 2 {
3277	t.Fatalf("workerStarts = %d — a wake for a worker with a lost lease must recycle it, not report success", n)
3278	}
3279	if !a.entryAwake("bot.x.internal") {
3280	t.Fatal("the re-wake did not leave the worker awake")
3281	}
3282	if acquires, _, _ := b.counts(); acquires != 2 {
3283	t.Fatalf("lease acquires = %d, want 2 — the stand-down must re-claim the lease", acquires)
3284	}
3285	}
3286
3287	// Concurrent wakes coalesce on one entry, the same singleflight the request
3288	// path gets: three goroutines, one child, one lease.
3289	func TestWakeWorkerCoalescesConcurrentCallers(t *testing.T) {
3290	t.Parallel()
3291	a, b, _, _ := testActivator(t, time.Hour)
3292	rt := testWorkerRoute("bot.x.internal")
3293
3294	var wg sync.WaitGroup
3295	for i := 0; i < 3; i++ {
3296	wg.Add(1)
3297	go func() {
3298	defer wg.Done()
3299	if err := a.WakeWorker(rt); err != nil {
3300	t.Errorf("WakeWorker: %v", err)
3301	}
3302	}()
3303	}
3304	wg.Wait()
3305	if n := b.workerStarts(); n != 1 {
3306	t.Fatalf("workerStarts = %d, want 1 — concurrent wakes must coalesce", n)
3307	}
3308	if acquires, _, _ := b.counts(); acquires != 1 {
3309	t.Fatalf("lease acquires = %d, want 1", acquires)
3310	}
3311	}
3312
3313	// Shutdown announces itself BEFORE it snapshots the entry map, so a wake
3314	// dispatched after that point would insert an entry nothing is left to stop:
3315	// an orphan child holding the tenant's lease past the agent's exit. A wake
3316	// arriving mid-drain is refused outright.
3317	func TestWakeWorkerRefusesOnceShutdownHasBegun(t *testing.T) {
3318	a, b, _, _ := testActivator(t, time.Hour)
3319	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
3320	defer cancel()
3321	a.Shutdown(ctx)
3322
3323	if err := a.WakeWorker(testWorkerRoute("bot.x.internal")); err == nil {
3324	t.Fatal("WakeWorker succeeded during shutdown — its child would outlive the agent, holding the lease")
3325	}
3326	if n := b.workerStarts(); n != 0 {
3327	t.Fatalf("workerStarts = %d during shutdown, want 0", n)
3328	}
3329	if a.entryCount() != 0 {
3330	t.Fatal("a wake during shutdown left an entry behind")
3331	}
3332	}
3333
3334	// A failed worker wake releases the lease and leaves nothing behind, exactly
3335	// as a failed instance wake does — the error paths that must release a lease
3336	// are the reason a worker reuses doWake rather than reimplementing it.
3337	func TestWorkerWakeFailureReleasesLeaseAndClearsEntry(t *testing.T) {
3338	a, b, _, _ := testActivator(t, time.Hour)
3339	b.workerErr = errors.New("exec: no argv")
3340
3341	if err := a.WakeWorker(testWorkerRoute("bot.x.internal")); err == nil {
3342	t.Fatal("WakeWorker returned nil for a spawn that failed")
3343	}
3344	if a.entryCount() != 0 {
3345	t.Fatal("a failed worker wake left an entry behind")
3346	}
3347	if n := b.releaseCount(); n != 1 {
3348	t.Fatalf("lease releases = %d, want exactly 1 on a failed worker wake", n)
3349	}
3350	}
3351
3352	// A worker is nobody's parent: the legacy co-spawn step must not run for one,
3353	// even on a box where the staging file happens to exist.
3354	func TestWorkerNeverCoSpawnsASidecar(t *testing.T) {
3355	t.Parallel()
3356	a, b, _, _ := testActivator(t, time.Hour)
3357	b.sidecarStaged = true
3358
3359	if err := a.WakeWorker(testWorkerRoute("bot.x.internal")); err != nil {
3360	t.Fatalf("WakeWorker: %v", err)
3361	}
3362	a.sweepOnce() // the sweep's mid-flight adoption must not pick one up either
3363	if n := b.sidecarStartCount(); n != 0 {
3364	t.Fatalf("sidecar starts = %d for a worker, want 0 — a worker is nobody's parent", n)
3365	}
3366	}
3367
3368	// RetireExcept reconciles the EDGE's managed set — and edge.Reload refuses a
3369	// sidecar row on kind before it builds that set, so a worker's host can never
3370	// be in `keep`. Without an explicit exemption every edge reload (any route
3371	// change at all) would stop every running standalone sidecar.
3372	func TestRetireExceptLeavesWorkersAlone(t *testing.T) {
3373	t.Parallel()
3374	a, _, _, _ := testActivator(t, time.Hour)
3375	if err := a.WakeWorker(testWorkerRoute("bot.x.internal")); err != nil {
3376	t.Fatalf("WakeWorker: %v", err)
3377	}
3378	release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
3379	if err != nil {
3380	t.Fatalf("beginRequest: %v", err)
3381	}
3382	release()
3383
3384	a.RetireExcept(map[string]bool{}) // an edge reload that manages nothing
3385
3386	waitFor(t, "the instance to retire", func() bool { return !a.Owns("one.example.com") })
3387	if a.WorkerToken("bot.x.internal") == "" {
3388	t.Fatal("an edge reload retired a running worker — its host can never appear in the edge's managed set")
3389	}
3390	}
3391
3392	// The due-time sweep is a sidecar's ONLY timer wake source (nothing can dial
3393	// one), so a due sidecar row must come up through the worker path — StartWorker
3394	// and no health probe — not the instance one.
3395	func TestDueSweepWakesASidecarAsAWorker(t *testing.T) {
3396	t.Parallel()
3397	a, b, clk, reg := testActivator(t, time.Hour)
3398	rt := testWorkerRoute("bot.x.internal")
3399	if err := reg.Upsert(rt); err != nil {
3400	t.Fatalf("upsert: %v", err)
3401	}
3402	if err := reg.SetNextDue(rt.Host, clk.now().Unix()-1); err != nil {
3403	t.Fatalf("set next due: %v", err)
3404	}
3405
3406	a.sweepOnce()
3407	waitFor(t, "due worker wake", func() bool { return b.workerStarts() == 1 })
3408	if b.startCount() != 0 {
3409	t.Fatal("a due sidecar was woken through the instance path (Start + health probe)")
3410	}
3411	}
3412
3413	// A FIRED timer is one-shot. The row's next_due is not rewritten by a
3414	// worker's stop (it has no socket to probe), so unless the wake itself
3415	// consumes the value that fired, DueInstances hands the same row back on
3416	// EVERY sweep: wake, restore, idle, park, re-wake, forever — one lease
3417	// acquire and one S3 restore/park per cycle, and nothing in the logs that
3418	// reads as a failure.
3419	func TestDueFiredWorkerWakeHappensExactlyOnce(t *testing.T) {
3420	t.Parallel()
3421	a, b, clk, reg := testActivator(t, 10*time.Second)
3422	rt := testWorkerRoute("bot.x.internal")
3423	if err := reg.Upsert(rt); err != nil {
3424	t.Fatalf("upsert: %v", err)
3425	}
3426	if err := reg.SetNextDue(rt.Host, clk.now().Unix()-1); err != nil {
3427	t.Fatalf("set next due: %v", err)
3428	}
3429
3430	a.sweepOnce()
3431	waitFor(t, "the due-fired wake", func() bool { return a.entryAwake(rt.Host) })
3432	got, err := reg.Get(rt.Host)
3433	if err != nil {
3434	t.Fatalf("get: %v", err)
3435	}
3436	if got.NextDue != 0 {
3437	t.Fatalf("next_due = %d after the wake it fired, want 0 — a fired timer must be consumed by the wake it caused", got.NextDue)
3438	}
3439
3440	// It idles out, parks, and STAYS parked: several more sweeps, no second
3441	// wake, no churn.
3442	clk.advance(11 * time.Second)
3443	a.sweepOnce()
3444	waitFor(t, "the worker to park", func() bool { return a.entryCount() == 0 })
3445	for i := 0; i < 5; i++ {
3446	clk.advance(11 * time.Second)
3447	a.sweepOnce()
3448	}
3449	if n := b.workerStarts(); n != 1 {
3450	t.Fatalf("workerStarts = %d, want 1 — one due time woke the sidecar more than once (wake/park churn, every sweep, forever)", n)
3451	}
3452	if n := b.releaseCount(); n > 1 {
3453	t.Fatalf("leases released %d times for one due time — the sidecar is churning", n)
3454	}
3455	}
3456
3457	// The other half of the same distinction: a sidecar that goes to sleep BEFORE
3458	// its timer fires must keep its schedule. Consuming at STOP (rather than on
3459	// the wake the timer caused) would delete the feature — a sleeping sidecar's
3460	// next_due is its only way back up, since nothing can dial one.
3461	func TestWorkerSleepingBeforeItsDueTimeStillWakesForIt(t *testing.T) {
3462	t.Parallel()
3463	a, b, clk, reg := testActivator(t, 10*time.Second)
3464	rt := testWorkerRoute("bot.x.internal")
3465	if err := reg.Upsert(rt); err != nil {
3466	t.Fatalf("upsert: %v", err)
3467	}
3468	if err := a.WakeWorker(rt); err != nil { // an operator wake, not a due-fired one
3469	t.Fatalf("WakeWorker: %v", err)
3470	}
3471	due := clk.now().Add(time.Minute).Unix()
3472	if err := a.SetWorkerNextDue(rt.Host, due); err != nil { // "wake me in a minute"
3473	t.Fatalf("SetWorkerNextDue: %v", err)
3474	}
3475
3476	clk.advance(11 * time.Second) // idles out well before the due time
3477	a.sweepOnce()
3478	waitFor(t, "the worker to park", func() bool { return a.entryCount() == 0 })
3479	got, err := reg.Get(rt.Host)
3480	if err != nil {
3481	t.Fatalf("get: %v", err)
3482	}
3483	if got.NextDue != due {
3484	t.Fatalf("next_due = %d after a sleep BEFORE the due time, want %d — a schedule that has not fired must survive the sleep", got.NextDue, due)
3485	}
3486
3487	clk.advance(time.Minute)
3488	a.sweepOnce()
3489	waitFor(t, "the scheduled wake", func() bool { return b.workerStarts() == 2 })
3490	}
3491
3492	// A FAILED due-fired wake must consume NOTHING. The index is the retry — the
3493	// due sweep's own contract for a wake that could not complete — and consuming
3494	// on failure would convert harmless churn into the far worse bug: a sidecar
3495	// whose only way back up has been erased, silently, by a wake that never
3496	// happened.
3497	func TestFailedDueFiredWakeConsumesNothing(t *testing.T) {
3498	t.Parallel()
3499	a, b, clk, reg := testActivator(t, 10*time.Second)
3500	rt := testWorkerRoute("bot.x.internal")
3501	if err := reg.Upsert(rt); err != nil {
3502	t.Fatalf("upsert: %v", err)
3503	}
3504	fired := clk.now().Unix() - 1
3505	if err := reg.SetNextDue(rt.Host, fired); err != nil {
3506	t.Fatalf("set next due: %v", err)
3507	}
3508	b.mu.Lock()
3509	b.workerErr = errors.New("no build to exec") // the wake fails at the spawn
3510	b.mu.Unlock()
3511
3512	a.sweepOnce()
3513	waitFor(t, "the failed wake to clear its entry", func() bool { return a.entryCount() == 0 })
3514	got, err := reg.Get(rt.Host)
3515	if err != nil {
3516	t.Fatalf("get: %v", err)
3517	}
3518	if got.NextDue != fired {
3519	t.Fatalf("next_due = %d after a FAILED due wake, want %d — the schedule is the retry, and a sidecar whose next_due is erased by a wake that never happened has no way back up at all", got.NextDue, fired)
3520	}
3521
3522	// The next sweep retries it, exactly as the due sweep's own comment says.
3523	a.sweepOnce()
3524	waitFor(t, "the retry", func() bool { return b.workerStarts() == 2 })
3525
3526	// …and once a wake actually succeeds, THAT one consumes the schedule.
3527	b.mu.Lock()
3528	b.workerErr = nil
3529	b.mu.Unlock()
3530	a.sweepOnce()
3531	waitFor(t, "the successful wake", func() bool { return a.entryAwake(rt.Host) })
3532	if got, err := reg.Get(rt.Host); err != nil || got.NextDue != 0 {
3533	t.Fatalf("next_due = %+v (%v) after the wake that SUCCEEDED, want 0", got, err)
3534	}
3535	}
3536
3537	// The consumption is conditional on the row still holding the value that
3538	// fired: the child spawned by this very wake may post its NEXT due time over
3539	// the agent socket during the settle window, and an unconditional clear would
3540	// eat it — a bot that ran once and never again.
3541	func TestDueFiredWakeKeepsANewNextDuePostedByTheChild(t *testing.T) {
3542	t.Parallel()
3543	a, b, clk, reg := testActivator(t, 10*time.Second)
3544	rt := testWorkerRoute("bot.x.internal")
3545	if err := reg.Upsert(rt); err != nil {
3546	t.Fatalf("upsert: %v", err)
3547	}
3548	if err := reg.SetNextDue(rt.Host, clk.now().Unix()-1); err != nil {
3549	t.Fatalf("set next due: %v", err)
3550	}
3551	next := clk.now().Add(time.Minute).Unix()
3552	b.mu.Lock()
3553	b.workerStartHook = func(r registry.Route) {
3554	// The child's own /v1/next-due, landing while the wake is still in
3555	// flight — before anything could consume the time that fired.
3556	if err := a.SetWorkerNextDue(r.Host, next); err != nil {
3557	t.Errorf("SetWorkerNextDue: %v", err)
3558	}
3559	}
3560	b.mu.Unlock()
3561
3562	a.sweepOnce()
3563	waitFor(t, "the due-fired wake", func() bool { return a.entryAwake(rt.Host) })
3564	got, err := reg.Get(rt.Host)
3565	if err != nil {
3566	t.Fatalf("get: %v", err)
3567	}
3568	if got.NextDue != next {
3569	t.Fatalf("next_due = %d, want %d — consuming the FIRED time must not eat the one the child posted during its wake", got.NextDue, next)
3570	}
3571
3572	// And that schedule is honoured: park, then wake again when it arrives.
3573	b.mu.Lock()
3574	b.workerStartHook = nil
3575	b.mu.Unlock()
3576	clk.advance(11 * time.Second)
3577	a.sweepOnce()
3578	waitFor(t, "the worker to park", func() bool { return a.entryCount() == 0 })
3579	clk.advance(time.Minute)
3580	a.sweepOnce()
3581	waitFor(t, "the child's own scheduled wake", func() bool { return b.workerStarts() == 2 })
3582	}
3583
3584	// --- the settle window --------------------------------------------------------
3585
3586	// noteReadyAsChild reports readiness the way the real child must: presenting
3587	// the token it was handed in its env, which the agent socket (Task 7) matches
3588	// against WorkerToken before accepting anything. Calling NoteWorkerReady
3589	// directly would walk around the authorization the feature actually depends
3590	// on — and hide a token that is not visible until the wake resolves, which
3591	// would make this whole early-ready path dead code in production.
3592	func noteReadyAsChild(t *testing.T, a *Service, b *fakeBackend, host, version string) {
3593	t.Helper()
3594	presented := b.lastWorkerEnv()["CARLOS_SIDECAR_TOKEN"]
3595	live := a.WorkerToken(host)
3596	if presented == "" || live == "" || presented != live {
3597	t.Fatalf("the agent socket would REFUSE this child's ready call: it presents %q, WorkerToken says %q", presented, live)
3598	}
3599	a.NoteWorkerReady(host, version)
3600	}
3601
3602	// The child's very first call arrives while its wake is still resolving — that
3603	// call is what ends the settle window — so its credential must be live then,
3604	// not only once the entry is awake.
3605	func TestWorkerTokenIsLiveDuringTheSettleWindow(t *testing.T) {
3606	a, b, _, _ := testActivator(t, time.Hour)
3607	old := sidecarSettle
3608	sidecarSettle = 5 * time.Second
3609	defer func() { sidecarSettle = old }()
3610
3611	done := make(chan error, 1)
3612	go func() { done <- a.WakeWorker(testWorkerRoute("bot.x.internal")) }()
3613	waitFor(t, "the waking entry to appear", func() bool { return a.Owns("bot.x.internal") })
3614	waitFor(t, "the child to be spawned", func() bool { return b.workerStarts() == 1 })
3615
3616	if tok := a.WorkerToken("bot.x.internal"); tok == "" {
3617	t.Fatal("no token during the settle window — the socket would refuse the /v1/ready call it is waiting for")
3618	}
3619	noteReadyAsChild(t, a, b, "bot.x.internal", "v9") // releases the wake
3620	if err := <-done; err != nil {
3621	t.Fatalf("WakeWorker: %v", err)
3622	}
3623	}
3624
3625	// A worker binds nothing, so there is no socket to probe: readiness is the
3626	// settle window. A child that exits INSIDE it never became a working sidecar —
3627	// that is a failed wake, and it must give the lease back exactly as a failed
3628	// instance start does, not sit in the map holding one.
3629	func TestWorkerExitInSettleWindowFailsWakeAndReleasesLease(t *testing.T) {
3630	a, b, _, _ := testActivator(t, time.Hour)
3631	b.workerDeadOnStart = true
3632
3633	err := a.WakeWorker(testWorkerRoute("bot.x.internal"))
3634	if err == nil {
3635	t.Fatal("a worker that died inside its settle window reported a successful wake")
3636	}
3637	if !errors.Is(err, errWorkerExitedEarly) {
3638	t.Fatalf("wake error = %v, want it to name the settle-window exit", err)
3639	}
3640	if a.entryCount() != 0 {
3641	t.Fatal("a failed settle left an entry behind")
3642	}
3643	if n := b.releaseCount(); n != 1 {
3644	t.Fatalf("lease releases = %d, want exactly 1", n)
3645	}
3646	if !b.lastWorker().isStopped() {
3647	t.Fatal("the half-started child was not stopped")
3648	}
3649	// The credential is published at mint time so the child can call
3650	// /v1/ready during the window — a wake that FAILED must still take it
3651	// away, which the entry's deletion does.
3652	if tok := a.WorkerToken("bot.x.internal"); tok != "" {
3653	t.Fatalf("WorkerToken = %q after a failed wake — a dead wake's credential must not stay live", tok)
3654	}
3655	}
3656
3657	// A child that survives the window is up — no probe, no readiness call needed.
3658	func TestWorkerSurvivingSettleWindowIsAwake(t *testing.T) {
3659	a, _, _, _ := testActivator(t, time.Hour)
3660	old := sidecarSettle
3661	sidecarSettle = 60 * time.Millisecond
3662	defer func() { sidecarSettle = old }()
3663
3664	start := time.Now()
3665	if err := a.WakeWorker(testWorkerRoute("bot.x.internal")); err != nil {
3666	t.Fatalf("WakeWorker: %v", err)
3667	}
3668	if elapsed := time.Since(start); elapsed < 50*time.Millisecond {
3669	t.Fatalf("wake resolved in %s — a worker nobody vouched for must survive the whole settle window first", elapsed)
3670	}
3671	if a.WorkerToken("bot.x.internal") == "" {
3672	t.Fatal("the worker is not awake after surviving its settle window")
3673	}
3674	}
3675
3676	// …and a child that reports ready ends the window early: a fast sidecar must
3677	// not pay ten seconds of dead time on every wake.
3678	func TestWorkerReadyEndsSettleWindowEarly(t *testing.T) {
3679	a, b, _, _ := testActivator(t, time.Hour)
3680	old := sidecarSettle
3681	sidecarSettle = 5 * time.Second
3682	defer func() { sidecarSettle = old }()
3683
3684	done := make(chan error, 1)
3685	start := time.Now()
3686	go func() { done <- a.WakeWorker(testWorkerRoute("bot.x.internal")) }()
3687
3688	waitFor(t, "the waking entry to appear", func() bool { return a.Owns("bot.x.internal") })
3689	// The entry appears before the wake has minted the token and spawned the
3690	// child; a child cannot report ready before it exists. Waiting only for
3691	// the entry raced the spawn under load (main 9c535162's CI).
3692	waitFor(t, "the child to be spawned", func() bool { return b.workerStarts() == 1 })
3693	noteReadyAsChild(t, a, b, "bot.x.internal", "v9")
3694
3695	select {
3696	case err := <-done:
3697	if err != nil {
3698	t.Fatalf("WakeWorker: %v", err)
3699	}
3700	case <-time.After(3 * time.Second):
3701	t.Fatal("a ready report did not end the settle window — the wake waited it out")
3702	}
3703	if elapsed := time.Since(start); elapsed > 2*time.Second {
3704	t.Fatalf("wake took %s despite a ready report", elapsed)
3705	}
3706	}
3707
3708	// --- always-on workers: supervision ------------------------------------------
3709
3710	func alwaysOnWorkerRoute(host string) registry.Route {
3711	rt := testWorkerRoute(host)
3712	rt.Hibernate = false // the record exists ⇒ the process runs
3713	return rt
3714	}
3715
3716	// An always-on worker's exit is a CRASH, not a finished job: it is restarted
3717	// in place, keeping its lease and replication — and the backoff GROWS, so a
3718	// crash-looping bot paces itself instead of hot-looping the box.
3719	func TestAlwaysOnWorkerCrashRestartsWithGrowingBackoff(t *testing.T) {
3720	t.Parallel()
3721	a, b, clk, _ := testActivator(t, time.Hour)
3722	if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil {
3723	t.Fatalf("WakeWorker: %v", err)
3724	}
3725
3726	// Crash 1: restarted at once, and the 2s delay is armed for the next.
3727	// The spawn runs off a.mu (a bucket materialize must never be held across
3728	// the global mutex), so the count is awaited, not read straight after.
3729	b.lastWorker().kill()
3730	a.sweepOnce()
3731	waitForRestart(t, a, b, "bot.x.internal", 2)
3732	if a.entryCount() != 1 {
3733	t.Fatal("the crashed worker's entry was torn down — an always-on worker restarts in place")
3734	}
3735	if n := b.releaseCount(); n != 0 {
3736	t.Fatalf("lease releases = %d — an in-place restart must not churn the lease", n)
3737	}
3738
3739	// Crash 2 inside the backoff: NOT restarted yet. entryRespawning is set
3740	// synchronously under the sweep's own lock, so its absence right after the
3741	// sweep is a fact, not a race.
3742	b.lastWorker().kill()
3743	a.sweepOnce()
3744	if a.entryRespawning("bot.x.internal") || b.workerStarts() != 2 {
3745	t.Fatalf("workerStarts = %d — a crash inside the backoff must wait, not hot-loop", b.workerStarts())
3746	}
3747	clk.advance(2 * time.Second)
3748	a.sweepOnce()
3749	waitForRestart(t, a, b, "bot.x.internal", 3)
3750
3751	// Crash 3: the delay has DOUBLED — 2s is no longer enough.
3752	b.lastWorker().kill()
3753	clk.advance(2 * time.Second)
3754	a.sweepOnce()
3755	if a.entryRespawning("bot.x.internal") || b.workerStarts() != 3 {
3756	t.Fatalf("workerStarts = %d — the backoff must double after each crash, not stay at %s", b.workerStarts(), sidecarRestartBase)
3757	}
3758	clk.advance(3 * time.Second)
3759	a.sweepOnce()
3760	waitForRestart(t, a, b, "bot.x.internal", 4)
3761	}
3762
3763	// waitForRestart waits until a supervised respawn has both spawned its child
3764	// and INSTALLED it — the install and the interlock clear happen in one
3765	// critical section, so waiting on the pair is what makes the next kill land on
3766	// the new child rather than the outgoing one.
3767	func waitForRestart(t *testing.T, a *Service, b *fakeBackend, host string, want int) {
3768	t.Helper()
3769	waitFor(t, fmt.Sprintf("restart #%d of %s", want, host), func() bool {
3770	return b.workerStarts() == want && !a.entryRespawning(host)
3771	})
3772	}
3773
3774	// A run that lasted sidecarHealthyRun earns a fresh backoff — one crash a day
3775	// must never escalate to the cap (the co-spawn supervisor's rule).
3776	func TestAlwaysOnWorkerHealthyRunResetsBackoff(t *testing.T) {
3777	t.Parallel()
3778	a, b, clk, _ := testActivator(t, time.Hour)
3779	if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil {
3780	t.Fatalf("WakeWorker: %v", err)
3781	}
3782	b.lastWorker().kill()
3783	a.sweepOnce() // restart 1 — backoff now 2s
3784	waitForRestart(t, a, b, "bot.x.internal", 2)
3785
3786	// The replacement runs healthily for well past the healthy-run mark, then
3787	// dies: the next restart is immediate, not delayed.
3788	clk.advance(sidecarHealthyRun + time.Minute)
3789	b.lastWorker().kill()
3790	a.sweepOnce()
3791	waitForRestart(t, a, b, "bot.x.internal", 3)
3792	}
3793
3794	// A restart is a new incarnation: a fresh token, so the dead child's
3795	// credential can never speak for the live one.
3796	func TestAlwaysOnWorkerRestartMintsANewToken(t *testing.T) {
3797	t.Parallel()
3798	a, b, _, _ := testActivator(t, time.Hour)
3799	if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil {
3800	t.Fatalf("WakeWorker: %v", err)
3801	}
3802	first := a.WorkerToken("bot.x.internal")
3803	b.lastWorker().kill()
3804	a.sweepOnce()
3805	waitForRestart(t, a, b, "bot.x.internal", 2)
3806
3807	second := a.WorkerToken("bot.x.internal")
3808	if second == "" || second == first {
3809	t.Fatalf("token after restart = %q (was %q) — every incarnation gets its own", second, first)
3810	}
3811	if env := b.lastWorkerEnv(); env["CARLOS_SIDECAR_TOKEN"] != second {
3812	t.Fatalf("restarted child was handed %q but the socket expects %q", env["CARLOS_SIDECAR_TOKEN"], second)
3813	}
3814	}
3815
3816	// A HIBERNATING worker's exit means "I finished my work" — the existing
3817	// dead-child stand-down is exactly right, and must not become a restart loop.
3818	func TestHibernatingWorkerExitSleepsRatherThanRestarting(t *testing.T) {
3819	t.Parallel()
3820	a, b, _, _ := testActivator(t, time.Hour)
3821	if err := a.WakeWorker(testWorkerRoute("bot.x.internal")); err != nil { // Hibernate: true
3822	t.Fatalf("WakeWorker: %v", err)
3823	}
3824	b.lastWorker().kill() // a CLEAN exit: "my work is done"
3825	a.sweepOnce()
3826
3827	waitFor(t, "the finished worker to sleep", func() bool { return a.entryCount() == 0 })
3828	if n := b.workerStarts(); n != 1 {
3829	t.Fatalf("workerStarts = %d — a hibernating worker that exited has FINISHED; it must not be restarted", n)
3830	}
3831	if !b.lastRep().isClosed() {
3832	t.Fatal("the lease was not released when the finished worker stopped")
3833	}
3834	}
3835
3836	// An always-on worker has no idle clock: idleness is no reason to stop
3837	// something whose record says it must be running, and a stopped one has no
3838	// wake source at all (no request can reach it, and it has no schedule).
3839	func TestAlwaysOnWorkerIgnoresTheIdleClock(t *testing.T) {
3840	t.Parallel()
3841	a, b, clk, _ := testActivator(t, 10*time.Second)
3842	rt := alwaysOnWorkerRoute("bot.x.internal")
3843	rt.IdleSeconds = 5
3844	if err := a.WakeWorker(rt); err != nil {
3845	t.Fatalf("WakeWorker: %v", err)
3846	}
3847
3848	clk.advance(time.Hour) // far past the idle window
3849	a.sweepOnce()
3850	// entryAwake, not entryCount: beginStopLocked flips the state under the
3851	// lock and stops on its own goroutine, so a count taken right after the
3852	// sweep would still see the entry it just condemned.
3853	if !a.entryAwake("bot.x.internal") {
3854	t.Fatal("an always-on worker was stopped by an idle window — nothing would ever wake it again")
3855	}
3856	if n := b.workerStarts(); n != 1 {
3857	t.Fatalf("workerStarts = %d, want 1 — nothing should have recycled a worker with no declared ceiling", n)
3858	}
3859	}
3860
3861	// A HIBERNATING worker that exits NON-ZERO has crashed, and design §7.2 says
3862	// a crash restarts, unqualified. Read as a finish, it would sleep forever:
3863	// it fell over before it could set a next-due, and nothing can dial one.
3864	func TestHibernatingWorkerCrashRestartsInsteadOfSleeping(t *testing.T) {
3865	t.Parallel()
3866	a, b, _, _ := testActivator(t, time.Hour)
3867	if err := a.WakeWorker(testWorkerRoute("bot.x.internal")); err != nil { // Hibernate: true
3868	t.Fatalf("WakeWorker: %v", err)
3869	}
3870	b.lastWorker().crash() // fell over, non-zero
3871	a.sweepOnce()
3872
3873	waitForRestart(t, a, b, "bot.x.internal", 2)
3874	if !a.entryAwake("bot.x.internal") {
3875	t.Fatal("a crashed hibernating worker was parked — it never set a next-due, so nothing would wake it again")
3876	}
3877	if b.lastRep().isClosed() {
3878	t.Fatal("the lease was released on a crash restart — only the child died")
3879	}
3880	}
3881
3882	// max_run is the wedged-reconnect-loop guard, and an always-on worker is
3883	// exactly the shape that gets wedged. It is spent as a RECYCLE there, not a
3884	// stop: stopping one leaves nothing to start it again.
3885	func TestAlwaysOnWorkerMaxRunRecyclesItsChild(t *testing.T) {
3886	t.Parallel()
3887	a, b, clk, _ := testActivator(t, time.Hour)
3888	rt := alwaysOnWorkerRoute("bot.x.internal")
3889	rt.MaxRunSeconds = 30
3890	if err := a.WakeWorker(rt); err != nil {
3891	t.Fatalf("WakeWorker: %v", err)
3892	}
3893	first := b.lastWorker()
3894
3895	clk.advance(31 * time.Second)
3896	a.sweepOnce()
3897	waitForRestart(t, a, b, "bot.x.internal", 2)
3898	if !a.entryAwake("bot.x.internal") {
3899	t.Fatal("an always-on worker was STOPPED by its ceiling — nothing would ever start it again")
3900	}
3901	if !first.isStopped() {
3902	t.Fatal("the wedged child was left running beside its replacement")
3903	}
3904	if b.lastRep().isClosed() {
3905	t.Fatal("the lease was released by a max_run recycle — the entry keeps its lease and replication")
3906	}
3907
3908	// And the ceiling re-arms on the new child rather than firing every sweep.
3909	a.sweepOnce()
3910	if n := b.workerStarts(); n != 2 {
3911	t.Fatalf("workerStarts = %d, want 2 — the fresh child's own max_run clock starts at its spawn", n)
3912	}
3913	}
3914
3915	// §7.2's contract is "the record exists ⇒ the process runs", and until the
3916	// sweep grew this arm it held only across agent RESTARTS: the boot pass runs
3917	// once, at arming, and the due-time loop gates on Hibernate — so an always-on
3918	// sidecar created while the agent was running had nothing at all that would
3919	// start it.
3920	func TestSweepStartsAnAlwaysOnWorkerThatAppearedMidFlight(t *testing.T) {
3921	t.Parallel()
3922	a, b, _, reg := testActivator(t, time.Hour)
3923	rt := alwaysOnWorkerRoute("bot.x.internal")
3924	if err := reg.Upsert(rt); err != nil { // the reconcile pass mints it, mid-flight
3925	t.Fatalf("upsert: %v", err)
3926	}
3927
3928	a.sweepOnce()
3929	waitFor(t, "the always-on sidecar to start", func() bool { return a.entryAwake(rt.Host) })
3930	if n := b.workerStarts(); n != 1 {
3931	t.Fatalf("workerStarts = %d, want 1", n)
3932	}
3933
3934	// And it is a one-shot, not a per-tick respawn: an entry that is already
3935	// live is left entirely alone.
3936	for i := 0; i < 3; i++ {
3937	a.sweepOnce()
3938	}
3939	if n := b.workerStarts(); n != 1 {
3940	t.Fatalf("workerStarts = %d after three more sweeps, want 1 — a running always-on sidecar must not be started again", n)
3941	}
3942	}
3943
3944	// The arm is for always-on rows only. A HIBERNATING sidecar exists precisely
3945	// so that nothing runs until a wake — starting one here would delete the
3946	// class.
3947	func TestSweepNeverStartsAHibernatingWorker(t *testing.T) {
3948	a, b, _, reg := testActivator(t, time.Hour)
3949	rt := testWorkerRoute("bot.x.internal") // Hibernate: true, no next_due
3950	if err := reg.Upsert(rt); err != nil {
3951	t.Fatalf("upsert: %v", err)
3952	}
3953	for i := 0; i < 3; i++ {
3954	a.sweepOnce()
3955	}
3956	if n := b.workerStarts(); n != 0 {
3957	t.Fatalf("workerStarts = %d — a hibernating sidecar must stay asleep until a wake source fires", n)
3958	}
3959	if a.entryCount() != 0 {
3960	t.Fatal("a hibernating sidecar was given a live entry by the sweep")
3961	}
3962	}
3963
3964	// A stranded always-on worker is a bounded batch, not a stampede: each wake
3965	// claims an S3 lease and restores a database.
3966	func TestSweepBoundsAlwaysOnStartsPerTick(t *testing.T) {
3967	t.Parallel()
3968	a, b, _, reg := testActivator(t, time.Hour)
3969	for i := 0; i < maxWorkerStartsPerSweep+3; i++ {
3970	rt := alwaysOnWorkerRoute(fmt.Sprintf("bot%02d.x.internal", i))
3971	if err := reg.Upsert(rt); err != nil {
3972	t.Fatalf("upsert: %v", err)
3973	}
3974	}
3975	a.sweepOnce()
3976	waitFor(t, "the first batch to start", func() bool { return b.workerStarts() == maxWorkerStartsPerSweep })
3977	if n := b.workerStarts(); n != maxWorkerStartsPerSweep {
3978	t.Fatalf("workerStarts = %d in one sweep, want the per-sweep bound %d", n, maxWorkerStartsPerSweep)
3979	}
3980	a.sweepOnce() // the remainder comes up on the next tick
3981	waitFor(t, "the remainder to start", func() bool { return b.workerStarts() == maxWorkerStartsPerSweep+3 })
3982	}
3983
3984	// A supervised restart must not exec while holding the activator's global
3985	// mutex: ExecBackend.StartWorker materializes the build from the bucket when
3986	// live/<host> is dangling, and a slow fetch held across a.mu stalls every
3987	// request wake, Owns and Shutdown on the box.
3988	func TestWorkerCrashRestartSpawnsOffTheActivatorLock(t *testing.T) {
3989	a, b, _, _ := testActivator(t, time.Hour)
3990	if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil {
3991	t.Fatalf("WakeWorker: %v", err)
3992	}
3993	b.mu.Lock()
3994	b.workerStartDelay = 400 * time.Millisecond // a slow exec: a bucket fetch
3995	b.mu.Unlock()
3996	b.lastWorker().kill()
3997
3998	start := time.Now()
3999	a.sweepOnce()
4000	if elapsed := time.Since(start); elapsed > 150*time.Millisecond {
4001	t.Fatalf("sweepOnce took %s — the sweeper is doing the exec itself, under a.mu", elapsed)
4002	}
4003	waitFor(t, "the restart to be in flight", func() bool { return a.entryRespawning("bot.x.internal") })
4004
4005	// The lock must be free for everyone else while the spawn runs.
4006	start = time.Now()
4007	owns := a.Owns("bot.x.internal")
4008	if elapsed := time.Since(start); elapsed > 150*time.Millisecond {
4009	t.Fatalf("Owns blocked %s behind a respawn — the spawn is holding the global mutex", elapsed)
4010	}
4011	if !owns {
4012	t.Fatal("Owns lost the entry mid-restart")
4013	}
4014	waitFor(t, "the restart to land", func() bool { return b.workerStarts() == 2 })
4015	}
4016
4017	// A respawn must never spawn a child after the lease has gone. Shutdown
4018	// announces itself first, so a respawn still in flight abandons rather than
4019	// starting a process that would write the tenant's DB with no single-writer
4020	// claim behind it — and possibly outlive the agent as an orphan.
4021	func TestShutdownStopsARespawnFromOutlivingTheLease(t *testing.T) {
4022	t.Parallel()
4023	a, b, clk, ver := shippingActivator(t, time.Hour)
4024	if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil {
4025	t.Fatalf("WakeWorker: %v", err)
4026	}
4027	b.mu.Lock()
4028	b.workerStartDelay = 500 * time.Millisecond
4029	b.mu.Unlock()
4030
4031	ver.Store("v2")
4032	a.sweepOnce()
4033	clk.advance(ConvergeGrace + time.Minute)
4034	a.sweepOnce() // starts the respawn
4035	waitFor(t, "the respawn to be in flight", func() bool { return a.entryRespawning("bot.x.internal") })
4036
4037	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
4038	defer cancel()
4039	a.Shutdown(ctx)
4040
4041	if a.entryCount() != 0 {
4042	t.Fatal("Shutdown returned with entries still live")
4043	}
4044	if n := b.workerStarts(); n != 1 {
4045	t.Fatalf("workerStarts = %d at shutdown-return — a child was spawned after the lease was released", n)
4046	}
4047	// …and none appears afterwards either: the goroutine must have abandoned,
4048	// not merely been outrun.
4049	time.Sleep(700 * time.Millisecond)
4050	if n := b.workerStarts(); n != 1 {
4051	t.Fatalf("workerStarts = %d after shutdown — an orphan was spawned once the agent had gone", n)
4052	}
4053	for i, p := range b.workers {
4054	if !p.isStopped() {
4055	t.Fatalf("worker child %d was left running by Shutdown", i)
4056	}
4057	}
4058	}
4059
4060	// THE invariant this task has been circling: when respawnDone is observed
4061	// closed, no child spawned by that respawn is still running.
4062	//
4063	// The case that proves it is the raced one — a stop lands after the respawn
4064	// has committed to spawning, so the child it produces must be SHED. If
4065	// respawnDone closed before that shed finished, stop() would fall through and
4066	// reach rep.Close — final sync plus lease release — with a live process still
4067	// pointed at the same database. Narrowing that window is not closing it: the
4068	// shed has to complete first.
4069	func TestRespawnShedsARacedChildBeforeTheLeaseIsReleased(t *testing.T) {
4070	t.Parallel()
4071	a, b, clk, ver := shippingActivator(t, time.Hour)
4072	if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil {
4073	t.Fatalf("WakeWorker: %v", err)
4074	}
4075	rep := b.lastRep()
4076	// The gate applies to children spawned from now on — i.e. the respawn's,
4077	// the one that will have to be shed. Its drain is held open so the
4078	// ordering is observable rather than a microsecond wide.
4079	gate := make(chan struct{})
4080	b.mu.Lock()
4081	b.workerStartDelay = 200 * time.Millisecond
4082	b.workerStopGate = gate
4083	b.mu.Unlock()
4084
4085	ver.Store("v2")
4086	a.sweepOnce()
4087	clk.advance(ConvergeGrace + time.Minute)
4088	a.sweepOnce()
4089	waitFor(t, "the respawn's exec to begin", func() bool { return b.workerEnters() == 2 })
4090
4091	// The stop lands mid-exec: past the pre-spawn check, so the child is
4092	// already coming and can only be shed.
4093	if !a.Bounce("bot.x.internal") {
4094	t.Fatal("Bounce found no awake worker to stop")
4095	}
4096	waitFor(t, "the raced child to be spawned", func() bool { return b.workerStarts() == 2 })
4097
4098	// It is now draining, held by the gate. The lease must still be held.
4099	deadline := time.Now().Add(300 * time.Millisecond)
4100	for time.Now().Before(deadline) {
4101	if rep.isClosed() {
4102	t.Fatal("the lease was released while the respawn's raced child was still draining — a live process is pointed at a database we no longer hold")
4103	}
4104	time.Sleep(10 * time.Millisecond)
4105	}
4106
4107	close(gate) // let the shed finish
4108	waitFor(t, "the stop to complete", func() bool { return a.entryCount() == 0 })
4109	if n := rep.closeCount(); n != 1 {
4110	t.Fatalf("lease closes = %d, want exactly 1", n)
4111	}
4112	for i, p := range b.workers {
4113	if !p.isStopped() {
4114	t.Fatalf("worker child %d survived the stop", i)
4115	}
4116	}
4117	}
4118
4119	// The same invariant, but against the SCHEDULING order rather than the exec:
4120	// the stop goroutine arrives after the respawn has cleared its interlock and
4121	// while the shed is still draining. Gating the wait on that flag would skip it
4122	// here — the flag is cleared in the respawn's final critical section, before
4123	// the shed — and the lease would go while a child was still alive.
4124	//
4125	// beginStopLocked flips the state under a.mu and then launches the stop, so in
4126	// production the two race for the lock with nothing ordering them. This test
4127	// arranges the losing order deliberately, from inside the package: it holds
4128	// a.mu until the respawn's final critical section is queued behind it, condemns
4129	// the entry, and releases — Go hands a mutex directly to a waiter that has
4130	// waited past its starvation threshold, so the respawn runs first and the stop
4131	// goroutine reads the entry second. No invariant is bent to produce it; only
4132	// the schedule is pinned.
4133	func TestStopScheduledAfterTheInterlockClearStillWaitsForTheShed(t *testing.T) {
4134	t.Parallel()
4135	a, b, clk, ver := shippingActivator(t, time.Hour)
4136	if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil {
4137	t.Fatalf("WakeWorker: %v", err)
4138	}
4139	rep := b.lastRep()
4140	gate := make(chan struct{})
4141	b.mu.Lock()
4142	b.workerStartDelay = 150 * time.Millisecond
4143	b.workerStopGate = gate // held on the respawn's child: its drain blocks
4144	b.mu.Unlock()
4145
4146	ver.Store("v2")
4147	a.sweepOnce()
4148	clk.advance(ConvergeGrace + time.Minute)
4149	a.sweepOnce()
4150	waitFor(t, "the respawn's exec to begin", func() bool { return b.workerEnters() == 2 })
4151
4152	// Take the lock while the exec is still running, and hold it: the respawn
4153	// finishes and queues behind us for its final critical section. THEN
4154	// condemn the entry — so its `live` check will fail and it must shed —
4155	// and launch the stop, which queues behind the respawn. On release the
4156	// respawn goes first (clearing the interlock, entering its gated shed) and
4157	// the stop reads second: the order that used to skip the wait.
4158	a.mu.Lock()
4159	time.Sleep(300 * time.Millisecond) // exec completes; that goroutine queues on a.mu
4160	e := a.entries["bot.x.internal"]
4161	if e != nil {
4162	a.beginStopLocked(e, StopIdle)
4163	}
4164	a.mu.Unlock()
4165	if e == nil {
4166	t.Fatal("the worker entry vanished before the stop could be arranged")
4167	}
4168	waitFor(t, "the raced child to be spawned", func() bool { return b.workerStarts() == 2 })
4169
4170	// The shed is now draining behind the gate. The lease must still be held:
4171	// releasing it here is releasing it under a live process.
4172	deadline := time.Now().Add(300 * time.Millisecond)
4173	for time.Now().Before(deadline) {
4174	if rep.isClosed() {
4175	t.Fatal("the lease was released while the respawn's raced child was still draining — the stop skipped its wait because the interlock had already been cleared")
4176	}
4177	time.Sleep(10 * time.Millisecond)
4178	}
4179
4180	close(gate)
4181	waitFor(t, "the stop to complete", func() bool { return a.entryCount() == 0 })
4182	if n := rep.closeCount(); n != 1 {
4183	t.Fatalf("lease closes = %d, want exactly 1", n)
4184	}
4185	for i, p := range b.workers {
4186	if !p.isStopped() {
4187	t.Fatalf("worker child %d survived the stop", i)
4188	}
4189	}
4190	}
4191
4192	// Shutdown CANCELS an in-flight worker spawn rather than waiting it out. The
4193	// wait is right everywhere else, but a shutdown answers to a deadline it does
4194	// not set: systemd's TimeoutStopSec defaults to 90s and workerSpawnTimeout
4195	// alone is 2 minutes, so blocking on one host's bucket materialize earns a
4196	// SIGKILL that strands leases and sockets for EVERY host on the box.
4197	func TestShutdownCancelsAnInFlightRespawnInsteadOfWaiting(t *testing.T) {
4198	a, b, clk, ver := shippingActivator(t, time.Hour)
4199	if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil {
4200	t.Fatalf("WakeWorker: %v", err)
4201	}
4202	rep := b.lastRep()
4203	b.mu.Lock()
4204	b.workerStartDelay = 5 * time.Second // a materialize nobody should wait for
4205	b.mu.Unlock()
4206
4207	ver.Store("v2")
4208	a.sweepOnce()
4209	clk.advance(ConvergeGrace + time.Minute)
4210	a.sweepOnce()
4211	// Inside the exec: past the pre-spawn check, so only cancellation — not
4212	// the shutdown announcement — can end this promptly.
4213	waitFor(t, "the respawn's exec to begin", func() bool { return b.workerEnters() == 2 })
4214
4215	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
4216	defer cancel()
4217	start := time.Now()
4218	a.Shutdown(ctx)
4219	if elapsed := time.Since(start); elapsed > 2*time.Second {
4220	t.Fatalf("Shutdown took %s — it waited out the spawn instead of cancelling it, which is how an agent gets SIGKILLed mid-shutdown", elapsed)
4221	}
4222
4223	// (c) nothing left wedged, and no child running.
4224	if a.entryCount() != 0 {
4225	t.Fatal("Shutdown returned with an entry still live")
4226	}
4227	if n := b.workerStarts(); n != 1 {
4228	t.Fatalf("workerStarts = %d — a cancelled exec must produce no child", n)
4229	}
4230	for i, p := range b.workers {
4231	if !p.isStopped() {
4232	t.Fatalf("worker child %d was left running by Shutdown", i)
4233	}
4234	}
4235	// The lease: released exactly once, by stop()'s rep.Close — the respawn
4236	// never owned it, so cancelling one can neither skip nor double the
4237	// release.
4238	if n := rep.closeCount(); n != 1 {
4239	t.Fatalf("lease closes = %d, want exactly 1 on the cancelled-respawn shutdown path", n)
4240	}
4241	// releaseCount counts the lease coming FREE by either route — fakeRep.Close
4242	// after a wake got past Replicate, or a raw Lease.Release before it (see
4243	// fakeRep.b). The two are mutually exclusive per wake, so exactly 1 here
4244	// says the same release rep.closeCount() just counted and no second one
4245	// beside it: cancelling a respawn that never owned the lease can neither
4246	// skip the release nor double it.
4247	if n := b.releaseCount(); n != 1 {
4248	t.Fatalf("lease releases = %d, want exactly 1 — the lease was traded for replication at wake, so it comes free through rep.Close and nowhere else", n)
4249	}
4250	// And the abandoned goroutine really is gone: nothing appears later.
4251	time.Sleep(300 * time.Millisecond)
4252	if n := b.workerStarts(); n != 1 {
4253	t.Fatalf("workerStarts = %d after shutdown — the cancelled respawn spawned anyway", n)
4254	}
4255	}
4256
4257	// A stop that lands AFTER a respawn has committed to spawning must not release
4258	// the lease under it: the child appearing moments later would write the
4259	// tenant's DB with no single-writer claim behind it. Bounce is the case
4260	// Shutdown's announcement does not cover — and, unlike a shutdown, it has no
4261	// deadline, so here the stop WAITS the spawn out rather than cancelling it.
4262	func TestStopWaitsOutAnInFlightRespawn(t *testing.T) {
4263	t.Parallel()
4264	a, b, clk, ver := shippingActivator(t, time.Hour)
4265	if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil {
4266	t.Fatalf("WakeWorker: %v", err)
4267	}
4268	b.mu.Lock()
4269	b.workerStartDelay = 400 * time.Millisecond
4270	b.mu.Unlock()
4271
4272	ver.Store("v2")
4273	a.sweepOnce()
4274	clk.advance(ConvergeGrace + time.Minute)
4275	a.sweepOnce()
4276	// Inside the exec, i.e. PAST the respawn's own pre-spawn check: from here
4277	// a stop can only be raced, which is what the wait exists for.
4278	waitFor(t, "the respawn's exec to begin", func() bool { return b.workerEnters() == 2 })
4279
4280	if !a.Bounce("bot.x.internal") {
4281	t.Fatal("Bounce found no awake worker to stop")
4282	}
4283	// The lease must still be held for as long as the exec is running. Watched
4284	// against the SPAWN's own progress (workerStarts rises only when
4285	// StartWorker returns), not against the entry: a stop that ignored the
4286	// respawn would delete the entry in the same breath as closing the lease,
4287	// leaving nothing to ask.
4288	deadline := time.Now().Add(300 * time.Millisecond)
4289	for time.Now().Before(deadline) {
4290	if b.lastRep().isClosed() && b.workerStarts() < 2 {
4291	t.Fatal("the lease was released while a respawn's exec was still running — the child appearing behind it would have no single-writer claim")
4292	}
4293	time.Sleep(10 * time.Millisecond)
4294	}
4295
4296	waitFor(t, "the stop to complete", func() bool { return a.entryCount() == 0 })
4297	for i, p := range b.workers {
4298	if !p.isStopped() {
4299	t.Fatalf("worker child %d survived the stop", i)
4300	}
4301	}
4302	}
4303
4304	// --- always-on workers: converging onto a shipped build -----------------------
4305
4306	// shippingActivator wires the fake backend and the sweep's liveVer seam to ONE
4307	// modelled live/<host> symlink, so a test can "ship" by moving it: a child
4308	// execs whatever it points at now, and the sweep reads the same source.
4309	func shippingActivator(t *testing.T, idle time.Duration) (*Service, *fakeBackend, *fakeClock, *atomic.Value) {
4310	t.Helper()
4311	a, b, clk, _ := testActivator(t, idle)
4312	var ver atomic.Value
4313	ver.Store("v1")
4314	b.execVerFn = func() string { return ver.Load().(string) }
4315	a.liveVer = func(string) (string, error) { return ver.Load().(string), nil }
4316	return a, b, clk, &ver
4317	}
4318
4319	// shippingActivatorWithRegistry is shippingActivator with the registry handle
4320	// kept — the version-clear-on-respawn tests need to read routes.version back,
4321	// which shippingActivator's callers have never needed before.
4322	func shippingActivatorWithRegistry(t *testing.T, idle time.Duration) (*Service, *fakeBackend, *fakeClock, *registry.Registry, *atomic.Value) {
4323	t.Helper()
4324	a, b, clk, reg := testActivator(t, idle)
4325	var ver atomic.Value
4326	ver.Store("v1")
4327	b.execVerFn = func() string { return ver.Load().(string) }
4328	a.liveVer = func(string) (string, error) { return ver.Load().(string), nil }
4329	return a, b, clk, reg, &ver
4330	}
4331
4332	// A running child never re-reads live/<host>, and an always-on worker never
4333	// sleeps, so without this it would serve the old binary forever while
4334	// `carlos deploy` reported success. It converges as one MOVE — stop, respawn,
4335	// same entry, same lease — because the ordinary cycle's second half ("a
4336	// request wakes it onto the new build") does not exist for a worker.
4337	func TestAlwaysOnWorkerRespawnsOntoANewBuild(t *testing.T) {
4338	t.Parallel()
4339	a, b, clk, ver := shippingActivator(t, time.Hour)
4340	if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil {
4341	t.Fatalf("WakeWorker: %v", err)
4342	}
4343	if got := a.entrySpawnVersion("bot.x.internal"); got != "v1" {
4344	t.Fatalf("spawn version = %q, want v1", got)
4345	}
4346	first := b.lastWorker()
4347
4348	ver.Store("v2") // a ship repoints the symlink under the running child
4349	a.sweepOnce() // first sighting: starts the grace clock, never acts
4350	// entryRespawning, not a spawn count: convergeWorkerLocked sets the
4351	// interlock synchronously, under the same lock the sweep holds, so this
4352	// sees a respawn that has merely been STARTED — a count would race the
4353	// goroutine and pass whether or not the grace window was honoured.
4354	if a.entryRespawning("bot.x.internal") {
4355	t.Fatal("a respawn began on the drift's first sighting — the ConvergeGrace window must pass first")
4356	}
4357	if n := b.workerStarts(); n != 1 {
4358	t.Fatalf("workerStarts = %d — the grace window must pass before anything is respawned", n)
4359	}
4360	clk.advance(ConvergeGrace + time.Minute)
4361	a.sweepOnce()
4362
4363	waitFor(t, "the respawn onto v2", func() bool { return a.entrySpawnVersion("bot.x.internal") == "v2" })
4364	if n := b.workerStarts(); n != 2 {
4365	t.Fatalf("workerStarts = %d, want 2 (one respawn)", n)
4366	}
4367	if !first.isStopped() {
4368	t.Fatal("the old child was left running beside its replacement — a sidecar would double its work")
4369	}
4370	if !a.entryAwake("bot.x.internal") {
4371	t.Fatal("the worker is not awake after converging — the respawn must land on the SAME entry")
4372	}
4373	if b.lastRep().isClosed() || b.releaseCount() != 0 {
4374	t.Fatal("converging respawned through a stop: the lease must not churn for an always-on worker")
4375	}
4376	// The new child holds the live token, so the agent socket still knows it.
4377	if tok := a.WorkerToken("bot.x.internal"); tok == "" || b.lastWorkerEnv()["CARLOS_SIDECAR_TOKEN"] != tok {
4378	t.Fatal("the respawned child was not handed the token the socket now expects")
4379	}
4380	}
4381
4382	// The one-per-sweep budget still applies: a bad ship must not respawn every
4383	// sidecar on the box at once.
4384	func TestWorkerConvergeRespawnIsRateLimitedPerSweep(t *testing.T) {
4385	t.Parallel()
4386	a, b, clk, ver := shippingActivator(t, time.Hour)
4387	for _, host := range []string{"bot-a.x.internal", "bot-b.x.internal"} {
4388	if err := a.WakeWorker(alwaysOnWorkerRoute(host)); err != nil {
4389	t.Fatalf("WakeWorker %s: %v", host, err)
4390	}
4391	}
4392	ver.Store("v2")
4393	a.sweepOnce() // grace clocks start for both
4394	clk.advance(ConvergeGrace + time.Minute)
4395
4396	a.sweepOnce()
4397	waitFor(t, "one respawn", func() bool { return b.workerStarts() == 3 })
4398	time.Sleep(100 * time.Millisecond) // give a second respawn every chance to appear
4399	if n := b.workerStarts(); n != 3 {
4400	t.Fatalf("workerStarts = %d after one sweep, want 3 — two workers, ONE respawn per tick", n)
4401	}
4402
4403	// The next tick takes the other one.
4404	a.sweepOnce()
4405	waitFor(t, "the second respawn", func() bool { return b.workerStarts() == 4 })
4406	}
4407
4408	// A HIBERNATING worker's convergence is unchanged: it is CYCLED (stopped, DB
4409	// parked, lease released) and comes back on its schedule, exactly as before.
4410	func TestHibernatingWorkerStillConvergenceCycles(t *testing.T) {
4411	t.Parallel()
4412	a, b, clk, ver := shippingActivator(t, time.Hour)
4413	if err := a.WakeWorker(testWorkerRoute("bot.x.internal")); err != nil { // Hibernate: true
4414	t.Fatalf("WakeWorker: %v", err)
4415	}
4416	ver.Store("v2")
4417	a.sweepOnce()
4418	clk.advance(ConvergeGrace + time.Minute)
4419	a.sweepOnce()
4420
4421	waitFor(t, "the convergence cycle", func() bool { return a.entryCount() == 0 })
4422	if n := b.workerStarts(); n != 1 {
4423	t.Fatalf("workerStarts = %d — a hibernating worker is cycled, not respawned in place", n)
4424	}
4425	if !b.lastRep().isClosed() {
4426	t.Fatal("the cycle did not release the lease")
4427	}
4428	}
4429
4430	// A respawn stops the old child before spawning the new one, so for a moment
4431	// the entry's proc reads as exited. A sweep landing in that window must do
4432	// NOTHING: read as a crash it would start a second child, and the box would
4433	// end up running two of a bot that must run once.
4434	func TestWorkerConvergeRespawnIsNotMistakenForACrash(t *testing.T) {
4435	t.Parallel()
4436	a, b, clk, ver := shippingActivator(t, time.Hour)
4437	if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil {
4438	t.Fatalf("WakeWorker: %v", err)
4439	}
4440	b.mu.Lock()
4441	b.workerStartDelay = 300 * time.Millisecond // hold the respawn's spawn open
4442	b.mu.Unlock()
4443
4444	ver.Store("v2")
4445	a.sweepOnce()
4446	clk.advance(ConvergeGrace + time.Minute)
4447	a.sweepOnce() // starts the respawn
4448
4449	waitFor(t, "the respawn to be in flight", func() bool { return a.entryRespawning("bot.x.internal") })
4450	a.sweepOnce() // a tick inside the window
4451	if n := b.workerStarts(); n != 1 {
4452	t.Fatalf("workerStarts = %d during a respawn — the sweep spawned a second child for a worker mid-converge", n)
4453	}
4454
4455	waitFor(t, "the respawn to land", func() bool { return a.entrySpawnVersion("bot.x.internal") == "v2" })
4456	if n := b.workerStarts(); n != 2 {
4457	t.Fatalf("workerStarts = %d after the respawn, want exactly 2", n)
4458	}
4459	}
4460
4461	// A respawn whose spawn FAILS must not leave a hole: the dead child stays in
4462	// place, and the ordinary crash-restart path picks it up on a later tick with
4463	// its backoff. Convergence degrades into supervision.
4464	func TestFailedConvergeRespawnFallsBackToCrashRestart(t *testing.T) {
4465	t.Parallel()
4466	a, b, clk, ver := shippingActivator(t, time.Hour)
4467	if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil {
4468	t.Fatalf("WakeWorker: %v", err)
4469	}
4470	b.mu.Lock()
4471	b.workerErr = errors.New("exec: live symlink gone")
4472	b.mu.Unlock()
4473
4474	ver.Store("v2")
4475	a.sweepOnce()
4476	clk.advance(ConvergeGrace + time.Minute)
4477	a.sweepOnce()
4478	waitFor(t, "the failed respawn to finish", func() bool { return !a.entryRespawning("bot.x.internal") })
4479	if !a.entryAwake("bot.x.internal") {
4480	t.Fatal("a failed respawn tore the entry down — the crash restart would have nothing left to fix")
4481	}
4482
4483	// The build comes back; the crash-restart path takes it from here.
4484	b.mu.Lock()
4485	b.workerErr = nil
4486	b.mu.Unlock()
4487	clk.advance(sidecarRestartBase + time.Second)
4488	a.sweepOnce()
4489	waitFor(t, "the crash restart to recover it", func() bool { return a.entrySpawnVersion("bot.x.internal") == "v2" })
4490	}
4491
4492	// --- /v1/idle and the always-on contract --------------------------------------
4493
4494	// "I have nothing to do" is not a request an always-on worker gets to make:
4495	// the stop would delete its entry, leaving nothing — not even the crash
4496	// restart — to bring it back before the agent restarts.
4497	func TestSleepWorkerIgnoresAnAlwaysOnWorker(t *testing.T) {
4498	t.Parallel()
4499	a, b, _, _ := testActivator(t, time.Hour)
4500	if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil {
4501	t.Fatalf("WakeWorker: %v", err)
4502	}
4503
4504	a.SleepWorker("bot.x.internal")
4505
4506	if !a.entryAwake("bot.x.internal") {
4507	t.Fatal("an always-on worker obeyed /v1/idle — nothing would ever start it again")
4508	}
4509	if b.releaseCount() != 0 {
4510	t.Fatal("an always-on worker's lease was released on a /v1/idle it should have refused")
4511	}
4512	}
4513
4514	// --- token minting ------------------------------------------------------------
4515
4516	// The agent supervises every route on the box. A wake that cannot mint a token
4517	// must fail THAT wake — lease released, error reported — never take the
4518	// process down with it.
4519	func TestWorkerTokenMintFailureFailsWakeAndReleasesLease(t *testing.T) {
4520	a, b, _, _ := testActivator(t, time.Hour)
4521	old := randRead
4522	randRead = func([]byte) (int, error) { return 0, errors.New("entropy pool exhausted") }
4523	defer func() { randRead = old }()
4524
4525	err := a.WakeWorker(testWorkerRoute("bot.x.internal"))
4526	if err == nil {
4527	t.Fatal("a wake with no token reported success")
4528	}
4529	if !strings.Contains(err.Error(), "mint worker token") {
4530	t.Fatalf("wake error = %v, want it to name the mint failure", err)
4531	}
4532	if a.entryCount() != 0 {
4533	t.Fatal("the failed wake left an entry behind")
4534	}
4535	if n := b.releaseCount(); n != 1 {
4536	t.Fatalf("lease releases = %d, want exactly 1 — a mint failure is a wake failure like any other", n)
4537	}
4538	if n := b.workerStarts(); n != 0 {
4539	t.Fatalf("workerStarts = %d — nothing may be spawned without a token to authorize it", n)
4540	}
4541	}
4542
4543	func TestConfigFromEnv(t *testing.T) {
4544	t.Setenv("CARLOS_HIBERNATE_BUCKET", "")
4545	if cfg, err := ConfigFromEnv(); err != nil || cfg != nil {
4546	t.Fatalf("unset bucket: got %+v, %v; want nil, nil (feature off)", cfg, err)
4547	}
4548
4549	t.Setenv("CARLOS_HIBERNATE_BUCKET", "b")
4550	t.Setenv("CARLOS_HIBERNATE_REGION", "eu-west-1")
4551	t.Setenv("CARLOS_HIBERNATE_PREFIX", "/instances/")
4552	t.Setenv("CARLOS_HIBERNATE_IDLE", "90s")
4553	cfg, err := ConfigFromEnv()
4554	if err != nil {
4555	t.Fatalf("config: %v", err)
4556	}
4557	if cfg.Bucket != "b" || cfg.Region != "eu-west-1" || cfg.Prefix != "instances" || cfg.Idle != 90*time.Second {
4558	t.Fatalf("got %+v", cfg)
4559	}
4560	if got := cfg.hibernatePrefix("x.example.com"); got != "instances/x.example.com" {
4561	t.Fatalf("prefix: %q", got)
4562	}
4563
4564	t.Setenv("CARLOS_HIBERNATE_IDLE", "")
4565	cfg, err = ConfigFromEnv()
4566	if err != nil || cfg.Idle != 15*time.Minute {
4567	t.Fatalf("default idle: %+v, %v", cfg, err)
4568	}
4569
4570	t.Setenv("CARLOS_HIBERNATE_IDLE", "banana")
4571	if _, err := ConfigFromEnv(); err == nil {
4572	t.Fatal("bad idle duration should error, not default silently")
4573	}
4574	}
4575
4576	// --- ExecBackend -------------------------------------------------------------
4577
4578	func testExecBackend(t *testing.T) *ExecBackend {
4579	t.Helper()
4580	return &ExecBackend{
4581	Cfg: Config{Bucket: "test-bucket", Idle: time.Minute},
4582	OptDir: t.TempDir(),
4583	DataDir: t.TempDir(),
4584	}
4585	}
4586
4587	// stageLive materializes <OptDir>/live/<host> → <OptDir>/versions/<ver>/app,
4588	// the layout the adopt pass maintains in production.
4589	func stageLive(t *testing.T, b *ExecBackend, host, ver string) {
4590	t.Helper()
4591	vdir := filepath.Join(b.OptDir, "versions", ver)
4592	if err := os.MkdirAll(vdir, 0o755); err != nil {
4593	t.Fatal(err)
4594	}
4595	bin := filepath.Join(vdir, "app")
4596	if err := os.WriteFile(bin, []byte("#!/bin/sh\n"), 0o755); err != nil {
4597	t.Fatal(err)
4598	}
4599	if err := os.MkdirAll(filepath.Join(b.OptDir, "live"), 0o755); err != nil {
4600	t.Fatal(err)
4601	}
4602	os.Remove(b.liveLink(host))
4603	if err := os.Symlink(bin, b.liveLink(host)); err != nil {
4604	t.Fatal(err)
4605	}
4606	}
4607
4608	// Already-materialized: the common case for every existing instance — the
4609	// Materialize hook is never invoked.
4610	func TestExecBackendLiveLinkAlreadyMaterialized(t *testing.T) {
4611	b := testExecBackend(t)
4612	stageLive(t, b, "existing.example.com", "v1")
4613	called := false
4614	b.Materialize = func(context.Context, string) error { called = true; return nil }
4615	if err := b.ensureLiveLink(context.Background(), "existing.example.com"); err != nil {
4616	t.Fatalf("ensureLiveLink: %v", err)
4617	}
4618	if called {
4619	t.Error("Materialize was called for an already-materialized host")
4620	}
4621	}
4622
4623	// Missing symlink + Materialize configured: routed through the hook, and the
4624	// hook's result is honored.
4625	func TestExecBackendLiveLinkUsesMaterializeWhenConfigured(t *testing.T) {
4626	b := testExecBackend(t)
4627	var gotHost string
4628	b.Materialize = func(_ context.Context, host string) error {
4629	gotHost = host
4630	stageLive(t, b, host, "v1")
4631	return nil
4632	}
4633	if err := b.ensureLiveLink(context.Background(), "seeded.example.com"); err != nil {
4634	t.Fatalf("ensureLiveLink: %v", err)
4635	}
4636	if gotHost != "seeded.example.com" {
4637	t.Fatalf("Materialize called with %q, want seeded.example.com", gotHost)
4638	}
4639	if _, err := os.Stat(b.liveLink("seeded.example.com")); err != nil {
4640	t.Fatalf("live symlink was not created via Materialize: %v", err)
4641	}
4642	}
4643
4644	// A Materialize failure must propagate — the exact live symptom found
4645	// 2026-07-13 in Eleven (a 503 "instance failed to wake"), never silently
4646	// swallowed into "looks fine, wakes anyway."
4647	func TestExecBackendLiveLinkPropagatesMaterializeError(t *testing.T) {
4648	b := testExecBackend(t)
4649	wantErr := fmt.Errorf("controld materialize seeded.example.com: HTTP 502")
4650	b.Materialize = func(context.Context, string) error { return wantErr }
4651	err := b.ensureLiveLink(context.Background(), "seeded.example.com")
4652	if err == nil {
4653	t.Fatal("expected an error, got nil")
4654	}
4655	if !errors.Is(err, wantErr) {
4656	t.Fatalf("got %v, want it to wrap %v", err, wantErr)
4657	}
4658	if _, statErr := os.Stat(b.liveLink("seeded.example.com")); statErr == nil {
4659	t.Fatal("live symlink should not exist after a failed Materialize")
4660	}
4661	}
4662
4663	// No hook and no symlink: an error, loudly. (Eleven self-staged its own
4664	// binary here — instance and control binary were one program; a carlos app is
4665	// a separate binary from the release store, so there is nothing to invent.)
4666	func TestExecBackendLiveLinkMissingWithoutMaterializeFails(t *testing.T) {
4667	b := testExecBackend(t)
4668	if err := b.ensureLiveLink(context.Background(), "ghost.example.com"); err == nil {
4669	t.Fatal("expected an error for a missing live symlink with no Materialize hook")
4670	}
4671	}
4672
4673	func TestExecBackendLiveVersion(t *testing.T) {
4674	b := testExecBackend(t)
4675	stageLive(t, b, "v.example.com", "abc1234")
4676	ver, err := b.LiveVersion("v.example.com")
4677	if err != nil || ver != "abc1234" {
4678	t.Fatalf("LiveVersion = %q, %v; want abc1234", ver, err)
4679	}
4680	// No symlink at all: "" with no error — version management not set up yet.
4681	ver, err = b.LiveVersion("none.example.com")
4682	if err != nil || ver != "" {
4683	t.Fatalf("LiveVersion(no link) = %q, %v; want \"\", nil", ver, err)
4684	}
4685	}
4686
4687	// A migrated tenant's live link points at <opt>/versions/<ver>/<acct>/<app>,
4688	// so reading "the parent directory" returns the ACCOUNT SQID as the version.
4689	// ConvergeDecision compares that against the route's version and would then
4690	// believe every migrated tenant is converged forever — a tenant that silently
4691	// stops taking deploys.
4692	func TestExecBackendLiveVersionOnAccountScopedPath(t *testing.T) {
4693	b := testExecBackend(t)
4694	stageLiveUnderAccount(t, b.OptDir, "kass.example.com", "kqmvexample", "abc1234", "kass")
4695	if ver, err := b.LiveVersion("kass.example.com"); err != nil || ver != "abc1234" {
4696	t.Fatalf("LiveVersion = %q, %v; want abc1234 (the version, not the account sqid)", ver, err)
4697	}
4698	}
4699
4700	// stageLiveUnderAccount builds the account-scoped version-store layout and
4701	// points a host's live symlink at it.
4702	func stageLiveUnderAccount(t *testing.T, optDir, host, acct, ver, app string) {
4703	t.Helper()
4704	dir := filepath.Join(optDir, "versions", ver, acct)
4705	if err := os.MkdirAll(dir, 0o755); err != nil {
4706	t.Fatal(err)
4707	}
4708	if err := os.WriteFile(filepath.Join(dir, app), []byte("bin"), 0o755); err != nil {
4709	t.Fatal(err)
4710	}
4711	if err := os.MkdirAll(filepath.Join(optDir, "live"), 0o755); err != nil {
4712	t.Fatal(err)
4713	}
4714	if err := os.Symlink(filepath.Join(dir, app), filepath.Join(optDir, "live", host)); err != nil {
4715	t.Fatal(err)
4716	}
4717	}
4718
4719	// childEnv strips every CARLOS_* var from the inherited environment (the
4720	// edge's own config must never leak into a child) and then layers the env
4721	// file on top — including that file's own CARLOS_* entries.
4722	func TestChildEnvStripsCarlosVars(t *testing.T) {
4723	t.Setenv("CARLOS_HIBERNATE_BUCKET", "leak-me-not")
4724	t.Setenv("CARLOS_TEST_HARMLESS", "also-me")
4725	t.Setenv("UNRELATED_VAR", "keep")
4726	env := childEnv(map[string]string{"CARLOS_ADMIN_TOKEN": "tok", "FOO": "bar"})
4727	got := map[string]string{}
4728	for _, kv := range env {
4729	if k, v, ok := splitEnv(kv); ok {
4730	got[k] = v
4731	}
4732	}
4733	if _, leaked := got["CARLOS_HIBERNATE_BUCKET"]; leaked {
4734	t.Error("inherited CARLOS_HIBERNATE_BUCKET leaked into the child env")
4735	}
4736	if _, leaked := got["CARLOS_TEST_HARMLESS"]; leaked {
4737	t.Error("inherited CARLOS_TEST_HARMLESS leaked into the child env")
4738	}
4739	if got["UNRELATED_VAR"] != "keep" {
4740	t.Error("non-CARLOS inherited var was stripped")
4741	}
4742	if got["CARLOS_ADMIN_TOKEN"] != "tok" || got["FOO"] != "bar" {
4743	t.Errorf("env file vars not layered: %v", got)
4744	}
4745	}
4746
4747	func splitEnv(kv string) (k, v string, ok bool) {
4748	for i := 0; i < len(kv); i++ {
4749	if kv[i] == '=' {
4750	return kv[:i], kv[i+1:], true
4751	}
4752	}
4753	return "", "", false
4754	}
4755
4756	// mergeEnv layers the sync-owned <host>.config.env UNDER the instance's
4757	// own <host>.env: the instance file wins every conflict, which is what
4758	// makes CARLOS_ADMIN_TOKEN structurally unclobberable from the bucket.
4759	func TestMergeEnvInstanceFileWins(t *testing.T) {
4760	base := map[string]string{"A": "bucket", "B": "bucket-only", "CARLOS_ADMIN_TOKEN": "forged"}
4761	overlay := map[string]string{"A": "instance", "CARLOS_ADMIN_TOKEN": "real"}
4762	got := mergeEnv(base, overlay)
4763	if got["A"] != "instance" || got["B"] != "bucket-only" || got["CARLOS_ADMIN_TOKEN"] != "real" {
4764	t.Fatalf("mergeEnv = %v, want instance values winning and bucket-only surviving", got)
4765	}
4766	// Inputs must not be mutated (withVar's same discipline).
4767	if base["A"] != "bucket" || len(overlay) != 2 {
4768	t.Fatal("mergeEnv mutated an input map")
4769	}
4770	}
4771
4772	func TestMergeEnvNilInputs(t *testing.T) {
4773	if got := mergeEnv(nil, map[string]string{"A": "1"}); got["A"] != "1" {
4774	t.Fatalf("nil base: got %v", got)
4775	}
4776	if got := mergeEnv(map[string]string{"A": "1"}, nil); got["A"] != "1" {
4777	t.Fatalf("nil overlay: got %v", got)
4778	}
4779	if got := mergeEnv(nil, nil); len(got) != 0 {
4780	t.Fatalf("nil/nil: got %v", got)
4781	}
4782	}
4783
4784	// The sleep-time next-due probe: GET /api/next-due over the instance's unix
4785	// socket, authorized with the CARLOS_ADMIN_TOKEN from its env file.
4786	func TestExecBackendNextDue(t *testing.T) {
4787	requireUnixSockets(t)
4788	b := testExecBackend(t)
4789	sock := tmpSock(t)
4790	ln, err := net.Listen("unix", sock)
4791	if err != nil {
4792	t.Fatalf("listen unix: %v", err)
4793	}
4794	srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
4795	if r.URL.Path != "/api/next-due" {
4796	http.NotFound(w, r)
4797	return
4798	}
4799	if r.Header.Get("Authorization") != "Bearer tok" {
4800	http.Error(w, "forbidden", http.StatusForbidden)
4801	return
4802	}
4803	io.WriteString(w, `{"due": 42}`)
4804	})}
4805	go srv.Serve(ln)
4806	t.Cleanup(func() { srv.Close() })
4807
4808	host := "due.example.com"
4809	rt := registry.Route{Host: host, Kind: "instance", Socket: sock, Hibernate: true}
4810
4811	// No env file at all: the probe fails (the state machine records 0).
4812	if _, err := b.NextDue(context.Background(), rt); err == nil {
4813	t.Fatal("NextDue with no env file should error")
4814	}
4815	// Env file without a token: same.
4816	envPath := filepath.Join(b.DataDir, host+".env")
4817	if err := os.WriteFile(envPath, []byte("# no token here\nFOO=bar\n"), 0o600); err != nil {
4818	t.Fatal(err)
4819	}
4820	if _, err := b.NextDue(context.Background(), rt); err == nil {
4821	t.Fatal("NextDue without CARLOS_ADMIN_TOKEN should error")
4822	}
4823	// With the token: the instance's answer comes back.
4824	if err := os.WriteFile(envPath, []byte("CARLOS_ADMIN_TOKEN=tok\n"), 0o600); err != nil {
4825	t.Fatal(err)
4826	}
4827	due, err := b.NextDue(context.Background(), rt)
4828	if err != nil || due != 42 {
4829	t.Fatalf("NextDue = %d, %v; want 42", due, err)
4830	}
4831	}
4832
4833	// TestExecBackendNextDueRefusesAFIFOEnvFile pins NextDue's nested dispatch:
4834	// its env file lives on tenant-writable ground exactly like instanceEnv's
4835	// does, so a FIFO planted there must fail the read fast rather than block
4836	// the probe forever.
4837	func TestExecBackendNextDueRefusesAFIFOEnvFile(t *testing.T) {
4838	b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
4839	instDir := provision.InstanceDir(b.DataDir, rt.Host)
4840	if err := os.MkdirAll(instDir, 0o755); err != nil {
4841	t.Fatalf("mkdir instance dir: %v", err)
4842	}
4843	envPath := filepath.Join(instDir, rt.Host+".env")
4844	if err := unix.Mkfifo(envPath, 0o600); err != nil {
4845	t.Fatalf("Mkfifo: %v", err)
4846	}
4847	done := make(chan error, 1)
4848	go func() {
4849	_, err := b.NextDue(context.Background(), rt)
4850	done <- err
4851	}()
4852	select {
4853	case err := <-done:
4854	if err == nil {
4855	t.Fatal("NextDue against a FIFO env file succeeded, want an error")
4856	}
4857	case <-time.After(2 * time.Second):
4858	t.Fatal("NextDue blocked on a FIFO env file instead of refusing it")
4859	}
4860	}
4861
4862	// HasSidecar is derived from the filesystem — the /data/<host>-sidecar.env
4863	// convention — no registry column to drift from what's actually staged.
4864	func TestExecBackendHasSidecar(t *testing.T) {
4865	b := testExecBackend(t)
4866	if b.HasSidecar(testRoute("plain.example.com")) {
4867	t.Fatal("HasSidecar true with no env file staged")
4868	}
4869	if err := os.WriteFile(filepath.Join(b.DataDir, "bots.example.com-sidecar.env"), []byte("CARLOS_BASE=x\n"), 0o600); err != nil {
4870	t.Fatal(err)
4871	}
4872	if !b.HasSidecar(testRoute("bots.example.com")) {
4873	t.Fatal("HasSidecar false with the sidecar env file staged")
4874	}
4875	}
4876
4877	// StartWorker spawns a child that binds nothing — no socket, no port — so it
4878	// must return the moment the exec succeeds, never waiting on waitHealthyCtx
4879	// (which would dial unix://"" every 150ms for the full spawnHealthTimeout,
4880	// 30s by default in this test, and still fail). The assertion is on elapsed
4881	// wall time against a child that sleeps well past any reasonable spawn
4882	// latency, specifically so a regression that reintroduces a health wait
4883	// fails this test instead of merely running slow.
4884	func TestExecBackendStartWorkerSpawnsWithoutAnyHealthProbe(t *testing.T) {
4885	b := testExecBackend(t)
4886	host := "bot.x.internal"
4887	stageSeedScript(t, b, host, "v1", "sleep 30\n")
4888
4889	rt := registry.Route{Host: host, Kind: "sidecar", Args: `["sidecar","run"]`,
4890	DBPath: filepath.Join(b.DataDir, "bot.db")}
4891
4892	start := time.Now()
4893	p, _, err := b.StartWorker(context.Background(), rt, map[string]string{"CARLOS_SIDECAR_NAME": "bot"})
4894	if err != nil {
4895	t.Fatalf("StartWorker: %v", err)
4896	}
4897	defer p.Stop(context.Background())
4898	if d := time.Since(start); d > 5*time.Second {
4899	t.Fatalf("StartWorker took %v — it waited for a health probe that can never answer", d)
4900	}
4901	if p.Exited() {
4902	t.Fatal("worker exited immediately")
4903	}
4904	}
4905
4906	// A route with no runnable argv has nothing to exec — refusing beats
4907	// spawning the bare binary with no arguments (which for most binaries is
4908	// either a no-op forever or an immediate usage-error crash loop).
4909	func TestExecBackendStartWorkerRefusesEmptyArgs(t *testing.T) {
4910	b := testExecBackend(t)
4911	if _, _, err := b.StartWorker(context.Background(), registry.Route{Host: "b.x.internal", Kind: "sidecar"}, nil); err == nil {
4912	t.Fatal("a worker with no argv has nothing to run — refuse rather than exec the bare binary")
4913	}
4914	}
4915
4916	// Malformed (not valid JSON) argv is refused exactly like empty argv, not
4917	// silently coerced into "run nothing" or "run the bare binary".
4918	func TestExecBackendStartWorkerRefusesMalformedArgs(t *testing.T) {
4919	b := testExecBackend(t)
4920	if _, _, err := b.StartWorker(context.Background(), registry.Route{Host: "b.x.internal", Kind: "sidecar", Args: "not json"}, nil); err == nil {
4921	t.Fatal("malformed args was silently accepted")
4922	}
4923	}
4924
4925	// The env argument is merged OVER the route's own env layers and wins every
4926	// conflict: it carries the per-wake sidecar token, which a tenant's own env
4927	// file must never be able to override.
4928	func TestExecBackendStartWorkerEnvWinsOverTenantFile(t *testing.T) {
4929	b := testExecBackend(t)
4930	host := "worker-env.x.internal"
4931	_, envPath := stageRecordingLiveFullEnv(t, b, host, "v1")
4932
4933	ownEnv := "CARLOS_SIDECAR_TOKEN=tenant-forged\nSHARED=instance\n"
4934	if err := os.WriteFile(filepath.Join(b.DataDir, host+".env"), []byte(ownEnv), 0o600); err != nil {
4935	t.Fatal(err)
4936	}
4937
4938	rt := registry.Route{Host: host, Kind: "sidecar", Args: `["sidecar","run"]`}
4939	p, _, err := b.StartWorker(context.Background(), rt, map[string]string{"CARLOS_SIDECAR_TOKEN": "agent-issued"})
4940	if err != nil {
4941	t.Fatalf("StartWorker: %v", err)
4942	}
4943	defer p.Stop(context.Background())
4944
4945	child := map[string]string{}
4946	for _, line := range strings.Split(waitForFile(t, envPath, time.Second), "\n") {
4947	if k, v, ok := strings.Cut(line, "="); ok {
4948	child[k] = v
4949	}
4950	}
4951	if got := child["CARLOS_SIDECAR_TOKEN"]; got != "agent-issued" {
4952	t.Fatalf("CARLOS_SIDECAR_TOKEN = %q, want %q — the agent-issued token must win over the tenant's own env file", got, "agent-issued")
4953	}
4954	if got := child["SHARED"]; got != "instance" {
4955	t.Fatalf("SHARED = %q, want %q — the route's own env layers must still flow through", got, "instance")
4956	}
4957	}
4958
4959	// TestExecBackendStartWorkerEmitsExitEvent pins the fix for a gap Task 5's
4960	// review carried into Task 15: Start's exit goroutine tells the member
4961	// "instance exited" (warn on a crash, info on exit 0), but StartWorker's
4962	// mirrored StartSidecar's silence instead. That mattered once sidecars got
4963	// their own health words a member reads on the Instances tab (Crashed among
4964	// them) — a crash with no event is a status word with no explanation. A
4965	// nonzero exit must land a warn-level event naming the worker's host.
4966	func TestExecBackendStartWorkerEmitsExitEvent(t *testing.T) {
4967	t.Parallel()
4968	b := testExecBackend(t)
4969	sink := &fakeSink{}
4970	b.Logs = sink
4971	host := "worker-exit.x.internal"
4972	stageSeedScript(t, b, host, "v1", "exit 7\n")
4973
4974	rt := registry.Route{Host: host, Kind: "sidecar", App: "eleven", Args: `["sidecar","run"]`}
4975	p, _, err := b.StartWorker(context.Background(), rt, nil)
4976	if err != nil {
4977	t.Fatalf("StartWorker: %v", err)
4978	}
4979	waitFor(t, "worker exit", p.Exited)
4980
4981	var found bool
4982	for _, e := range sink.events {
4983	if e.host == host && e.level == "warn" && strings.Contains(e.msg, "exited") {
4984	found = true
4985	}
4986	}
4987	if !found {
4988	t.Fatalf("no member-facing exit event for a crashed worker: %+v", sink.events)
4989	}
4990	}
4991
4992	// stageRecordingLive stages host's live binary as a shell script that dumps
4993	// its argv and its CARLOS_CONTROL_SOCKET to two files and exits — the only way
4994	// to see what the child was ACTUALLY handed, argv and environment both.
4995	// Returns the two paths.
4996	func stageRecordingLive(t *testing.T, b *ExecBackend, host, ver string) (argvPath, envPath string) {
4997	t.Helper()
4998	dir := t.TempDir()
4999	argvPath = filepath.Join(dir, "argv")
5000	envPath = filepath.Join(dir, "env")
5001	script := "#!/bin/sh\n" +
5002	"printf '%s\\n' \"$@\" > '" + argvPath + "'\n" +
5003	"printf '%s\\n' \"${CARLOS_CONTROL_SOCKET-<unset>}\" > '" + envPath + "'\n"
5004
5005	vdir := filepath.Join(b.OptDir, "versions", ver)
5006	if err := os.MkdirAll(vdir, 0o755); err != nil {
5007	t.Fatal(err)
5008	}
5009	bin := filepath.Join(vdir, "app")
5010	if err := os.WriteFile(bin, []byte(script), 0o755); err != nil {
5011	t.Fatal(err)
5012	}
5013	if err := os.MkdirAll(filepath.Join(b.OptDir, "live"), 0o755); err != nil {
5014	t.Fatal(err)
5015	}
5016	os.Remove(b.liveLink(host))
5017	if err := os.Symlink(bin, b.liveLink(host)); err != nil {
5018	t.Fatal(err)
5019	}
5020	return argvPath, envPath
5021	}
5022
5023	// stageRecordingLiveFullEnv is stageRecordingLive's sibling: instead of
5024	// picking out CARLOS_CONTROL_SOCKET alone, it dumps the child's ENTIRE
5025	// real environment (`env`) to envPath. TestExecBackendStartMergesConfigEnvUnderOwnEnv
5026	// needs the full dump — it isn't just checking that a var made it into the
5027	// map Start builds, it's checking what the exec'd process itself actually
5028	// saw.
5029	func stageRecordingLiveFullEnv(t *testing.T, b *ExecBackend, host, ver string) (argvPath, envPath string) {
5030	t.Helper()
5031	dir := t.TempDir()
5032	argvPath = filepath.Join(dir, "argv")
5033	envPath = filepath.Join(dir, "env")
5034	script := "#!/bin/sh\n" +
5035	"printf '%s\\n' \"$@\" > '" + argvPath + ".tmp' && mv '" + argvPath + ".tmp' '" + argvPath + "'\n" +
5036	"env > '" + envPath + ".tmp' && mv '" + envPath + ".tmp' '" + envPath + "'\n"
5037
5038	vdir := filepath.Join(b.OptDir, "versions", ver)
5039	if err := os.MkdirAll(vdir, 0o755); err != nil {
5040	t.Fatal(err)
5041	}
5042	bin := filepath.Join(vdir, "app")
5043	if err := os.WriteFile(bin, []byte(script), 0o755); err != nil {
5044	t.Fatal(err)
5045	}
5046	if err := os.MkdirAll(filepath.Join(b.OptDir, "live"), 0o755); err != nil {
5047	t.Fatal(err)
5048	}
5049	os.Remove(b.liveLink(host))
5050	if err := os.Symlink(bin, b.liveLink(host)); err != nil {
5051	t.Fatal(err)
5052	}
5053	return argvPath, envPath
5054	}
5055
5056	// waitForFile polls until path exists (the child writes it concurrently with
5057	// Start's health wait) and returns its contents.
5058	func waitForFile(t *testing.T, path string, within time.Duration) string {
5059	t.Helper()
5060	deadline := time.Now().Add(within)
5061	for time.Now().Before(deadline) {
5062	if b, err := os.ReadFile(path); err == nil {
5063	return string(b)
5064	}
5065	time.Sleep(10 * time.Millisecond)
5066	}
5067	t.Fatalf("%s never appeared within %s", path, within)
5068	return ""
5069	}
5070
5071	// Start hands the spawned instance its control socket through the ENVIRONMENT
5072	// (CARLOS_CONTROL_SOCKET), never as a CLI flag, and binds a listener on it.
5073	// This is the WITH-CACHE case, where that listener also serves the purge
5074	// verb; the nil-cache sibling below covers the socket existing anyway.
5075	//
5076	// The argv assertion is the regression guard: a --control-socket flag makes
5077	// every app binary that predates this feature exit 2 at startup (Go's flag
5078	// package with ExitOnError, which is what app binaries use), i.e. a
5079	// fleet-wide outage the moment the agent upgrades. An unknown environment
5080	// variable is simply ignored by such a binary.
5081	//
5082	// Deviation from the task brief's literal test: the brief asserted the
5083	// control socket file still exists AFTER Start returns, even on the
5084	// health-wait error the fake binary (which never binds rt.Socket) is expected
5085	// to produce here. That's not actually true of the implementation the same
5086	// brief specifies — Start's pre-existing health-wait-failure path calls
5087	// p.Stop() before returning, and Step 3's own childProc.Stop change closes
5088	// the purge listener there, which (per Go's net.Listener.Close on a unix
5089	// socket the listener itself created) unlinks the socket file. So by the time
5090	// Start returns with an error, the file is already gone — confirmed
5091	// empirically, not a guess. This test instead polls for the socket WHILE
5092	// Start's health-wait is still running (i.e. before its internal Stop cleanup
5093	// fires), which is the invariant that's actually guaranteed and is what
5094	// "Start wires up the purge listener" means in practice.
5095	func TestExecBackendStartPassesControlSocketInEnvAndStartsPurgeListener(t *testing.T) {
5096	b := testExecBackend(t)
5097	b.Cache = edge.NewCache(edge.DefaultCacheMaxEntries, edge.DefaultCacheMaxBodyBytes)
5098	argvPath, envPath := stageRecordingLive(t, b, "control-socket.example.com", "v1")
5099
5100	rt := registry.Route{
5101	Host: "control-socket.example.com",
5102	Kind: "instance",
5103	Socket: tmpSock(t),
5104	}
5105	wantControlSocket := rt.Socket + ".control"
5106
5107	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
5108	defer cancel()
5109
5110	done := make(chan struct{})
5111	var proc Proc
5112	go func() {
5113	proc, _, _ = b.Start(ctx, rt) // health-wait error from the fake binary is expected and irrelevant here
5114	close(done)
5115	}()
5116
5117	found := false
5118	deadline := time.Now().Add(1500 * time.Millisecond)
5119	for time.Now().Before(deadline) {
5120	if _, err := os.Stat(wantControlSocket); err == nil {
5121	found = true
5122	break
5123	}
5124	time.Sleep(10 * time.Millisecond)
5125	}
5126
5127	gotArgv := strings.Fields(waitForFile(t, argvPath, time.Second))
5128	gotEnv := strings.TrimSpace(waitForFile(t, envPath, time.Second))
5129
5130	// Everything above was seen while the health wait ran; the fake never
5131	// binds, so end the wait rather than sit out its timeout.
5132	cancel()
5133	<-done
5134	if proc != nil {
5135	defer proc.Stop(context.Background())
5136	}
5137
5138	wantArgv := []string{"--socket", rt.Socket, "--db", filepath.Join(b.DataDir, rt.Host+".db")}
5139	if !slices.Equal(gotArgv, wantArgv) {
5140	t.Fatalf("child argv = %v, want %v — a flag the app binary doesn't parse makes it exit 2 at startup", gotArgv, wantArgv)
5141	}
5142	if gotEnv != wantControlSocket {
5143	t.Fatalf("child CARLOS_CONTROL_SOCKET = %q, want %q", gotEnv, wantControlSocket)
5144	}
5145	if !found {
5146	t.Fatalf("expected control socket %s to exist while Start's health-wait ran", wantControlSocket)
5147	}
5148	}
5149
5150	// The control channel is no longer the purge channel. It is bound for EVERY
5151	// exec-backed child, cache or no cache (spec 2026-08-23-scheduled-work-design
5152	// §7.2 — the scheduled-work verbs are served on it, and a box with no edge
5153	// cache still runs scheduled work), so a nil ExecBackend.Cache no longer means
5154	// "no control socket". What it does still mean is no CARLOS_PURGE: nothing on
5155	// this box can purge anything, and the rule that a variable naming a channel
5156	// must not be set where that channel does nothing is exactly what makes the
5157	// two names diverge here.
5158	//
5159	// The same spawn now also mints the instance's CARLOS_ADMIN_TOKEN before it
5160	// reads the env, so the child is handed the same secret the agent presents
5161	// back to it (NextDue, the tick) and the control socket checks bearers
5162	// against — nothing minted it before this, which is why the next-due probe
5163	// silently failed for every console-created instance.
5164	func TestExecBackendStartWithNilCacheStillBindsControlSocket(t *testing.T) {
5165	b := testExecBackend(t) // b.Cache left nil
5166	host := "no-cache.example.com"
5167	_, envPath := stageRecordingLiveFullEnv(t, b, host, "v1")
5168
5169	rt := registry.Route{
5170	Host: host,
5171	Kind: "instance",
5172	Socket: tmpSock(t),
5173	}
5174	wantControlSocket := rt.Socket + ".control"
5175
5176	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
5177	defer cancel()
5178
5179	// Same shape as the cached case above: poll for the socket WHILE Start's
5180	// health-wait is still running, because the fake binary never binds
5181	// rt.Socket and Start's failure path Stops the child, which unlinks the
5182	// control socket before Start returns.
5183	done := make(chan struct{})
5184	var proc Proc
5185	go func() {
5186	proc, _, _ = b.Start(ctx, rt) // health-wait error from the fake binary is expected here
5187	close(done)
5188	}()
5189
5190	found := false
5191	deadline := time.Now().Add(1500 * time.Millisecond)
5192	for time.Now().Before(deadline) {
5193	if _, err := os.Stat(wantControlSocket); err == nil {
5194	found = true
5195	break
5196	}
5197	time.Sleep(10 * time.Millisecond)
5198	}
5199	child := map[string]string{}
5200	for _, line := range strings.Split(waitForFile(t, envPath, time.Second), "\n") {
5201	if k, v, ok := strings.Cut(line, "="); ok {
5202	child[k] = v
5203	}
5204	}
5205
5206	// Everything above was seen while the health wait ran; the fake never
5207	// binds, so end the wait rather than sit out its timeout.
5208	cancel()
5209	<-done
5210	if proc != nil {
5211	defer proc.Stop(context.Background())
5212	}
5213
5214	if !found {
5215	t.Fatalf("expected control socket %s while Start's health-wait ran — it is bound for every exec child now, cache or not", wantControlSocket)
5216	}
5217	if got := child["CARLOS_CONTROL_SOCKET"]; got != wantControlSocket {
5218	t.Fatalf("child CARLOS_CONTROL_SOCKET = %q, want %q", got, wantControlSocket)
5219	}
5220	if got, ok := child["CARLOS_PURGE"]; ok {
5221	t.Fatalf("child CARLOS_PURGE = %q, want it unset when there is no cache to purge", got)
5222	}
5223
5224	minted, err := readEnvFile(filepath.Join(b.DataDir, host+".env"))
5225	if err != nil {
5226	t.Fatalf("read the instance env file the spawn should have minted into: %v", err)
5227	}
5228	if len(minted["CARLOS_ADMIN_TOKEN"]) != 64 {
5229	t.Fatalf("minted CARLOS_ADMIN_TOKEN = %q, want 64 hex chars", minted["CARLOS_ADMIN_TOKEN"])
5230	}
5231	if got := child["CARLOS_ADMIN_TOKEN"]; got != minted["CARLOS_ADMIN_TOKEN"] {
5232	t.Fatalf("child CARLOS_ADMIN_TOKEN = %q, want the minted %q — the child must hold the same secret the agent presents back to it", got, minted["CARLOS_ADMIN_TOKEN"])
5233	}
5234	}
5235
5236	// fakeTimerStore is the registry subset schedule.Mount needs, stubbed: these
5237	// tests are about the WIRING (are the verbs mounted, is the bearer the
5238	// instance's minted token), not about timer semantics — internal/schedule
5239	// owns those.
5240	type fakeTimerStore struct{ rows []registry.Timer }
5241
5242	func (f *fakeTimerStore) UpsertTimer(t registry.Timer) error { f.rows = append(f.rows, t); return nil }
5243	func (f *fakeTimerStore) DeleteTimer(host, name string) error { return nil }
5244	func (f *fakeTimerStore) GetTimer(host, name string) (*registry.Timer, error) {
5245	return nil, os.ErrNotExist
5246	}
5247	func (f *fakeTimerStore) TimersFor(host string) ([]registry.Timer, error) { return f.rows, nil }
5248	func (f *fakeTimerStore) CountTimers(host, kind string) (int, error) { return len(f.rows), nil }
5249
5250	// startAndReachControlSocket spawns host through the real Start path and hands
5251	// the test the control socket's path plus the token that spawn minted, while
5252	// Start's health-wait is STILL RUNNING — the fake binary never binds rt.Socket,
5253	// so once Start returns its failure path has already Stopped the child and
5254	// unlinked the control socket. Returns a stop func the caller must defer.
5255	func startAndReachControlSocket(t *testing.T, b *ExecBackend, host string) (controlSocket, token string, stop func()) {
5256	t.Helper()
5257	stageRecordingLiveFullEnv(t, b, host, "v1")
5258	rt := registry.Route{Host: host, Kind: "instance", Socket: tmpSock(t)}
5259	controlSocket = rt.Socket + ".control"
5260
5261	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
5262	done := make(chan struct{})
5263	var proc Proc
5264	go func() {
5265	proc, _, _ = b.Start(ctx, rt)
5266	close(done)
5267	}()
5268
5269	deadline := time.Now().Add(3 * time.Second)
5270	for time.Now().Before(deadline) {
5271	if _, err := os.Stat(controlSocket); err == nil {
5272	break
5273	}
5274	time.Sleep(10 * time.Millisecond)
5275	}
5276	env, err := readEnvFile(filepath.Join(b.DataDir, host+".env"))
5277	if err != nil {
5278	cancel()
5279	<-done
5280	t.Fatalf("read minted env file: %v", err)
5281	}
5282	return controlSocket, env["CARLOS_ADMIN_TOKEN"], func() {
5283	cancel()
5284	<-done
5285	if proc != nil {
5286	proc.Stop(context.Background())
5287	}
5288	}
5289	}
5290
5291	// The whole point of the generalised control channel: with ExecBackend.Timers
5292	// set, the scheduled-work verbs are served on the spawned instance's control
5293	// socket, gated on the token THIS spawn minted into <host>.env. The wrong
5294	// bearer is a 401 — which keeps out other UNIX users on the box and any
5295	// unauthenticated dialer, and is all this test claims. It does NOT close the
5296	// same-uid-sibling gap purge.go documents: a sibling shares the agent's uid,
5297	// so it can read another instance's 0600 env file and present that token.
5298	// Per-instance uids are the fix; see purge.go for why the residual is bounded.
5299	func TestExecBackendStartMountsScheduleVerbsOnControlSocket(t *testing.T) {
5300	b := testExecBackend(t) // b.Cache left nil — the schedule verbs do not need one
5301	b.Timers = &fakeTimerStore{}
5302	ctl, token, stop := startAndReachControlSocket(t, b, "timers.example.com")
5303	defer stop()
5304
5305	if len(token) != 64 {
5306	t.Fatalf("minted CARLOS_ADMIN_TOKEN = %q, want 64 hex chars", token)
5307	}
5308
5309	get := func(bearer string) int {
5310	req, err := http.NewRequest("GET", "http://carlos/schedule", nil)
5311	if err != nil {
5312	t.Fatal(err)
5313	}
5314	if bearer != "" {
5315	req.Header.Set("Authorization", "Bearer "+bearer)
5316	}
5317	resp, err := socketClient(ctl).Do(req)
5318	if err != nil {
5319	t.Fatalf("dial control socket %s: %v", ctl, err)
5320	}
5321	defer resp.Body.Close()
5322	return resp.StatusCode
5323	}
5324
5325	if got := get(token); got != http.StatusOK {
5326	t.Fatalf("GET /schedule with the minted token = %d, want 200 — the verbs are not mounted", got)
5327	}
5328	if got := get("not-the-token"); got != http.StatusUnauthorized {
5329	t.Fatalf("GET /schedule with a wrong bearer = %d, want 401 — a same-uid sibling could reach this socket", got)
5330	}
5331	if got := get(""); got != http.StatusUnauthorized {
5332	t.Fatalf("GET /schedule with no bearer = %d, want 401", got)
5333	}
5334	}
5335
5336	// Timers is nil-tolerant, and honestly so: the socket is still bound (the
5337	// channel is per-child now, not per-feature) but nothing answers /schedule.
5338	// A 404 is the truth for an agent that predates the feature; a socket that
5339	// wasn't there at all would make the child's CARLOS_CONTROL_SOCKET a lie.
5340	func TestExecBackendStartWithNilTimersMountsNoScheduleVerbs(t *testing.T) {
5341	b := testExecBackend(t) // b.Timers and b.Cache both nil
5342	ctl, token, stop := startAndReachControlSocket(t, b, "no-timers.example.com")
5343	defer stop()
5344
5345	req, err := http.NewRequest("GET", "http://carlos/schedule", nil)
5346	if err != nil {
5347	t.Fatal(err)
5348	}
5349	req.Header.Set("Authorization", "Bearer "+token)
5350	resp, err := socketClient(ctl).Do(req)
5351	if err != nil {
5352	t.Fatalf("dial control socket %s: %v — it must exist even with no verbs mounted", ctl, err)
5353	}
5354	defer resp.Body.Close()
5355	if resp.StatusCode != http.StatusNotFound {
5356	t.Fatalf("GET /schedule with nil Timers = %d, want 404", resp.StatusCode)
5357	}
5358	}
5359
5360	// TestExecBackendStartMergesConfigEnvUnderOwnEnv is the load-bearing
5361	// merge-precedence regression: unlike a unit test of mergeEnv in isolation,
5362	// this goes through the real ExecBackend.Start spawn path and inspects the
5363	// spawned process's ACTUAL environment (via `env`, dumped by the recorder
5364	// script) — the only way to see what the child was really handed.
5365	//
5366	// It stages both <host>.config.env (sync-owned, bucket-derived —
5367	// CARLOS_ADMIN_TOKEN=forged, SHARED=bucket, BUCKET_ONLY=1) and <host>.env
5368	// (the instance's own file — CARLOS_ADMIN_TOKEN=real, SHARED=instance) and
5369	// asserts the child saw the instance's own values winning the conflicts
5370	// while the bucket-only var still flowed through. This must fail if
5371	// Start's `mergeEnv(cfgEnv, ownEnv)` call is ever transposed to
5372	// `mergeEnv(ownEnv, cfgEnv)`: that would let bucket-owned config forge
5373	// CARLOS_ADMIN_TOKEN, a secret held only in the instance's own file.
5374	// Verified empirically by transposing the call and watching this test
5375	// fail, then restoring it (see the fix report for the RED output).
5376	func TestExecBackendStartMergesConfigEnvUnderOwnEnv(t *testing.T) {
5377	b := testExecBackend(t)
5378	host := "merge-precedence.example.com"
5379	_, envPath := stageRecordingLiveFullEnv(t, b, host, "v1")
5380
5381	cfgEnv := "CARLOS_ADMIN_TOKEN=forged\nSHARED=bucket\nBUCKET_ONLY=1\n"
5382	if err := os.WriteFile(filepath.Join(b.DataDir, host+".config.env"), []byte(cfgEnv), 0o600); err != nil {
5383	t.Fatal(err)
5384	}
5385	ownEnv := "CARLOS_ADMIN_TOKEN=real\nSHARED=instance\n"
5386	if err := os.WriteFile(filepath.Join(b.DataDir, host+".env"), []byte(ownEnv), 0o600); err != nil {
5387	t.Fatal(err)
5388	}
5389
5390	rt := registry.Route{
5391	Host: host,
5392	Kind: "instance",
5393	Socket: filepath.Join(t.TempDir(), "instance.sock"),
5394	}
5395
5396	ctx, cancel := recordedCtx(2*time.Second, envPath)
5397	defer cancel()
5398	proc, _, _ := b.Start(ctx, rt) // health-wait error from the fake binary is expected and irrelevant here
5399	if proc != nil {
5400	defer proc.Stop(context.Background())
5401	}
5402
5403	child := map[string]string{}
5404	for _, line := range strings.Split(waitForFile(t, envPath, time.Second), "\n") {
5405	if k, v, ok := strings.Cut(line, "="); ok {
5406	child[k] = v
5407	}
5408	}
5409	if got := child["CARLOS_ADMIN_TOKEN"]; got != "real" {
5410	t.Fatalf("child CARLOS_ADMIN_TOKEN = %q, want %q — the instance's own env must win over bucket config", got, "real")
5411	}
5412	if got := child["SHARED"]; got != "instance" {
5413	t.Fatalf("child SHARED = %q, want %q — the instance's own env must win over bucket config", got, "instance")
5414	}
5415	if got := child["BUCKET_ONLY"]; got != "1" {
5416	t.Fatalf("child BUCKET_ONLY = %q, want %q — bucket config must still flow through when not overridden", got, "1")
5417	}
5418	}
5419
5420	// CARLOS_PURGE=header is the advisory half of the response-header purge
5421	// channel (internal/edge/purge_header.go): it tells the app this box's edge
5422	// reads Carlos-Purge, so the app can log or branch on it. It is ADDITIVE —
5423	// CARLOS_CONTROL_SOCKET must still be there beside it, because the header is
5424	// an additional channel and not a replacement — and it must never be a CLI
5425	// flag, for the same flag.ExitOnError reason the control socket isn't.
5426	//
5427	// The gate matches CARLOS_CONTROL_SOCKET's (b.Cache != nil, i.e. "there is
5428	// actually a cache to purge"), which is the same no-inert-feature discipline:
5429	// a variable naming a channel must not be set on a box where that channel
5430	// does nothing.
5431	func TestExecBackendStartAdvertisesHeaderPurgeChannel(t *testing.T) {
5432	b := testExecBackend(t)
5433	b.Cache = edge.NewCache(edge.DefaultCacheMaxEntries, edge.DefaultCacheMaxBodyBytes)
5434	host := "purge-header.example.com"
5435	argvPath, envPath := stageRecordingLiveFullEnv(t, b, host, "v1")
5436
5437	rt := registry.Route{
5438	Host: host,
5439	Kind: "instance",
5440	Socket: tmpSock(t),
5441	}
5442
5443	ctx, cancel := recordedCtx(2*time.Second, envPath)
5444	defer cancel()
5445	proc, _, _ := b.Start(ctx, rt) // health-wait error from the fake binary is expected and irrelevant here
5446	if proc != nil {
5447	defer proc.Stop(context.Background())
5448	}
5449
5450	child := map[string]string{}
5451	for _, line := range strings.Split(waitForFile(t, envPath, time.Second), "\n") {
5452	if k, v, ok := strings.Cut(line, "="); ok {
5453	child[k] = v
5454	}
5455	}
5456	if got := child["CARLOS_PURGE"]; got != "header" {
5457	t.Fatalf("child CARLOS_PURGE = %q, want %q", got, "header")
5458	}
5459	if got := child["CARLOS_CONTROL_SOCKET"]; got != rt.Socket+".control" {
5460	t.Fatalf("child CARLOS_CONTROL_SOCKET = %q, want %q — the header channel is ADDITIONAL, it does not replace the socket", got, rt.Socket+".control")
5461	}
5462	gotArgv := strings.Fields(waitForFile(t, argvPath, time.Second))
5463	wantArgv := []string{"--socket", rt.Socket, "--db", filepath.Join(b.DataDir, rt.Host+".db")}
5464	if !slices.Equal(gotArgv, wantArgv) {
5465	t.Fatalf("child argv = %v, want %v — a flag the app binary doesn't parse makes it exit 2 at startup", gotArgv, wantArgv)
5466	}
5467	}
5468
5469	// The advisory var is gated on there being a cache at all: with none, there
5470	// is no purge channel to advertise and setting the var would be a lie.
5471	func TestExecBackendStartWithNilCacheSetsNoPurgeVar(t *testing.T) {
5472	b := testExecBackend(t) // b.Cache left nil
5473	host := "no-cache-purge.example.com"
5474	_, envPath := stageRecordingLiveFullEnv(t, b, host, "v1")
5475
5476	rt := registry.Route{
5477	Host: host,
5478	Kind: "instance",
5479	Socket: filepath.Join(t.TempDir(), "instance.sock"),
5480	}
5481
5482	ctx, cancel := recordedCtx(2*time.Second, envPath)
5483	defer cancel()
5484	proc, _, _ := b.Start(ctx, rt)
5485	if proc != nil {
5486	defer proc.Stop(context.Background())
5487	}
5488
5489	for _, line := range strings.Split(waitForFile(t, envPath, time.Second), "\n") {
5490	if k, _, ok := strings.Cut(line, "="); ok && k == "CARLOS_PURGE" {
5491	t.Fatalf("child saw %q with no cache configured", line)
5492	}
5493	}
5494	}
5495
5496	// TestExecBackendStartTenantCannotClaimAbsentPurgeChannel closes the hole the
5497	// gate alone left open. fileEnv is TENANT-supplied — <host>.config.env arrives
5498	// from the bucket via `carlos env`, <host>.env is the instance's own file — so
5499	// gating only on "does the agent SET the var" left a tenant free to set it
5500	// itself. On a box with no cache the child would then be told a purge channel
5501	// exists when none does, and an app that trusts CARLOS_PURGE would skip its
5502	// own invalidation and serve stale content indefinitely. CARLOS_PURGE is
5503	// therefore REMOVED here.
5504	//
5505	// CARLOS_CONTROL_SOCKET is the same hole and a worse one — a tenant-chosen
5506	// socket path is a name the agent never vouched for — but its channel now
5507	// exists on this box (bound for every exec child, §7.2), so the fix is
5508	// OVERRIDE rather than removal: the child must be pointed at the socket the
5509	// agent actually bound, never at /tmp/attacker.sock.
5510	func TestExecBackendStartTenantCannotClaimAbsentPurgeChannel(t *testing.T) {
5511	b := testExecBackend(t) // b.Cache left nil — no purge channel on this box
5512	host := "liar.example.com"
5513	_, envPath := stageRecordingLiveFullEnv(t, b, host, "v1")
5514
5515	// The tenant declares both channels in its own config.
5516	tenantEnv := "CARLOS_PURGE=header\nCARLOS_CONTROL_SOCKET=/tmp/attacker.sock\nKEPT=1\n"
5517	if err := os.WriteFile(filepath.Join(b.DataDir, host+".config.env"), []byte(tenantEnv), 0o600); err != nil {
5518	t.Fatal(err)
5519	}
5520
5521	rt := registry.Route{
5522	Host: host,
5523	Kind: "instance",
5524	Socket: tmpSock(t),
5525	}
5526
5527	ctx, cancel := recordedCtx(2*time.Second, envPath)
5528	defer cancel()
5529	proc, _, _ := b.Start(ctx, rt)
5530	if proc != nil {
5531	defer proc.Stop(context.Background())
5532	}
5533
5534	child := map[string]string{}
5535	for _, line := range strings.Split(waitForFile(t, envPath, time.Second), "\n") {
5536	if k, v, ok := strings.Cut(line, "="); ok {
5537	child[k] = v
5538	}
5539	}
5540	if got, ok := child["CARLOS_PURGE"]; ok {
5541	t.Fatalf("tenant-supplied CARLOS_PURGE=%q survived onto a box with no cache", got)
5542	}
5543	if want := rt.Socket + ".control"; child["CARLOS_CONTROL_SOCKET"] != want {
5544	t.Fatalf("child CARLOS_CONTROL_SOCKET = %q, want %q — a tenant must not redirect the control socket, cache or no cache",
5545	child["CARLOS_CONTROL_SOCKET"], want)
5546	}
5547	// Only the agent-owned names are stripped; ordinary tenant config is
5548	// untouched.
5549	if got := child["KEPT"]; got != "1" {
5550	t.Fatalf("child KEPT = %q, want %q — only the agent-owned vars may be removed", got, "1")
5551	}
5552	}
5553
5554	// The other branch: with a cache, the agent's values must OVERRIDE whatever
5555	// the tenant supplied, not merely be absent-or-present. A tenant that writes
5556	// CARLOS_CONTROL_SOCKET=/tmp/attacker.sock must not have the child pointed at
5557	// a socket the agent did not create.
5558	func TestExecBackendStartOverridesTenantSuppliedPurgeVars(t *testing.T) {
5559	b := testExecBackend(t)
5560	b.Cache = edge.NewCache(edge.DefaultCacheMaxEntries, edge.DefaultCacheMaxBodyBytes)
5561	host := "override.example.com"
5562	_, envPath := stageRecordingLiveFullEnv(t, b, host, "v1")
5563
5564	tenantEnv := "CARLOS_PURGE=socket-only\nCARLOS_CONTROL_SOCKET=/tmp/attacker.sock\n"
5565	if err := os.WriteFile(filepath.Join(b.DataDir, host+".config.env"), []byte(tenantEnv), 0o600); err != nil {
5566	t.Fatal(err)
5567	}
5568
5569	rt := registry.Route{
5570	Host: host,
5571	Kind: "instance",
5572	Socket: tmpSock(t),
5573	}
5574
5575	ctx, cancel := recordedCtx(2*time.Second, envPath)
5576	defer cancel()
5577	proc, _, _ := b.Start(ctx, rt)
5578	if proc != nil {
5579	defer proc.Stop(context.Background())
5580	}
5581
5582	child := map[string]string{}
5583	for _, line := range strings.Split(waitForFile(t, envPath, time.Second), "\n") {
5584	if k, v, ok := strings.Cut(line, "="); ok {
5585	child[k] = v
5586	}
5587	}
5588	if got := child["CARLOS_PURGE"]; got != "header" {
5589	t.Fatalf("child CARLOS_PURGE = %q, want %q — the agent's value must win over the tenant's", got, "header")
5590	}
5591	if want := rt.Socket + ".control"; child["CARLOS_CONTROL_SOCKET"] != want {
5592	t.Fatalf("child CARLOS_CONTROL_SOCKET = %q, want %q — a tenant must not redirect the control socket",
5593	child["CARLOS_CONTROL_SOCKET"], want)
5594	}
5595	}
5596
5597	// Spawning a fresh process for a host is a deploy boundary: the code behind
5598	// the host just changed, so every response cached from the old process is
5599	// stale. Start drops them all before the new process serves anything —
5600	// without this, a redeploy keeps serving old HTML until the TTL expires.
5601	func TestExecBackendStartPurgesHostCacheOnSpawn(t *testing.T) {
5602	b := testExecBackend(t)
5603	cache := edge.NewCache(edge.DefaultCacheMaxEntries, edge.DefaultCacheMaxBodyBytes)
5604	b.Cache = cache
5605	stageLive(t, b, "deploy.example.com", "v1")
5606
5607	stale := edge.CacheKey{Host: "deploy.example.com", Method: "GET", Path: "/event"}
5608	otherHost := edge.CacheKey{Host: "bystander.example.com", Method: "GET", Path: "/event"}
5609	for _, k := range []edge.CacheKey{stale, otherHost} {
5610	cache.Put(k, "", &edge.CachedResponse{
5611	StatusCode: 200, Header: http.Header{}, Body: []byte("old html"),
5612	StoredAt: time.Now(), TTL: time.Minute,
5613	})
5614	}
5615
5616	// GetFor is the cache's request-time lookup (the by-VaryValue one is
5617	// package-internal); both entries were stored from responses with no Vary,
5618	// so any request for the URL reaches them.
5619	cached := func(k edge.CacheKey) bool {
5620	_, ok := cache.GetFor(k, httptest.NewRequest("GET", "http://"+k.Host+k.Path, nil))
5621	return ok
5622	}
5623	// The purge comes before the health wait, which the fake never answers:
5624	// once it has happened that wait has nothing left to show.
5625	ctx, cancel := ctxUntil(2*time.Second, func() bool { return !cached(stale) })
5626	defer cancel()
5627	rt := registry.Route{
5628	Host: "deploy.example.com",
5629	Kind: "instance",
5630	Socket: filepath.Join(t.TempDir(), "instance.sock"),
5631	}
5632	proc, _, _ := b.Start(ctx, rt) // health-wait failure is expected; the spawn still happened
5633	if proc != nil {
5634	defer proc.Stop(context.Background())
5635	}
5636
5637	if cached(stale) {
5638	t.Fatal("spawning a fresh process for a host must drop that host's cached responses")
5639	}
5640	if !cached(otherHost) {
5641	t.Fatal("another host's cached responses must survive a spawn")
5642	}
5643	}
5644
5645	// --- ExecBackend: fresh detection + seed --------------------------------------
5646
5647	// stageSeedScript stages host's live binary as a shell script with a chosen
5648	// body — the seed tests need exits and stderr the recording scripts above
5649	// don't produce. Returns nothing: each test knows the files its body writes.
5650	func stageSeedScript(t *testing.T, b *ExecBackend, host, ver, body string) {
5651	t.Helper()
5652	vdir := filepath.Join(b.OptDir, "versions", ver)
5653	if err := os.MkdirAll(vdir, 0o755); err != nil {
5654	t.Fatal(err)
5655	}
5656	bin := filepath.Join(vdir, "app")
5657	if err := os.WriteFile(bin, []byte("#!/bin/sh\n"+body), 0o755); err != nil {
5658	t.Fatal(err)
5659	}
5660	if err := os.MkdirAll(filepath.Join(b.OptDir, "live"), 0o755); err != nil {
5661	t.Fatal(err)
5662	}
5663	os.Remove(b.liveLink(host))
5664	if err := os.Symlink(bin, b.liveLink(host)); err != nil {
5665	t.Fatal(err)
5666	}
5667	}
5668
5669	// The warm cache — the local DB still on disk from the last sleep — is never a
5670	// first boot, and the bucket isn't even consulted.
5671	func TestExecBackendRestoreWarmCacheIsNotFresh(t *testing.T) {
5672	b := testExecBackend(t)
5673	host := "warm.example.com"
5674	if err := os.WriteFile(filepath.Join(b.DataDir, host+".db"), []byte("db"), 0o600); err != nil {
5675	t.Fatal(err)
5676	}
5677	called := false
5678	swapRestoreReplica(t, func(context.Context, string, string, string, string) (bool, error) {
5679	called = true
5680	return false, nil
5681	})
5682
5683	fresh, err := b.Restore(context.Background(), registry.Route{Host: host, Kind: "instance"})
5684	if err != nil {
5685	t.Fatalf("Restore: %v", err)
5686	}
5687	if fresh {
5688	t.Fatal("warm cache reported fresh — the seed would run over live data")
5689	}
5690	if called {
5691	t.Fatal("warm cache consulted the bucket")
5692	}
5693	}
5694
5695	// DB absent but a replica exists: the data came back from S3, so this is a
5696	// restore, not a first boot.
5697	func TestExecBackendRestoreFromReplicaIsNotFresh(t *testing.T) {
5698	b := testExecBackend(t)
5699	swapRestoreReplica(t, func(_ context.Context, dbPath, _, _, _ string) (bool, error) {
5700	writeUsableDB(t, dbPath)
5701	return true, nil
5702	})
5703
5704	fresh, err := b.Restore(context.Background(), registry.Route{Host: "restored.example.com", Kind: "instance"})
5705	if err != nil {
5706	t.Fatalf("Restore: %v", err)
5707	}
5708	if fresh {
5709	t.Fatal("a restored instance reported fresh — a seed here would overwrite recovered data")
5710	}
5711	}
5712
5713	// A replica that comes back DAMAGED must fail the wake, not boot. Two things
5714	// are on the line: `fresh` must stay false (a fresh verdict runs the route's
5715	// seed, writing an empty database over recoverable data), and the bad file
5716	// must not be left where the next wake's warm-cache branch will serve it
5717	// unchecked.
5718	func TestExecBackendRestoreCorruptReplicaFailsAndQuarantines(t *testing.T) {
5719	b := testExecBackend(t)
5720	host := "damaged.example.com"
5721	dbPath := filepath.Join(b.DataDir, host+".db")
5722
5723	swapRestoreReplica(t, func(_ context.Context, out, _, _, _ string) (bool, error) {
5724	if err := os.WriteFile(out, []byte("this is not a sqlite database at all"), 0o600); err != nil {
5725	t.Fatal(err)
5726	}
5727	return true, nil
5728	})
5729
5730	fresh, err := b.Restore(context.Background(), registry.Route{Host: host, Kind: "instance"})
5731	if err == nil {
5732	t.Fatal("Restore accepted a corrupt replica")
5733	}
5734	if fresh {
5735	t.Fatal("a corrupt restore reported fresh — the seed would run over recoverable data")
5736	}
5737	if _, statErr := os.Stat(dbPath); statErr == nil {
5738	t.Fatal("the corrupt file was left at the live path — the next wake would serve it as a warm cache")
5739	}
5740	if _, statErr := os.Stat(dbPath + quarantineSuffix); statErr != nil {
5741	t.Fatalf("no quarantined copy for the operator to look at: %v", statErr)
5742	}
5743
5744	// And the next wake must genuinely re-attempt the restore rather than
5745	// short-circuit on leftover state.
5746	called := false
5747	swapRestoreReplica(t, func(_ context.Context, out, _, _, _ string) (bool, error) {
5748	called = true
5749	writeUsableDB(t, out)
5750	return true, nil
5751	})
5752	fresh, err = b.Restore(context.Background(), registry.Route{Host: host, Kind: "instance"})
5753	if err != nil {
5754	t.Fatalf("second Restore: %v", err)
5755	}
5756	if !called {
5757	t.Fatal("the second wake never consulted the bucket")
5758	}
5759	if fresh {
5760	t.Fatal("the recovered instance reported fresh")
5761	}
5762	}
5763
5764	// writeUsableDB creates the kind of file a healthy restore produces: a real
5765	// SQLite database with a schema, which is what CheckDB demands.
5766	func writeUsableDB(t *testing.T, path string) {
5767	t.Helper()
5768	db, err := sql.Open("sqlite", path)
5769	if err != nil {
5770	t.Fatal(err)
5771	}
5772	defer db.Close()
5773	if _, err := db.Exec("CREATE TABLE t (id INTEGER PRIMARY KEY); INSERT INTO t VALUES (1)"); err != nil {
5774	t.Fatal(err)
5775	}
5776	}
5777
5778	// DB absent AND no replica in the bucket: the one true first boot.
5779	func TestExecBackendRestoreNoReplicaIsFresh(t *testing.T) {
5780	b := testExecBackend(t)
5781	swapRestoreReplica(t, func(context.Context, string, string, string, string) (bool, error) {
5782	return false, nil
5783	})
5784
5785	fresh, err := b.Restore(context.Background(), registry.Route{Host: "brand-new.example.com", Kind: "instance"})
5786	if err != nil {
5787	t.Fatalf("Restore: %v", err)
5788	}
5789	if !fresh {
5790	t.Fatal("a never-replicated host did not report fresh — it would never be seeded")
5791	}
5792	}
5793
5794	// A restore ERROR is not a first boot: "we couldn't tell" must never be
5795	// reported as "there was nothing there".
5796	func TestExecBackendRestoreErrorIsNotFresh(t *testing.T) {
5797	b := testExecBackend(t)
5798	swapRestoreReplica(t, func(context.Context, string, string, string, string) (bool, error) {
5799	return false, errors.New("s3: connection refused")
5800	})
5801
5802	fresh, err := b.Restore(context.Background(), registry.Route{Host: "unreachable.example.com", Kind: "instance"})
5803	if err == nil {
5804	t.Fatal("Restore swallowed the replica error")
5805	}
5806	if fresh {
5807	t.Fatal("a failed restore reported fresh")
5808	}
5809	}
5810
5811	// swapRestoreReplica points the replica-restore seam at a stub for one test.
5812	func swapRestoreReplica(t *testing.T, fn func(ctx context.Context, dbPath, bucket, prefix, region string) (bool, error)) {
5813	t.Helper()
5814	prev := restoreReplica
5815	restoreReplica = fn
5816	t.Cleanup(func() { restoreReplica = prev })
5817	}
5818
5819	// The seed contract, end to end through a real exec: the app's declared argv
5820	// followed by --db <path>, under the instance's own merged env layers, with
5821	// the agent's CARLOS_* vars scrubbed and the purge/control vars (Start's, not
5822	// the seed's) absent.
5823	func TestExecBackendSeedRunsDeclaredArgvWithDBPath(t *testing.T) {
5824	b := testExecBackend(t)
5825	b.Cache = edge.NewCache(edge.DefaultCacheMaxEntries, edge.DefaultCacheMaxBodyBytes)
5826	host := "seeded.example.com"
5827	argvPath, envPath := stageRecordingLiveFullEnv(t, b, host, "v1")
5828
5829	cfgEnv := "SHARED=bucket\nBUCKET_ONLY=1\n"
5830	if err := os.WriteFile(filepath.Join(b.DataDir, host+".config.env"), []byte(cfgEnv), 0o600); err != nil {
5831	t.Fatal(err)
5832	}
5833	// The tenant's own env file claims the purge channel. It is not the
5834	// tenant's to claim in either direction — those two names are
5835	// agent-owned (see withoutVars), and a seed told a control socket
5836	// exists at a path the agent never vouched for is exactly the leak the
5837	// spawn path's else-branch closes.
5838	ownEnv := "SHARED=instance\nCARLOS_ADMIN_TOKEN=real\nCARLOS_PURGE=header\nCARLOS_CONTROL_SOCKET=/tmp/tenant-chosen.sock\n"
5839	if err := os.WriteFile(filepath.Join(b.DataDir, host+".env"), []byte(ownEnv), 0o600); err != nil {
5840	t.Fatal(err)
5841	}
5842	t.Setenv("CARLOS_HIBERNATE_BUCKET", "agent-only")
5843
5844	rt := registry.Route{Host: host, Kind: "instance", Seed: `["seed","--fixtures"]`}
5845	if err := b.Seed(context.Background(), rt); err != nil {
5846	t.Fatalf("Seed: %v", err)
5847	}
5848
5849	gotArgv := strings.Fields(waitForFile(t, argvPath, time.Second))
5850	wantArgv := []string{"seed", "--fixtures", "--db", filepath.Join(b.DataDir, host+".db")}
5851	if !slices.Equal(gotArgv, wantArgv) {
5852	t.Fatalf("seed argv = %v, want %v", gotArgv, wantArgv)
5853	}
5854
5855	child := map[string]string{}
5856	for _, line := range strings.Split(waitForFile(t, envPath, time.Second), "\n") {
5857	if k, v, ok := strings.Cut(line, "="); ok {
5858	child[k] = v
5859	}
5860	}
5861	if got := child["SHARED"]; got != "instance" {
5862	t.Fatalf("seed SHARED = %q, want %q — the instance's own env must win over bucket config", got, "instance")
5863	}
5864	if got := child["BUCKET_ONLY"]; got != "1" {
5865	t.Fatalf("seed BUCKET_ONLY = %q, want %q — bucket config must flow through", got, "1")
5866	}
5867	if got := child["CARLOS_ADMIN_TOKEN"]; got != "real" {
5868	t.Fatalf("seed CARLOS_ADMIN_TOKEN = %q, want %q — the seed gets the instance's own env file", got, "real")
5869	}
5870	if got := child["CARLOS_HOST"]; got != host {
5871	t.Fatalf("seed CARLOS_HOST = %q, want %q (#256: the seed is told its host like the instance is)", got, host)
5872	}
5873	if _, ok := child["CARLOS_HIBERNATE_BUCKET"]; ok {
5874	t.Fatal("the agent's own CARLOS_* config leaked into the seed process")
5875	}
5876	if _, ok := child["CARLOS_CONTROL_SOCKET"]; ok {
5877	t.Fatal("seed was handed a control socket — the purge channel belongs to the serving process, not a one-shot seed")
5878	}
5879	if _, ok := child["CARLOS_PURGE"]; ok {
5880	t.Fatal("seed was told a purge channel exists — that layering belongs to Start")
5881	}
5882	}
5883
5884	// The route's own DB path wins over the <DataDir>/<host>.db default, exactly
5885	// as it does for the spawn.
5886	func TestExecBackendSeedHonoursRouteDBPath(t *testing.T) {
5887	b := testExecBackend(t)
5888	host := "custom-db.example.com"
5889	argvPath, _ := stageRecordingLiveFullEnv(t, b, host, "v1")
5890	dbPath := filepath.Join(t.TempDir(), "elsewhere.db")
5891
5892	if err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", DBPath: dbPath, Seed: `["seed"]`}); err != nil {
5893	t.Fatalf("Seed: %v", err)
5894	}
5895	gotArgv := strings.Fields(waitForFile(t, argvPath, time.Second))
5896	wantArgv := []string{"seed", "--db", dbPath}
5897	if !slices.Equal(gotArgv, wantArgv) {
5898	t.Fatalf("seed argv = %v, want %v", gotArgv, wantArgv)
5899	}
5900	}
5901
5902	// A seed that exits non-zero fails, naming the host, the exit state, and what
5903	// it printed — the wake dies with it (see doWake), so this error is the whole
5904	// diagnosis an operator gets.
5905	func TestExecBackendSeedNonZeroExitFails(t *testing.T) {
5906	b := testExecBackend(t)
5907	host := "badseed.example.com"
5908	stageSeedScript(t, b, host, "v1", "echo 'schema is corrupt' >&2\nexit 1\n")
5909
5910	err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`})
5911	if err == nil {
5912	t.Fatal("a seed exiting 1 was reported as success")
5913	}
5914	for _, want := range []string{host, "exit status 1", "schema is corrupt"} {
5915	if !strings.Contains(err.Error(), want) {
5916	t.Fatalf("seed error = %q, want it to contain %q", err, want)
5917	}
5918	}
5919	}
5920
5921	// Unparseable seed argv is an ERROR, never a silent skip: a route whose seed
5922	// column got mangled must fail loudly at the first wake rather than boot an
5923	// empty instance that looks fine.
5924	func TestExecBackendSeedRejectsMalformedArgv(t *testing.T) {
5925	b := testExecBackend(t)
5926	host := "malformed.example.com"
5927	argvPath, _ := stageRecordingLiveFullEnv(t, b, host, "v1")
5928
5929	err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `seed --fixtures`})
5930	if err == nil {
5931	t.Fatal("malformed seed argv was silently skipped")
5932	}
5933	if !strings.Contains(err.Error(), host) {
5934	t.Fatalf("seed parse error = %q, want it to name the host", err)
5935	}
5936	if _, statErr := os.Stat(argvPath); statErr == nil {
5937	t.Fatal("the binary was execed despite unparseable seed argv")
5938	}
5939	}
5940
5941	// An empty argv array is a no-op, not an exec of the bare binary (defensive:
5942	// doWake only calls Seed for a non-empty route seed).
5943	func TestExecBackendSeedEmptyArgvRunsNothing(t *testing.T) {
5944	b := testExecBackend(t)
5945	host := "empty.example.com"
5946	argvPath, _ := stageRecordingLiveFullEnv(t, b, host, "v1")
5947
5948	if err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `[]`}); err != nil {
5949	t.Fatalf("empty seed argv: %v", err)
5950	}
5951	if _, statErr := os.Stat(argvPath); statErr == nil {
5952	t.Fatal("an empty seed argv execed the binary")
5953	}
5954	}
5955
5956	// A seed that never returns is bounded by seedTimeout — a wedged seed must
5957	// fail the wake, not hold the lease until the wake's own 5m budget expires.
5958	func TestExecBackendSeedIsTimeBounded(t *testing.T) {
5959	b := testExecBackend(t)
5960	host := "hangs.example.com"
5961	stageSeedScript(t, b, host, "v1", "sleep 30\n")
5962
5963	prevTimeout, prevGrace := seedTimeout, seedDrainGrace
5964	seedTimeout, seedDrainGrace = 200*time.Millisecond, 200*time.Millisecond
5965	t.Cleanup(func() { seedTimeout, seedDrainGrace = prevTimeout, prevGrace })
5966
5967	start := time.Now()
5968	err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`})
5969	if err == nil {
5970	t.Fatal("a hanging seed was reported as success")
5971	}
5972	if !strings.Contains(err.Error(), "timed out") {
5973	t.Fatalf("seed error = %q, want it to say the seed timed out", err)
5974	}
5975	// The bound that matters: the script sleeps 30s, and a seed still running
5976	// when Seed returns would be holding the lease behind a wake that already
5977	// failed.
5978	if elapsed := time.Since(start); elapsed > 5*time.Second {
5979	t.Fatalf("seed took %s — the timeout did not bound it", elapsed)
5980	}
5981	}
5982
5983	// A missing live binary is the same loud failure the spawn path gives: the
5984	// seed goes through ensureLiveLink, so a host with nothing staged never
5985	// silently "seeds".
5986	func TestExecBackendSeedRequiresLiveBinary(t *testing.T) {
5987	b := testExecBackend(t)
5988	err := b.Seed(context.Background(), registry.Route{Host: "unstaged.example.com", Kind: "instance", Seed: `["seed"]`})
5989	if err == nil {
5990	t.Fatal("seed with no live binary reported success")
5991	}
5992	if !strings.Contains(err.Error(), "unstaged.example.com") {
5993	t.Fatalf("seed error = %q, want it to name the host", err)
5994	}
5995	}
5996
5997	// A crashed seed takes its half-built database with it. Otherwise the next
5998	// wake stats a file that exists, calls it a warm cache, reports NOT fresh, and
5999	// boots the instance on a half-seeded database — the exact outcome failing the
6000	// wake was meant to prevent.
6001	func TestExecBackendSeedFailureRemovesHalfBuiltDB(t *testing.T) {
6002	b := testExecBackend(t)
6003	host := "crashed.example.com"
6004	// argv is: seed --db <path> → $3 is the database path.
6005	stageSeedScript(t, b, host, "v1", "printf half > \"$3\"\necho 'died mid-schema' >&2\nexit 1\n")
6006
6007	if err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`}); err == nil {
6008	t.Fatal("a seed exiting 1 was reported as success")
6009	}
6010	if _, err := os.Stat(filepath.Join(b.DataDir, host+".db")); err == nil {
6011	t.Fatal("a failed seed left its half-built database behind — the next wake would treat it as a warm cache")
6012	}
6013	}
6014
6015	// The cleanup is guarded: a database that already existed when Seed was called
6016	// is never ours to delete (the wake path can't produce this — Restore reported
6017	// fresh — but a direct call must not be able to destroy live data).
6018	func TestExecBackendSeedFailureKeepsPreexistingDB(t *testing.T) {
6019	b := testExecBackend(t)
6020	host := "existing-db.example.com"
6021	stageSeedScript(t, b, host, "v1", "echo nope >&2\nexit 1\n")
6022	if err := os.WriteFile(filepath.Join(b.DataDir, host+".db"), []byte("live data"), 0o600); err != nil {
6023	t.Fatal(err)
6024	}
6025
6026	if err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`}); err == nil {
6027	t.Fatal("a seed exiting 1 was reported as success")
6028	}
6029	got, err := os.ReadFile(filepath.Join(b.DataDir, host+".db"))
6030	if err != nil || string(got) != "live data" {
6031	t.Fatalf("pre-existing database = %q, %v; want it untouched by a failed seed", got, err)
6032	}
6033	}
6034
6035	// A seed that exits 0 but backgrounds a child holding its stderr pipe open is
6036	// a SUCCESS. os/exec's WaitDelay fires on the success path too — Run returns
6037	// ErrWaitDelay instead of nil — and treating that as a failure would fail the
6038	// wake and delete the database the seed had just correctly built, leaving the
6039	// host fresh again on the next wake: permanently un-wakeable.
6040	func TestExecBackendSeedSucceedsDespiteLingeringChild(t *testing.T) {
6041	b := testExecBackend(t)
6042	host := "leaky-seed.example.com"
6043	// argv is: seed --db <path> → $3 is the database path. The backgrounded
6044	// sleeper inherits stderr and holds the pipe open past the drain grace.
6045	stageSeedScript(t, b, host, "v1", "printf seeded > \"$3\"\nsleep 5 &\nexit 0\n")
6046
6047	prevTimeout, prevGrace := seedTimeout, seedDrainGrace
6048	seedTimeout, seedDrainGrace = 10*time.Second, 200*time.Millisecond
6049	t.Cleanup(func() { seedTimeout, seedDrainGrace = prevTimeout, prevGrace })
6050
6051	if err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`}); err != nil {
6052	t.Fatalf("Seed = %v, want nil — the seed exited 0; a leaked output pipe is not a failed seed", err)
6053	}
6054	got, err := os.ReadFile(filepath.Join(b.DataDir, host+".db"))
6055	if err != nil || string(got) != "seeded" {
6056	t.Fatalf("seeded database = %q, %v; want it kept — deleting it would make the host fresh again next wake", got, err)
6057	}
6058	}
6059
6060	// The delete guard is fail-safe by construction: it arms ONLY on a definite
6061	// "not there". Asserted on the rule itself, because the interesting inputs —
6062	// a stat that failed for a reason other than absence — are exactly the ones an
6063	// integration test can't stage reliably.
6064	func TestSeedCreatedTheDB(t *testing.T) {
6065	for _, tc := range []struct {
6066	name string
6067	statErr error
6068	want bool
6069	}{
6070	{"database is there", nil, false},
6071	{"definitely absent", &os.PathError{Op: "stat", Path: "/data/x.db", Err: syscall.ENOENT}, true},
6072	{"bare ErrNotExist", os.ErrNotExist, true},
6073	{"unreadable", &os.PathError{Op: "stat", Path: "/data/x.db", Err: syscall.EACCES}, false},
6074	{"path not traversable", &os.PathError{Op: "stat", Path: "/data/x.db", Err: syscall.ENOTDIR}, false},
6075	{"i/o error", &os.PathError{Op: "stat", Path: "/data/x.db", Err: syscall.EIO}, false},
6076	} {
6077	t.Run(tc.name, func(t *testing.T) {
6078	if got := seedCreatedTheDB(tc.statErr); got != tc.want {
6079	t.Fatalf("seedCreatedTheDB(%v) = %v, want %v — an inconclusive stat must never arm the removal of a database", tc.statErr, got, tc.want)
6080	}
6081	})
6082	}
6083	}
6084
6085	// --- Instance-socket mode normalization (issue #103) -------------------------
6086
6087	// TestMain lets this test binary double as a fake instance app: Start execs
6088	// the live/<host> symlink, and stageServingLive points that symlink at the
6089	// test binary itself with ACTIVATOR_FAKE_APP=1 in the inherited environment
6090	// (childEnv strips only CARLOS_* vars, so it flows through to the child).
6091	// The shell-script recorders elsewhere in this file can't answer the health
6092	// probe — serving HTTP on a unix socket needs a real program, and the test
6093	// binary is the one program every `go test` run is guaranteed to have.
6094	func TestMain(m *testing.M) {
6095	// This binary places processes on a real cgroupfs in its kernel tests
6096	// (TestGenerationStartRefusedBesideADirectParentMember,
6097	// TestClaimContainmentCleanupRespawn), so it serves the placement shim
6098	// like the carlos binary does (cgroup.RunShimIfRequested).
6099	cgroup.RunShimIfRequested()
6100	if os.Getenv("ACTIVATOR_FAKE_APP") == "1" {
6101	fakeAppMain()
6102	return
6103	}
6104	// A worker's wake waits out its settle window unless the child reports
6105	// ready, and no fake child does. Shrink it once for the whole package —
6106	// the tests that are ABOUT the window set their own value and restore it
6107	// (spawnHealthTimeout's pattern in backend_unit_test.go).
6108	sidecarSettle = 20 * time.Millisecond
6109	os.Exit(m.Run())
6110	}
6111
6112	// Which tests call t.Parallel(). A parallel test runs beside the other
6113	// parallel tests once every sequential one has finished, so it may not
6114	// write package state (a shrunk timeout, a hook, the generations switch,
6115	// the log's output), set the environment or the working directory, depend
6116	// on time it measures itself (time.Since), or use the /proc child waits in
6117	// childwait_test.go, which count every child of this binary. A test that
6118	// does any of these stays sequential. The tests marked parallel were
6119	// checked for all of it, through every test helper they call; a test that
6120	// later starts shrinking a package var or setting the environment must
6121	// drop its t.Parallel().
6122
6123	// fakeAppMain is the app half: bind the --socket, tighten it to 0600 — the
6124	// exact titogo habit that stranded the sibling router in issue #103 — and
6125	// serve 200s until killed (500s under ACTIVATOR_FAKE_APP_UNHEALTHY, the
6126	// never-becomes-healthy app).
6127	func fakeAppMain() {
6128	var socket string
6129	for i, a := range os.Args {
6130	if a == "--socket" && i+1 < len(os.Args) {
6131	socket = os.Args[i+1]
6132	}
6133	}
6134	if socket == "" {
6135	os.Exit(2)
6136	}
6137	ln, err := net.Listen("unix", socket)
6138	if err != nil {
6139	os.Exit(1)
6140	}
6141	if err := os.Chmod(socket, 0o600); err != nil {
6142	os.Exit(1)
6143	}
6144	status := http.StatusOK
6145	if os.Getenv("ACTIVATOR_FAKE_APP_UNHEALTHY") == "1" {
6146	status = http.StatusInternalServerError
6147	}
6148	_ = http.Serve(ln, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
6149	w.WriteHeader(status)
6150	}))
6151	}
6152
6153	// stageServingLive points live/<host> at the test binary itself (see
6154	// TestMain) so Start spawns a child that actually binds and serves.
6155	func stageServingLive(t *testing.T, b *ExecBackend, host string) {
6156	t.Helper()
6157	self, err := os.Executable()
6158	if err != nil {
6159	t.Fatal(err)
6160	}
6161	t.Setenv("ACTIVATOR_FAKE_APP", "1")
6162	if err := os.MkdirAll(filepath.Join(b.OptDir, "live"), 0o755); err != nil {
6163	t.Fatal(err)
6164	}
6165	os.Remove(b.liveLink(host))
6166	if err := os.Symlink(self, b.liveLink(host)); err != nil {
6167	t.Fatal(err)
6168	}
6169	}
6170
6171	// TestExecBackendStartNormalizesSocketMode pins the issue #103 contract: the
6172	// mode an instance socket carries after a successful wake is the PLATFORM's
6173	// (0660 — group members, i.e. enrolled carlos-sockets consumers, may dial),
6174	// never the app's own umask or chmod habit. The fake app deliberately
6175	// tightens its socket to 0600 after binding; Start must win.
6176	func TestExecBackendStartNormalizesSocketMode(t *testing.T) {
6177	requireUnixSockets(t)
6178	b := testExecBackend(t)
6179	host := "mode.example.com"
6180	stageServingLive(t, b, host)
6181
6182	rt := registry.Route{Host: host, Kind: "instance", Socket: tmpSock(t)}
6183	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
6184	defer cancel()
6185	proc, _, err := b.Start(ctx, rt)
6186	if err != nil {
6187	t.Fatalf("Start: %v", err)
6188	}
6189	defer proc.Stop(context.Background())
6190
6191	fi, err := os.Stat(rt.Socket)
6192	if err != nil {
6193	t.Fatal(err)
6194	}
6195	if fi.Mode().Perm() != 0o660 {
6196	t.Fatalf("socket mode after healthy Start = %o, want 660 — an app's own chmod must not strand a sibling consumer", fi.Mode().Perm())
6197	}
6198	}
6199
6200	// TestExecBackendFailedWakeDoesNotNormalizeSocketMode pins the other half of
6201	// the contract: normalization happens only AFTER a successful health wait —
6202	// a wake that fails must leave the socket exactly as the app left it. The
6203	// fake app binds and tightens to 0600 but never answers healthily.
6204	func TestExecBackendFailedWakeDoesNotNormalizeSocketMode(t *testing.T) {
6205	requireUnixSockets(t)
6206	b := testExecBackend(t)
6207	host := "unhealthy.example.com"
6208	stageServingLive(t, b, host)
6209	t.Setenv("ACTIVATOR_FAKE_APP_UNHEALTHY", "1")
6210
6211	old := spawnHealthTimeout
6212	spawnHealthTimeout = 500 * time.Millisecond
6213	defer func() { spawnHealthTimeout = old }()
6214
6215	rt := registry.Route{Host: host, Kind: "instance", Socket: tmpSock(t)}
6216	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
6217	defer cancel()
6218	if proc, _, err := b.Start(ctx, rt); err == nil {
6219	defer proc.Stop(context.Background())
6220	t.Fatal("Start succeeded for a never-healthy app")
6221	}
6222	fi, err := os.Stat(rt.Socket)
6223	if err != nil {
6224	t.Fatal(err)
6225	}
6226	if fi.Mode().Perm() != 0o600 {
6227	t.Fatalf("socket mode after failed wake = %o, want the app's own 600 — a failed wake must not widen anything", fi.Mode().Perm())
6228	}
6229	}
6230
6231	// TestExecBackendDoesNotNormalizeSquatterSocket pins the only-our-child
6232	// guard: a stray supervisor squatting the route's socket (the #17 advisory's
6233	// subject) can answer the health probe while the spawned child — which lost
6234	// the bind race and exited — cannot. Start must not widen the squatter's
6235	// socket. The squatter here holds its 200 until the child has verifiably
6236	// exited, making the race the guard closes deterministic.
6237	func TestExecBackendDoesNotNormalizeSquatterSocket(t *testing.T) {
6238	t.Parallel()
6239	requireUnixSockets(t)
6240	b := testExecBackend(t)
6241	host := "squatted.example.com"
6242	marker := filepath.Join(t.TempDir(), "child-ran")
6243	// The child: proof-of-life marker, then immediate exit — the shape of a
6244	// child that could not bind.
6245	script := "#!/bin/sh\ntouch '" + marker + "'\nexit 0\n"
6246	vdir := filepath.Join(b.OptDir, "versions", "v1")
6247	if err := os.MkdirAll(vdir, 0o755); err != nil {
6248	t.Fatal(err)
6249	}
6250	bin := filepath.Join(vdir, "app")
6251	if err := os.WriteFile(bin, []byte(script), 0o755); err != nil {
6252	t.Fatal(err)
6253	}
6254	if err := os.MkdirAll(filepath.Join(b.OptDir, "live"), 0o755); err != nil {
6255	t.Fatal(err)
6256	}
6257	if err := os.Symlink(bin, b.liveLink(host)); err != nil {
6258	t.Fatal(err)
6259	}
6260
6261	sock := tmpSock(t)
6262	ln, err := net.Listen("unix", sock)
6263	if err != nil {
6264	t.Fatal(err)
6265	}
6266	defer ln.Close()
6267	if err := os.Chmod(sock, 0o600); err != nil {
6268	t.Fatal(err)
6269	}
6270	srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
6271	// Answer only once the child is provably dead: marker written, plus
6272	// a grace for the exit itself to be reaped.
6273	deadline := time.Now().Add(5 * time.Second)
6274	for time.Now().Before(deadline) {
6275	if _, err := os.Stat(marker); err == nil {
6276	break
6277	}
6278	time.Sleep(10 * time.Millisecond)
6279	}
6280	time.Sleep(300 * time.Millisecond)
6281	w.WriteHeader(http.StatusOK)
6282	})}
6283	go srv.Serve(ln)
6284	defer srv.Close()
6285
6286	rt := registry.Route{Host: host, Kind: "instance", Socket: sock}
6287	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
6288	defer cancel()
6289	proc, _, err := b.Start(ctx, rt)
6290	if err != nil {
6291	t.Fatalf("Start: %v (the squatter answers healthily; the wake itself succeeds)", err)
6292	}
6293	defer proc.Stop(context.Background())
6294
6295	fi, err := os.Stat(rt.Socket)
6296	if err != nil {
6297	t.Fatal(err)
6298	}
6299	if fi.Mode().Perm() != 0o600 {
6300	t.Fatalf("squatter socket mode after Start = %o, want its own 600 — a socket our child did not bind is never ours to widen", fi.Mode().Perm())
6301	}
6302	}
6303
6304	// The control socket must OUTLIVE the drain. Stop used to close it before the
6305	// child was even signalled, which made the spec's §7.1 story — an app that
6306	// re-asserts its once-timers as it winds down, over the §7.2 verbs on
6307	// CARLOS_CONTROL_SOCKET — a connection-refused during exactly the window the
6308	// app was told to use it.
6309	//
6310	// The test drives the real childProc: a child that traps SIGTERM and stays up
6311	// until the test releases it, a real edge.ControlListener with a verb
6312	// mounted, and a request issued strictly between "SIGTERM delivered" and
6313	// "child exited". It also pins the other half — that the listener is still
6314	// closed, on this path, once the child is gone.
6315	func TestChildProcStopKeepsTheControlSocketUpUntilTheChildExits(t *testing.T) {
6316	t.Parallel()
6317	requireUnixSockets(t)
6318	dir := t.TempDir()
6319	ready := filepath.Join(dir, "ready") // the child's SIGTERM trap is installed
6320	termed := filepath.Join(dir, "termed") // the child saw SIGTERM
6321	release := filepath.Join(dir, "release") // the test lets the child finish
6322	script := filepath.Join(dir, "app")
6323	// The ready marker is touched AFTER the trap is installed, and the test
6324	// waits for it before signalling: a SIGTERM that arrives in the shell's
6325	// first few instructions takes the default action and kills the child
6326	// outright, and there is no drain window to observe.
6327	body := "#!/bin/sh\n" +
6328	"trap \"touch '" + termed + "'\" TERM\n" +
6329	"touch '" + ready + "'\n" +
6330	"while [ ! -f '" + release + "' ]; do sleep 0.02; done\n" +
6331	"exit 0\n"
6332	if err := os.WriteFile(script, []byte(body), 0o755); err != nil {
6333	t.Fatal(err)
6334	}
6335
6336	ctlSock := tmpSock(t)
6337	ctl, err := edge.ListenControl(ctlSock, "drain.example.com", nil, func(mux *http.ServeMux) {
6338	mux.HandleFunc("GET /schedule", func(w http.ResponseWriter, r *http.Request) {
6339	w.WriteHeader(http.StatusOK)
6340	})
6341	})
6342	if err != nil {
6343	t.Fatalf("ListenControl: %v", err)
6344	}
6345
6346	cmd := exec.Command(script)
6347	if err := cmd.Start(); err != nil {
6348	ctl.Close()
6349	t.Fatalf("start the fake child: %v", err)
6350	}
6351	// The child traps TERM and polls a file inside t.TempDir(). Any t.Fatalf
6352	// before the release file is written — the failure this test exists to
6353	// produce, among others — leaves it spinning on a directory the cleanup
6354	// then deletes, immune to the very signal this test is about: only
6355	// SIGKILL ends it, and a red CI run would leak one per attempt.
6356	t.Cleanup(func() { _ = cmd.Process.Kill() })
6357	p := &childProc{cmd: cmd, done: make(chan struct{}), ctl: ctl}
6358	go func() {
6359	p.err = cmd.Wait()
6360	close(p.done)
6361	}()
6362
6363	waitForFile(t, ready, 10*time.Second)
6364
6365	stopped := make(chan error, 1)
6366	go func() {
6367	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
6368	defer cancel()
6369	stopped <- p.Stop(ctx)
6370	}()
6371
6372	// Strictly inside the drain: SIGTERM has been delivered (the child's own
6373	// trap says so) and the child has not exited.
6374	waitForFile(t, termed, 10*time.Second)
6375	if p.Exited() {
6376	t.Fatal("the child exited before the drain window could be observed — the test proves nothing")
6377	}
6378	req, err := http.NewRequest("GET", "http://carlos/schedule", nil)
6379	if err != nil {
6380	t.Fatal(err)
6381	}
6382	resp, err := socketClient(ctlSock).Do(req)
6383	if err != nil {
6384	t.Fatalf("control request during the drain: %v — the socket must still answer while the child drains", err)
6385	}
6386	resp.Body.Close()
6387	if resp.StatusCode != http.StatusOK {
6388	t.Fatalf("control request during the drain = %d, want 200", resp.StatusCode)
6389	}
6390
6391	if err := os.WriteFile(release, nil, 0o600); err != nil {
6392	t.Fatal(err)
6393	}
6394	select {
6395	case err := <-stopped:
6396	if err != nil {
6397	t.Fatalf("Stop: %v", err)
6398	}
6399	case <-time.After(30 * time.Second):
6400	t.Fatal("Stop never returned after the child was released")
6401	}
6402	// The close is only deferred, not dropped: a listener left bound to a
6403	// socket with no process behind it is the #129 stale-socket wake trap.
6404	if _, err := os.Stat(ctlSock); !os.IsNotExist(err) {
6405	t.Fatalf("control socket after Stop returned: stat err = %v, want it unlinked", err)
6406	}
6407	}
6408
6409	// The already-exited path takes an early return, and the listener close has
6410	// to survive it: an instance whose child crashed on its own still leaves the
6411	// agent holding a bound socket nothing is behind.
6412	func TestChildProcStopClosesTheControlSocketWhenTheChildIsAlreadyGone(t *testing.T) {
6413	requireUnixSockets(t)
6414	ctlSock := tmpSock(t)
6415	ctl, err := edge.ListenControl(ctlSock, "gone.example.com", nil, nil)
6416	if err != nil {
6417	t.Fatalf("ListenControl: %v", err)
6418	}
6419	cmd := exec.Command("/bin/sh", "-c", "exit 0")
6420	if err := cmd.Start(); err != nil {
6421	ctl.Close()
6422	t.Fatal(err)
6423	}
6424	p := &childProc{cmd: cmd, done: make(chan struct{}), ctl: ctl}
6425	p.err = cmd.Wait()
6426	close(p.done) // the child is gone before Stop is ever called
6427
6428	if err := p.Stop(context.Background()); err != nil {
6429	t.Fatalf("Stop on an already-exited child: %v", err)
6430	}
6431	if _, err := os.Stat(ctlSock); !os.IsNotExist(err) {
6432	t.Fatalf("control socket after Stop on an exited child: stat err = %v, want it unlinked", err)
6433	}
6434	}
6435
6436	func TestChildEnvKeepsOneEntryPerKeyLastWins(t *testing.T) {
6437	t.Setenv("PATH", "/usr/bin")
6438	env := childEnv(map[string]string{"PATH": "/tenant/bin", "A": "1"})
6439	seen := map[string]string{}
6440	for _, kv := range env {
6441	k, v, _ := strings.Cut(kv, "=")
6442	if _, dup := seen[k]; dup {
6443	t.Fatalf("duplicate key %s in %v", k, env)
6444	}
6445	seen[k] = v
6446	}
6447	if seen["PATH"] != "/tenant/bin" {
6448	t.Errorf("PATH = %q, want the tenant's value to win over the inherited one", seen["PATH"])
6449	}
6450	if seen["A"] != "1" {
6451	t.Errorf("A = %q", seen["A"])
6452	}
6453	}
6454
6455	// A Proc whose Stop reports ErrStillRunning must not have its lease
6456	// released: stop() keeps the entry in stopping and the sweeper finishes it
6457	// once the proc reports exited (design §6.3, codex R2-5).
6458	func TestStopDoesNotReleaseTheLeaseOverAnUnprovenStop(t *testing.T) {
6459	t.Parallel()
6460	a, b, clk, _ := testActivator(t, time.Minute)
6461	b.procStuck = true // Start returns a fakeProc whose Stop reports ErrStillRunning until exit()
6462	rt := testRoute("h.example.com")
6463	arrangeAwake(t, a, rt) // wake rt and release the request, as the idle-hibernate test does
6464
6465	clk.advance(2 * time.Minute)
6466	a.sweepOnce() // idle: stop() runs, Stop reports ErrStillRunning
6467	p := b.lastProc()
6468	waitFor(t, "the stuck stop to be attempted", func() bool { return p.stopCalls() >= 1 })
6469	if n := b.releaseCount(); n != 0 {
6470	t.Fatalf("lease released %d times while the stop is unproven", n)
6471	}
6472	if a.entryCount() != 1 {
6473	t.Fatal("entry forgotten before the stop was proven")
6474	}
6475	p.exit()
6476	// Drive the sweep from inside the wait itself: finishing a pendingStop
6477	// entry now retries proc.Stop (design §6.3) on the sweeper's own
6478	// cadence, so a single sweepOnce() right after exit() is a race against
6479	// exactly when the sweeper would next notice. Polling sweepOnce()
6480	// alongside the release count removes that race instead of hoping one
6481	// call lands after exit() is visible.
6482	waitFor(t, "the lease to release once termination is proven", func() bool {
6483	a.sweepOnce()
6484	return b.releaseCount() == 1
6485	})
6486	}
6487
6488	// finishPendingStops must retry the stop itself while the process is still
6489	// running, not only once it has already exited (design §6.3: "sweepOnce
6490	// retries proc.Stop on its cadence until done closes"). Without the retry, a
6491	// stop whose first Control.Stop call itself failed would never ask again.
6492	func TestFinishPendingStopsRetriesTheStopWhileStillRunning(t *testing.T) {
6493	t.Parallel()
6494	a, b, clk, _ := testActivator(t, time.Minute)
6495	b.procStuck = true // Stop reports ErrStillRunning until exit(), and this proc never exits here
6496	rt := testRoute("stuck.example.com")
6497	arrangeAwake(t, a, rt)
6498
6499	clk.advance(2 * time.Minute)
6500	a.sweepOnce() // idle: stop() runs, Stop reports ErrStillRunning, entry goes pendingStop
6501	p := b.lastProc()
6502	waitFor(t, "the first stop attempt", func() bool { return p.stopCalls() >= 1 })
6503
6504	// Immediately after: no retry yet, the entry's own actStopTimeout budget
6505	// has not passed since the first attempt.
6506	a.sweepOnce()
6507	if n := p.stopCalls(); n != 1 {
6508	t.Fatalf("stop retried before actStopTimeout elapsed (stopCalls=%d)", n)
6509	}
6510
6511	clk.advance(actStopTimeout + time.Second)
6512	waitFor(t, "a second stop attempt once actStopTimeout has passed", func() bool {
6513	a.sweepOnce()
6514	return p.stopCalls() >= 2
6515	})
6516	if b.releaseCount() != 0 {
6517	t.Fatal("lease released while the process still reports running")
6518	}
6519	if a.entryCount() != 1 {
6520	t.Fatal("entry forgotten while the process still reports running")
6521	}
6522	}
6523
6524	// The deferred Shutdown drain (ledger T5): a pendingStop entry whose process
6525	// finally exits DURING Shutdown still gets its final checkpoint and lease
6526	// release before Shutdown returns, via the same finishPendingStops pass
6527	// Shutdown's own drain loop re-runs on its poll ticker.
6528	func TestShutdownDrainsAPendingStopThatExitsMidShutdown(t *testing.T) {
6529	t.Parallel()
6530	a, b, clk, _ := testActivator(t, time.Minute)
6531	b.procStuck = true
6532	rt := testRoute("shutdown-pending.example.com")
6533	arrangeAwake(t, a, rt)
6534
6535	clk.advance(2 * time.Minute)
6536	a.sweepOnce() // stop() runs, Stop reports ErrStillRunning, entry goes pendingStop
6537	p := b.lastProc()
6538	waitFor(t, "the stop to be attempted", func() bool { return p.stopCalls() >= 1 })
6539
6540	// The process exits on its own shortly after Shutdown begins draining.
6541	// Shutdown's 100ms poll ticker is what has to notice it, not this test.
6542	go func() {
6543	time.Sleep(150 * time.Millisecond)
6544	p.exit()
6545	}()
6546
6547	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
6548	defer cancel()
6549	a.Shutdown(ctx)
6550
6551	if got := b.releaseCount(); got != 1 {
6552	t.Fatalf("lease releases after Shutdown = %d, want exactly 1", got)
6553	}
6554	if a.entryCount() != 0 {
6555	t.Fatal("Shutdown returned with a pending-stop entry still on the books")
6556	}
6557	}
6558
6559	// The sidecar restart must not hold the activator mutex across the start
6560	// call (design §6.7, codex R2-6): a stalled sidecar start does not block
6561	// another host's wake.
6562	func TestSidecarRestartDoesNotHoldTheLockAcrossTheStart(t *testing.T) {
6563	a, b, clk, _ := testActivator(t, time.Hour)
6564	b.sidecarStaged = true
6565
6566	// wake a.example.com first, ungated. The gate below is for the
6567	// sweeper's RESTART call, not this initial co-spawn.
6568	arrangeAwake(t, a, testRoute("a.example.com"))
6569	b.lastSidecar().exit() // the sweeper now wants a restart
6570
6571	gate := make(chan struct{})
6572	entered := make(chan struct{})
6573	b.mu.Lock()
6574	b.sidecarGate = gate // StartSidecar blocks on this channel when non-nil
6575	b.sidecarStartEntered = entered
6576	b.mu.Unlock()
6577	clk.advance(time.Minute)
6578	a.sweepOnce() // launches the restart; it blocks inside StartSidecar on gate
6579	<-entered // wait for the restart's call to actually begin, not a guessed sleep
6580	// host b has no sidecar of its own. HasSidecar false keeps its wake's
6581	// own co-spawn (unrelated to what's under test) off the same gate.
6582	b.mu.Lock()
6583	b.sidecarStaged = false
6584	b.mu.Unlock()
6585	done := make(chan struct{})
6586	go func() {
6587	// wake b.example.com
6588	release, err := a.beginRequest(context.Background(), testRoute("b.example.com"))
6589	if err != nil {
6590	t.Errorf("beginRequest b.example.com: %v", err)
6591	return
6592	}
6593	release()
6594	close(done)
6595	}()
6596	select {
6597	case <-done:
6598	case <-time.After(2 * time.Second):
6599	t.Fatal("a second host's wake blocked behind a stalled sidecar start")
6600	}
6601	close(gate)
6602	}
6603
6604	// doWake runs on the committed row, not the copy the edge supplied
6605	// (design §6.8, codex R8-1).
6606	func TestDoWakeRefreshesTheRouteFromTheRegistry(t *testing.T) {
6607	a, b, _, reg := testActivator(t, time.Hour)
6608	a.Routes = reg
6609	old := testRoute("h.example.com")
6610	fresh := old
6611	fresh.DBPath = "/data/instances/h.example.com/h.example.com.db"
6612	fresh.Socket = "/run/carlos/instances/h.example.com/app.sock"
6613	// insert fresh into reg the way the file's registry-backed tests do
6614	if err := reg.Upsert(fresh); err != nil {
6615	t.Fatalf("Upsert: %v", err)
6616	}
6617	arrangeAwake(t, a, old) // wake with old (the edge's stale copy)
6618	if got := b.lastStartedRoute().DBPath; got != fresh.DBPath {
6619	t.Fatalf("Start saw DBPath %q, want the registry's %q", got, fresh.DBPath)
6620	}
6621	if got := b.lastStartedRoute().Socket; got != fresh.Socket {
6622	t.Fatalf("Start saw Socket %q, want the registry's %q", got, fresh.Socket)
6623	}
6624	}
6625
6626	// A restart whose StartSidecar call is still stalled when its host stops
6627	// (Task 8, F6 plan 5, flag off): stop() waits for it only so long
6628	// (sidecarStartJoinWait), and then the stop stays pending, with the lease,
6629	// until the start returns. The sidecar the start brought back is stopped,
6630	// and only then is the host cgroup proven empty and the lease released. A
6631	// stale restart therefore can no longer land on a fresh wake of the host:
6632	// that wake cannot begin until this entry is gone. The restart still
6633	// compares entry identity (the *actEntry captured at launch) rather than a
6634	// counter, and the fresh wake below still gets its own sidecar.
6635	func TestAStopKeepsTheLeaseUntilAnInFlightSidecarStartReturns(t *testing.T) {
6636	prev := sidecarStartJoinWait
6637	sidecarStartJoinWait = 50 * time.Millisecond
6638	t.Cleanup(func() { sidecarStartJoinWait = prev })
6639	a, b, _, _ := testActivator(t, time.Hour)
6640	h := &hostEmptyFake{fakeBackend: b}
6641	a.backend = h
6642	b.sidecarStaged = true
6643	host := "stale.example.com"
6644
6645	arrangeAwake(t, a, testRoute(host))
6646	h.reset() // the wake's own proof (finding 4) is not the stop's
6647	first := b.lastSidecar()
6648	first.exit() // the sweeper now wants a restart
6649
6650	gate := make(chan struct{})
6651	entered := make(chan struct{})
6652	b.mu.Lock()
6653	b.sidecarGate = gate
6654	b.sidecarStartEntered = entered
6655	b.mu.Unlock()
6656	a.sweepOnce() // launches the restart; it blocks inside StartSidecar on gate
6657	<-entered
6658
6659	rep, inst := b.lastRep(), b.lastProc()
6660	rep.setOnClose(func() {
6661	stale := b.lastSidecar()
6662	if stale == first || stale.stopCalls() < 1 {
6663	t.Error("the lease was released before the in-flight start's sidecar was stopped")
6664	}
6665	ev := b.events()
6666	stopped := slices.Index(ev, "stop:sidecar:late")
6667	if stopped < 0 || !slices.Contains(ev[stopped:], "emptyhost") {
6668	t.Errorf("events %v: want the late sidecar stopped, then the host proven empty, before the release", ev)
6669	}
6670	})
6671	if !a.Bounce(host) {
6672	t.Fatal("Bounce did not find the awake entry")
6673	}
6674	waitFor(t, "the instance to stop", inst.isStopped)
6675	time.Sleep(4 * sidecarStartJoinWait) // well past stop()'s own wait
6676	for range 3 {
6677	a.sweepOnce()
6678	}
6679	if rep.isClosed() || a.entryCount() != 1 {
6680	t.Fatalf("closed=%v entries=%d while a sidecar start was in flight: the lease went first", rep.isClosed(), a.entryCount())
6681	}
6682	if n := h.count(); n != 0 {
6683	t.Fatalf("the host cgroup was proven %d times before the in-flight start returned", n)
6684	}
6685
6686	b.mu.Lock()
6687	b.sidecarGate = nil // the fresh wake's own co-spawn must not block on it too
6688	b.mu.Unlock()
6689	b.lateSidecarLabel("sidecar:late")
6690	close(gate) // let the stale StartSidecar call finally return
6691	waitFor(t, "the stop to finish once the start returned", func() bool {
6692	a.sweepOnce()
6693	return rep.isClosed() && a.entryCount() == 0
6694	})
6695	b.lateSidecarLabel("")
6696
6697	// A fresh wake of the same host gets its own sidecar.
6698	arrangeAwake(t, a, testRoute(host))
6699	freshSidecar := b.lastSidecar()
6700	a.mu.Lock()
6701	installed := a.entries[host].sidecar
6702	a.mu.Unlock()
6703	if installed != freshSidecar || freshSidecar.label != "sidecar" {
6704	t.Fatal("the fresh wake did not get its own sidecar")
6705	}
6706	}
6707
6708	// seedCountingBackend is fakeBackend plus the optional seed recorder.
6709	type seedCountingBackend struct {
6710	fakeBackend
6711	placed []string
6712	rep Rep
6713	replicated int // Replicate calls seen when SeedPlaced ran
6714	err error
6715	}
6716
6717	func (b *seedCountingBackend) SeedPlaced(_ context.Context, rt registry.Route, rep Rep) error {
6718	b.mu.Lock()
6719	defer b.mu.Unlock()
6720	b.placed = append(b.placed, rt.Host)
6721	b.rep, b.replicated = rep, len(b.syncIntervals)
6722	return b.err
6723	}
6724
6725	func TestDoWakeRecordsSeedPlacementAfterReplication(t *testing.T) {
6726	t.Parallel()
6727	for _, recErr := range []error{nil, errors.New("record unreachable")} {
6728	b := &seedCountingBackend{err: recErr}
6729	reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db"))
6730	if err != nil {
6731	t.Fatal(err)
6732	}
6733	a := New(Config{Bucket: "test-bucket", Idle: time.Hour}, b, reg)
6734	rt := testRoute("shop.example.com")
6735	rt.SeedSource = seeds.RowSeed{ID: "sd_aaaaaaaaaaaaaaaaaaaaaaaaaa", Kind: seeds.KindReplica, State: seeds.StateReady}.Encode()
6736	if err := reg.Upsert(rt); err != nil {
6737	t.Fatal(err)
6738	}
6739	// A failed record never fails the wake: the instance serves.
6740	arrangeAwake(t, a, rt)
6741	if len(b.placed) != 1 || b.placed[0] != rt.Host {
6742	t.Fatalf("SeedPlaced calls = %q, want one, after replication started", b.placed)
6743	}
6744	if len(b.reps) != 1 || b.rep != Rep(b.reps[0]) || b.replicated != 1 {
6745	t.Fatalf("SeedPlaced saw rep %v after %d replications; want the wake's own, after Replicate", b.rep, b.replicated)
6746	}
6747	if !a.Owns(rt.Host) {
6748	t.Fatalf("recorder error %v: the wake did not serve", recErr)
6749	}
6750	}
6751	}
6752
6753	// hostEmptyFake is fakeBackend with Task 8's host-cgroup proof
6754	// (hostEmptier), the flag-off terminal stop's: err is its answer, and each
6755	// call is counted and logged as "emptyhost" in the backend's order log.
6756	type hostEmptyFake struct {
6757	*fakeBackend
6758	mu sync.Mutex
6759	err error
6760	calls int
6761	passes int // calls that answered nil
6762	}
6763
6764	func (h *hostEmptyFake) EmptyHost(context.Context, registry.Route) error {
6765	h.mu.Lock()
6766	h.calls++
6767	err := h.err
6768	if err == nil {
6769	h.passes++
6770	}
6771	h.mu.Unlock()
6772	h.record("emptyhost")
6773	return err
6774	}
6775
6776	func (h *hostEmptyFake) passed() int {
6777	h.mu.Lock()
6778	defer h.mu.Unlock()
6779	return h.passes
6780	}
6781
6782	func (h *hostEmptyFake) set(err error) {
6783	h.mu.Lock()
6784	h.err = err
6785	h.mu.Unlock()
6786	}
6787
6788	func (h *hostEmptyFake) count() int {
6789	h.mu.Lock()
6790	defer h.mu.Unlock()
6791	return h.calls
6792	}
6793
6794	// reset forgets the proofs made so far: a stop test calls it after its wake,
6795	// whose own flag-off proof (finding 4) it does not count.
6796	func (h *hostEmptyFake) reset() {
6797	h.mu.Lock()
6798	h.calls, h.passes = 0, 0
6799	h.mu.Unlock()
6800	}
6801
6802	// Task 8 fix round 1, finding 4: after an agent restart the in-memory
6803	// pending stop and kill marks are gone, so a flag-off wake proves the host
6804	// cgroup empty itself, under its lease and before Restore. A proof that
6805	// fails refuses the wake before anything is restored or started, and keeps
6806	// the lease (a leftover may still write the database). One that holds lets
6807	// the wake go on.
6808	func TestAFlagOffWakeProvesTheHostCgroupBeforeItRestores(t *testing.T) {
6809	a, b, _, _ := testActivator(t, time.Minute)
6810	h := &hostEmptyFake{fakeBackend: b}
6811	a.backend = h
6812	h.set(errors.New("a leftover still populates the host cgroup"))
6813	rt := testRoute("leftover.example.com")
6814
6815	if _, err := a.beginRequest(context.Background(), rt); !errors.Is(err, errHostUnproven) {
6816	t.Fatalf("wake = %v, want errHostUnproven", err)
6817	}
6818	if _, restores, starts := b.counts(); restores != 0 || starts != 0 {
6819	t.Fatalf("restores=%d starts=%d over a host cgroup not proven empty, want neither", restores, starts)
6820	}
6821	if got := b.releaseCount(); got != 0 {
6822	t.Fatalf("lease releases = %d, want 0: a leftover may still write the database", got)
6823	}
6824	if got := memberSafeWakeReason(fmt.Errorf("leftover.example.com: refusing to wake: %w", errHostUnproven)); got != memberSafeWakeReason(ErrFenced) {
6825	t.Fatalf("member reads %q, want the fence's existing text", got)
6826	}
6827
6828	h.set(nil)
6829	if _, err := a.beginRequest(context.Background(), rt); err != nil {
6830	t.Fatalf("wake = %v once the proof holds", err)
6831	}
6832	ev := b.events()
6833	last := func(name string) int {
6834	for i := len(ev) - 1; i >= 0; i-- {
6835	if ev[i] == name {
6836	return i
6837	}
6838	}
6839	return -1
6840	}
6841	proven, restored := last("emptyhost"), last("restore")
6842	if proven < 0 || restored < proven {
6843	t.Fatalf("events %v: want the host proven empty before the restore", ev)
6844	}
6845	}
6846
6847	// No delegated cgroup: the wake goes on as today. A row whose child is not
6848	// in carlos-exec@<host> (nested, unit-backed) asks no proof, and neither
6849	// does a wake with generations on, whose proofs are its own.
6850	func TestAWakeAsksTheHostProofOnlyFlagOffForAnExecChild(t *testing.T) {
6851	t.Parallel()
6852	t.Run("no delegation", func(t *testing.T) {
6853	a, b, _, _ := testActivator(t, time.Minute)
6854	h := &hostEmptyFake{fakeBackend: b}
6855	h.set(cgroup.ErrNoDelegation)
6856	a.backend = h
6857	if _, err := a.beginRequest(context.Background(), testRoute("undelegated-wake.example.com")); err != nil {
6858	t.Fatalf("wake = %v, want today's wake with no delegation", err)
6859	}
6860	if h.count() != 1 {
6861	t.Fatalf("proofs = %d, want 1", h.count())
6862	}
6863	})
6864	for _, tc := range []struct {
6865	name string
6866	setup func(*Service, *fakeBackend, *registry.Route)
6867	}{
6868	{"nested", func(_ *Service, b *fakeBackend, _ *registry.Route) {
6869	b.layoutFn = func(registry.Route) (Layout, string) { return LayoutNested, "" }
6870	}},
6871	{"unit-backed", func(_ *Service, _ *fakeBackend, rt *registry.Route) { rt.Backing = registry.BackingUnit }},
6872	{"generations on", func(a *Service, _ *fakeBackend, _ *registry.Route) { a.gens = true }},
6873	} {
6874	t.Run(tc.name, func(t *testing.T) {
6875	a, b, _, _ := testActivator(t, time.Minute)
6876	h := &hostEmptyFake{fakeBackend: b}
6877	h.set(errors.New("must not be asked"))
6878	a.backend = h
6879	rt := testRoute("no-host-proof.example.com")
6880	tc.setup(a, b, &rt)
6881	_, _ = a.beginRequest(context.Background(), rt)
6882	if n := h.count(); n != 0 {
6883	t.Fatalf("host proofs = %d, want none", n)
6884	}
6885	})
6886	}
6887	}
6888
6889	// lateSidecarLabel labels every sidecar StartSidecar returns from now on.
6890	func (b *fakeBackend) lateSidecarLabel(label string) {
6891	b.mu.Lock()
6892	b.sidecarLabel = label
6893	b.mu.Unlock()
6894	}
6895
6896	// Task 8 (F6, flag off): an idle stop proves the host cgroup empty before
6897	// the lease goes. While the proof fails the stop stays pending with the
6898	// lease held; the sweep retries, and the lease goes only after a proof
6899	// that holds.
6900	func TestAFlagOffStopKeepsTheLeaseUntilTheHostCgroupIsProvenEmpty(t *testing.T) {
6901	t.Parallel()
6902	a, b, clk, _ := testActivator(t, time.Minute)
6903	h := &hostEmptyFake{fakeBackend: b}
6904	a.backend = h
6905	arrangeAwake(t, a, testRoute("f6.example.com"))
6906	h.reset() // the wake's own proof (finding 4) is not the stop's
6907	h.set(errors.New("the host cgroup still holds a process"))
6908
6909	clk.advance(2 * time.Minute)
6910	a.sweepOnce()
6911	waitFor(t, "the stop to go pending on the proof", func() bool {
6912	a.mu.Lock()
6913	defer a.mu.Unlock()
6914	e := a.entries["f6.example.com"]
6915	return h.count() >= 1 && e != nil && e.pendingStop && !e.genReaping
6916	})
6917	rep := b.lastRep()
6918	if rep.isClosed() || a.entryCount() != 1 {
6919	t.Fatal("the lease was released over a host cgroup not proven empty")
6920	}
6921
6922	rep.setOnClose(func() {
6923	if n := h.passed(); n != 1 {
6924	t.Errorf("passing proofs before the release = %d, want 1", n)
6925	}
6926	})
6927	h.set(nil)
6928	waitFor(t, "the lease to go once the proof holds", func() bool {
6929	a.sweepOnce()
6930	return rep.isClosed() && a.entryCount() == 0
6931	})
6932	}
6933
6934	// A passing proof releases the lease in the stop itself, after the proof.
6935	func TestAFlagOffStopProvesTheHostCgroupBeforeTheLeaseGoes(t *testing.T) {
6936	t.Parallel()
6937	a, b, clk, _ := testActivator(t, time.Minute)
6938	h := &hostEmptyFake{fakeBackend: b}
6939	a.backend = h
6940	arrangeAwake(t, a, testRoute("proven.example.com"))
6941	h.reset() // the wake's own proof (finding 4) is not the stop's
6942	rep := b.lastRep()
6943	var atClose atomic.Int32
6944	atClose.Store(-1)
6945	rep.setOnClose(func() { atClose.Store(int32(h.count())) })
6946
6947	clk.advance(2 * time.Minute)
6948	a.sweepOnce()
6949	waitFor(t, "the stop", func() bool { return a.entryCount() == 0 })
6950	if got := atClose.Load(); got != 1 {
6951	t.Fatalf("proofs before the release = %d, want 1", got)
6952	}
6953	}
6954
6955	// No delegated cgroup (cgroup.ErrNoDelegation): the leader's exit is the
6956	// only proof there is (ledger 127), and the lease goes at once.
6957	func TestAFlagOffStopWithNoDelegationReleasesAtOnce(t *testing.T) {
6958	t.Parallel()
6959	a, b, clk, _ := testActivator(t, time.Minute)
6960	h := &hostEmptyFake{fakeBackend: b}
6961	h.set(cgroup.ErrNoDelegation)
6962	a.backend = h
6963	arrangeAwake(t, a, testRoute("undelegated.example.com"))
6964	h.reset() // the wake's own proof (finding 4) is not the stop's
6965
6966	clk.advance(2 * time.Minute)
6967	a.sweepOnce()
6968	waitFor(t, "the stop, with no sweep behind it", func() bool { return a.entryCount() == 0 })
6969	if !b.lastRep().isClosed() || h.count() != 1 {
6970	t.Fatalf("closed=%v proofs=%d, want the lease released after one proof", b.lastRep().isClosed(), h.count())
6971	}
6972	}
6973
6974	// A nested row's child ran as a unit in its host slice, not in
6975	// carlos-exec@<host>: its stop asks no host proof, decided from the layout
6976	// the wake launched on (actEntry.launchLayout, set at install).
6977	func TestAFlagOffStopOfANestedRowAsksNoHostProof(t *testing.T) {
6978	t.Parallel()
6979	a, b, clk, _ := testActivator(t, time.Minute)
6980	b.layoutFn = func(registry.Route) (Layout, string) { return LayoutNested, "" }
6981	h := &hostEmptyFake{fakeBackend: b}
6982	h.set(errors.New("must not be asked"))
6983	a.backend = h
6984	arrangeAwake(t, a, testRoute("nested-stop.example.com"))
6985
6986	clk.advance(2 * time.Minute)
6987	a.sweepOnce()
6988	waitFor(t, "the stop", func() bool { return a.entryCount() == 0 })
6989	if n := h.count(); n != 0 || !b.lastRep().isClosed() {
6990	t.Fatalf("proofs=%d closed=%v, want no host proof and the lease released", n, b.lastRep().isClosed())
6991	}
6992	}
6993
6994	// EmptyHost's edges: no delegation is ErrNoDelegation (the caller's
6995	// leader-only proof), and a unit-backed row has no host cgroup of ours.
6996	func TestEmptyHostDispatch(t *testing.T) {
6997	eb := testExecBackend(t)
6998	rt := registry.Route{Host: "edge.example.com"}
6999	if err := eb.EmptyHost(context.Background(), rt); !errors.Is(err, cgroup.ErrNoDelegation) {
7000	t.Fatalf("EmptyHost with no cgroup manager = %v, want ErrNoDelegation", err)
7001	}
7002	sb := &SplitBackend{Exec: eb, Unit: &UnitBackend{}}
7003	if err := sb.EmptyHost(context.Background(), rt); !errors.Is(err, cgroup.ErrNoDelegation) {
7004	t.Fatalf("SplitBackend.EmptyHost (exec row) = %v, want the exec side's answer", err)
7005	}
7006	rt.Backing = registry.BackingUnit
7007	if err := sb.EmptyHost(context.Background(), rt); err != nil {
7008	t.Fatalf("SplitBackend.EmptyHost (unit row) = %v, want nil", err)
7009	}
7010	}
7011
7012	// A kernel before 5.14 has no cgroup.kill, so the host cgroup cannot be
7013	// killed there at all. EmptyHost answers that as no delegation (the
7014	// leader-only proof, ledger 127), not as an unproven kill that would hold
7015	// every flag-off stop's lease on such a box.
7016	func TestEmptyHostWithNoCgroupKillIsTheLeaderOnlyProof(t *testing.T) {
7017	eb := testExecBackend(t)
7018	opened := 0
7019	eb.openHostCleanup = func(string) (*cgroup.Cleanup, error) {
7020	opened++
7021	return nil, fmt.Errorf("%w: open cgroup.kill: no such file or directory", cgroup.ErrNoKill)
7022	}
7023	err := eb.EmptyHost(context.Background(), registry.Route{Host: "old-kernel.example.com"})
7024	if opened != 1 {
7025	t.Fatalf("host cgroup opened %d times, want 1", opened)
7026	}
7027	if !errors.Is(err, cgroup.ErrNoDelegation) || !errors.Is(err, cgroup.ErrNoKill) {
7028	t.Fatalf("EmptyHost with no cgroup.kill = %v, want ErrNoDelegation naming ErrNoKill", err)
7029	}
7030	}
7031
7032	// The pending-stop path (a leader that was still running when stop() asked)
7033	// is held to the same proof once the leader exits.
7034	func TestAPendingFlagOffStopProvesTheHostCgroupBeforeTheLeaseGoes(t *testing.T) {
7035	t.Parallel()
7036	a, b, clk, _ := testActivator(t, time.Minute)
7037	h := &hostEmptyFake{fakeBackend: b}
7038	a.backend = h
7039	b.procStuck = true
7040	arrangeAwake(t, a, testRoute("pending-f6.example.com"))
7041	h.reset() // the wake's own proof (finding 4) is not the stop's
7042	h.set(errors.New("the host cgroup still holds a process"))
7043
7044	clk.advance(2 * time.Minute)
7045	a.sweepOnce()
7046	p := b.lastProc()
7047	waitFor(t, "the stuck stop", func() bool { return p.stopCalls() >= 1 })
7048	p.exit()
7049	waitFor(t, "the pending stop's proof", func() bool {
7050	a.sweepOnce()
7051	return h.count() >= 1
7052	})
7053	rep := b.lastRep()
7054	if rep.isClosed() {
7055	t.Fatal("a pending stop released the lease over a host cgroup not proven empty")
7056	}
7057
7058	rep.setOnClose(func() {
7059	if n := h.passed(); n != 1 {
7060	t.Errorf("passing proofs before the release = %d, want 1", n)
7061	}
7062	})
7063	h.set(nil)
7064	waitFor(t, "the lease to go once the proof holds", func() bool {
7065	a.sweepOnce()
7066	return rep.isClosed() && a.entryCount() == 0
7067	})
7068	}
7069