carlos / platform Public

Download this file

1	package appconfig
2
3	import (
4	"bytes"
5	"context"
6	"errors"
7	"io"
8	"os"
9	"path/filepath"
10	"reflect"
11	"slices"
12	"strings"
13	"sync"
14	"testing"
15	"time"
16
17	"golang.org/x/sys/unix"
18
19	"github.com/carlosframework/platform/internal/provision"
20	"github.com/carlosframework/platform/internal/registry"
21	"github.com/carlosframework/platform/internal/relay"
22	"github.com/carlosframework/platform/internal/release"
23	)
24
25	// execKit is the shared setup: store, registry, dataDir, keypair.
26	func execKit(t *testing.T) (ctx context.Context, s release.Store, reg *registry.Registry, dataDir, priv, pub string) {
27	t.Helper()
28	s = release.NewDirStore(t.TempDir())
29	ctx = context.Background()
30	reg = testRegistry(t)
31	dataDir = t.TempDir()
32	var err error
33	priv, pub, err = GenerateKey()
34	if err != nil {
35	t.Fatalf("GenerateKey: %v", err)
36	}
37	return
38	}
39
40	// captureStdout runs fn with os.Stdout redirected to a pipe and returns
41	// everything fn printed. SyncExec logs via fmt.Printf directly to
42	// os.Stdout, matching every other pass in this package — this is the only
43	// way to pin a log line's presence without restructuring every call site
44	// to take a Writer, so it's used sparingly: only where a test's whole
45	// point IS that a specific line got printed (a `fmt.Printf` deleted by a
46	// future edit would otherwise go unnoticed by every other test, which
47	// only assert on the filesystem).
48	func captureStdout(t *testing.T, fn func()) string {
49	t.Helper()
50	r, w, err := os.Pipe()
51	if err != nil {
52	t.Fatalf("os.Pipe: %v", err)
53	}
54	old := os.Stdout
55	os.Stdout = w
56	defer func() { os.Stdout = old }()
57	fn()
58	if err := w.Close(); err != nil {
59	t.Fatalf("close pipe writer: %v", err)
60	}
61	var buf bytes.Buffer
62	if _, err := io.Copy(&buf, r); err != nil {
63	t.Fatalf("read captured stdout: %v", err)
64	}
65	return buf.String()
66	}
67
68	// failingKeyStore wraps a release.Store and makes Get fail with a fixed
69	// error for exactly one key — simulating a transient bucket blip against a
70	// single app's default bundle while every other key (other apps' bundles,
71	// including in the SAME pass) resolves normally, the shape a real S3
72	// SlowDown/503 against one object takes.
73	type failingKeyStore struct {
74	release.Store
75	failKey string
76	err error
77	}
78
79	func (f *failingKeyStore) Get(ctx context.Context, key string) ([]byte, string, error) {
80	if key == f.failKey {
81	return nil, "", f.err
82	}
83	return f.Store.Get(ctx, key)
84	}
85
86	func TestSyncExecWritesConfigFileAndBouncesOnChange(t *testing.T) {
87	ctx, s, reg, dd, priv, _ := execKit(t)
88	pk, err := ParsePrivateKey(priv, "test")
89	if err != nil {
90	t.Fatalf("ParsePrivateKey: %v", err)
91	}
92	if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil {
93	t.Fatalf("SetEnv: %v", err)
94	}
95	if err := reg.Upsert(registry.Route{
96	Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab",
97	Status: "ready", Socket: "/run/carlos/inst.sock",
98	}); err != nil {
99	t.Fatalf("Upsert: %v", err)
100	}
101	var bounced []string
102	bounce := func(h string) error { bounced = append(bounced, h); return nil }
103	if err := SyncExec(ctx, reg, s, pk, dd, bounce, "", nil); err != nil {
104	t.Fatalf("SyncExec: %v", err)
105	}
106	got, err := os.ReadFile(filepath.Join(dd, "inst.example.com.config.env"))
107	if err != nil {
108	t.Fatalf("read config env: %v", err)
109	}
110	if string(got) != "LOG_LEVEL=debug\n" {
111	t.Fatalf("config env = %q, want LOG_LEVEL=debug", got)
112	}
113	if fi, _ := os.Stat(filepath.Join(dd, "inst.example.com.config.env")); fi.Mode().Perm() != 0o600 {
114	t.Fatalf("config env mode = %v, want 0600", fi.Mode().Perm())
115	}
116	if len(bounced) != 1 || bounced[0] != "inst.example.com" {
117	t.Fatalf("bounced = %v, want exactly [inst.example.com]", bounced)
118	}
119	// Second pass, nothing changed: no bounce.
120	bounced = nil
121	if err := SyncExec(ctx, reg, s, pk, dd, bounce, "", nil); err != nil {
122	t.Fatalf("SyncExec (second): %v", err)
123	}
124	if len(bounced) != 0 {
125	t.Fatalf("unchanged pass bounced %v, want none", bounced)
126	}
127	}
128
129	// TestSyncExecWritesANestedRouteIntoItsInstanceDir pins the isolated-exec
130	// half of config delivery (design §7): a route whose DB and socket are the
131	// nested layout's defaults gets its rendered bundle inside its OWN instance
132	// directory, never at the flat <dataDir>/<host>.config.env path a foreign or
133	// flat row would use.
134	func TestSyncExecWritesANestedRouteIntoItsInstanceDir(t *testing.T) {
135	ctx, s, reg, dd, priv, _ := execKit(t)
136	pk, err := ParsePrivateKey(priv, "test")
137	if err != nil {
138	t.Fatalf("ParsePrivateKey: %v", err)
139	}
140	if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil {
141	t.Fatalf("SetEnv: %v", err)
142	}
143	host := "inst.example.com"
144	// The nested layout's directory pre-exists here exactly as it would on
145	// a real box: the migration pass that flips a row's DBPath/Socket to
146	// the nested defaults is what creates it, before any route can
147	// classify as nested at all.
148	if err := os.MkdirAll(provision.InstanceDir(dd, host), 0o755); err != nil {
149	t.Fatalf("mkdir instance dir: %v", err)
150	}
151	if err := reg.Upsert(registry.Route{
152	Host: host, Kind: "instance", App: "eleven", Account: "bab",
153	Status: "ready", DBPath: provision.NestedDBPath(dd, host), Socket: provision.NestedSocket(host),
154	}); err != nil {
155	t.Fatalf("Upsert: %v", err)
156	}
157	var bounced []string
158	bounce := func(h string) error { bounced = append(bounced, h); return nil }
159	if err := SyncExec(ctx, reg, s, pk, dd, bounce, "", nil); err != nil {
160	t.Fatalf("SyncExec: %v", err)
161	}
162	nestedPath := filepath.Join(provision.InstanceDir(dd, host), host+".config.env")
163	got, err := os.ReadFile(nestedPath)
164	if err != nil {
165	t.Fatalf("read nested config env: %v", err)
166	}
167	if string(got) != "LOG_LEVEL=debug\n" {
168	t.Fatalf("config env = %q, want LOG_LEVEL=debug", got)
169	}
170	if fi, _ := os.Stat(nestedPath); fi.Mode().Perm() != 0o600 {
171	t.Fatalf("nested config env mode = %v, want 0600", fi.Mode().Perm())
172	}
173	if _, err := os.Stat(filepath.Join(dd, host+".config.env")); !os.IsNotExist(err) {
174	t.Fatalf("a nested route's config also landed at the flat path, stat err = %v", err)
175	}
176	if len(bounced) != 1 || bounced[0] != host {
177	t.Fatalf("bounced = %v, want exactly [%s]", bounced, host)
178	}
179	}
180
181	// TestSyncExecSkipsAHostTheLayoutPassHolds pins the second net under the
182	// layout pass: a host with a migration journal is between two directories,
183	// so SyncExec writes no bundle for it at either path, bounces nothing, and
184	// says why, while a sibling on the same pass converges as usual.
185	func TestSyncExecSkipsAHostTheLayoutPassHolds(t *testing.T) {
186	ctx, s, reg, dd, priv, _ := execKit(t)
187	pk, err := ParsePrivateKey(priv, "test")
188	if err != nil {
189	t.Fatalf("ParsePrivateKey: %v", err)
190	}
191	if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil {
192	t.Fatalf("SetEnv: %v", err)
193	}
194	held, sibling := "inst.example.com", "other.example.com"
195	if err := os.MkdirAll(provision.InstanceDir(dd, held), 0o755); err != nil {
196	t.Fatalf("mkdir instance dir: %v", err)
197	}
198	for _, rt := range []registry.Route{
199	{Host: held, Kind: "instance", App: "eleven", Account: "bab",
200	Status: "ready", DBPath: provision.NestedDBPath(dd, held), Socket: provision.NestedSocket(held)},
201	{Host: sibling, Kind: "instance", App: "eleven", Account: "bab",
202	Status: "ready", Socket: "/run/carlos/" + sibling + ".sock"},
203	} {
204	if err := reg.Upsert(rt); err != nil {
205	t.Fatalf("Upsert: %v", err)
206	}
207	}
208	var bounced []string
209	bounce := func(h string) error { bounced = append(bounced, h); return nil }
210	var asked []string
211	fenced := func(h string) bool { asked = append(asked, h); return h == held }
212	out := captureStdout(t, func() {
213	if err := SyncExec(ctx, reg, s, pk, dd, bounce, "", nil, SkipFenced(fenced)); err != nil {
214	t.Fatalf("SyncExec: %v", err)
215	}
216	})
217	for _, path := range []string{
218	filepath.Join(provision.InstanceDir(dd, held), held+".config.env"),
219	filepath.Join(dd, held+".config.env"),
220	} {
221	if _, err := os.Lstat(path); !os.IsNotExist(err) {
222	t.Errorf("a held host's bundle was written at %s (err %v)", path, err)
223	}
224	}
225	if got, err := os.ReadFile(filepath.Join(dd, sibling+".config.env")); err != nil || string(got) != "LOG_LEVEL=debug\n" {
226	t.Errorf("sibling bundle = %q, %v", got, err)
227	}
228	if len(bounced) != 1 || bounced[0] != sibling {
229	t.Errorf("bounced = %v, want exactly [%s]", bounced, sibling)
230	}
231	if !strings.Contains(out, held+": layout migration in flight; bundle not written this pass") {
232	t.Errorf("the skip was not reported:\n%s", out)
233	}
234	if !slices.Contains(asked, held) || !slices.Contains(asked, sibling) {
235	t.Errorf("fenced was asked about %v", asked)
236	}
237	}
238
239	// TestSyncExecFIFOAtANestedBundleDoesNotHangThePass pins the review fix: a
240	// nested row's existing bundle is read through instancefs, so a FIFO a
241	// tenant planted at that name (isolated-exec design §3.3, no tenant-planted
242	// inode may block the agent) must fail fast rather than hang SyncExec's
243	// whole pass forever behind one host's blocking read(2). A sibling host on
244	// the same pass must still converge normally.
245	func TestSyncExecFIFOAtANestedBundleDoesNotHangThePass(t *testing.T) {
246	ctx, s, reg, dd, priv, _ := execKit(t)
247	pk, err := ParsePrivateKey(priv, "test")
248	if err != nil {
249	t.Fatalf("ParsePrivateKey: %v", err)
250	}
251	if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil {
252	t.Fatalf("SetEnv: %v", err)
253	}
254	if err := SetEnv(ctx, s, "bab", "twelve", map[string]string{"LOG_LEVEL": "debug"}); err != nil {
255	t.Fatalf("SetEnv: %v", err)
256	}
257	okHost, fifoHost := "ok.example.com", "fifo.example.com"
258	for _, h := range []string{okHost, fifoHost} {
259	if err := os.MkdirAll(provision.InstanceDir(dd, h), 0o755); err != nil {
260	t.Fatalf("mkdir instance dir for %s: %v", h, err)
261	}
262	}
263	if err := reg.Upsert(registry.Route{
264	Host: okHost, Kind: "instance", App: "eleven", Account: "bab",
265	Status: "ready", DBPath: provision.NestedDBPath(dd, okHost), Socket: provision.NestedSocket(okHost),
266	}); err != nil {
267	t.Fatalf("Upsert %s: %v", okHost, err)
268	}
269	if err := reg.Upsert(registry.Route{
270	Host: fifoHost, Kind: "instance", App: "twelve", Account: "bab",
271	Status: "ready", DBPath: provision.NestedDBPath(dd, fifoHost), Socket: provision.NestedSocket(fifoHost),
272	}); err != nil {
273	t.Fatalf("Upsert %s: %v", fifoHost, err)
274	}
275	// Planted where syncExecHost's existed-check reads BEFORE it ever writes
276	// anything: a tenant that got here first, not a crash artifact.
277	fifoPath := filepath.Join(provision.InstanceDir(dd, fifoHost), fifoHost+".config.env")
278	if err := unix.Mkfifo(fifoPath, 0o600); err != nil {
279	t.Fatalf("Mkfifo: %v", err)
280	}
281
282	var bounced []string
283	bounce := func(h string) error { bounced = append(bounced, h); return nil }
284	done := make(chan error, 1)
285	go func() { done <- SyncExec(ctx, reg, s, pk, dd, bounce, "", nil) }()
286	select {
287	case err := <-done:
288	if err != nil {
289	t.Fatalf("SyncExec: %v", err)
290	}
291	case <-time.After(5 * time.Second):
292	t.Fatal("SyncExec hung behind the FIFO'd host instead of failing that one host fast")
293	}
294
295	okPath := filepath.Join(provision.InstanceDir(dd, okHost), okHost+".config.env")
296	got, err := os.ReadFile(okPath)
297	if err != nil {
298	t.Fatalf("sibling host's config was not written: %v", err)
299	}
300	if string(got) != "LOG_LEVEL=debug\n" {
301	t.Fatalf("sibling config env = %q, want LOG_LEVEL=debug", got)
302	}
303	if len(bounced) != 1 || bounced[0] != okHost {
304	t.Fatalf("bounced = %v, want exactly [%s] (the FIFO'd host must not bounce)", bounced, okHost)
305	}
306	// The FIFO itself is untouched: syncExecHost's read failed before it
307	// ever reached a write.
308	if fi, err := os.Lstat(fifoPath); err != nil || fi.Mode()&os.ModeNamedPipe == 0 {
309	t.Fatalf("FIFO at %s was replaced or removed: %v %v", fifoPath, fi, err)
310	}
311	}
312
313	// TestReapConfigEnvWalksInstanceDirs pins reapConfigEnv's nested half: a
314	// stale config file inside a gone host's instance directory is removed, and
315	// a live nested route's own file survives.
316	func TestReapConfigEnvWalksInstanceDirs(t *testing.T) {
317	dd := t.TempDir()
318	reg := testRegistry(t)
319	live, gone := "live.example.com", "gone.example.com"
320	for _, host := range []string{live, gone} {
321	if err := os.MkdirAll(provision.InstanceDir(dd, host), 0o755); err != nil {
322	t.Fatalf("mkdir instance dir for %s: %v", host, err)
323	}
324	p := filepath.Join(provision.InstanceDir(dd, host), host+configEnvSuffix)
325	if err := os.WriteFile(p, []byte("A=1\n"), 0o600); err != nil {
326	t.Fatalf("stage %s: %v", host, err)
327	}
328	}
329	if err := reg.Upsert(registry.Route{
330	Host: live, Kind: "instance", App: "eleven", Account: "bab",
331	Status: "ready", DBPath: provision.NestedDBPath(dd, live), Socket: provision.NestedSocket(live),
332	}); err != nil {
333	t.Fatalf("Upsert: %v", err)
334	}
335	routes, err := reg.List()
336	if err != nil {
337	t.Fatalf("List: %v", err)
338	}
339	reapConfigEnv(routes, dd)
340
341	livePath := filepath.Join(provision.InstanceDir(dd, live), live+configEnvSuffix)
342	if _, err := os.Stat(livePath); err != nil {
343	t.Fatalf("live nested route's config was reaped: %v", err)
344	}
345	gonePath := filepath.Join(provision.InstanceDir(dd, gone), gone+configEnvSuffix)
346	if _, err := os.Stat(gonePath); !os.IsNotExist(err) {
347	t.Fatalf("stale nested config survived reap, stat err = %v", err)
348	}
349	}
350
351	func TestSyncExecRemovesFileWhenManifestDeleted(t *testing.T) {
352	ctx, s, reg, dd, priv, _ := execKit(t)
353	pk, _ := ParsePrivateKey(priv, "test")
354	if err := reg.Upsert(registry.Route{
355	Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab",
356	Status: "ready", Socket: "/run/carlos/inst.sock",
357	}); err != nil {
358	t.Fatalf("Upsert: %v", err)
359	}
360	// A config file sync wrote earlier, whose manifest object is now gone:
361	// only sync writes this namespace, so it must be removed — and the
362	// running instance bounced, or it keeps stale config until next wake.
363	path := filepath.Join(dd, "inst.example.com.config.env")
364	if err := os.WriteFile(path, []byte("OLD=1\n"), 0o600); err != nil {
365	t.Fatalf("stage: %v", err)
366	}
367	var bounced []string
368	if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error {
369	bounced = append(bounced, h)
370	return nil
371	}, "", nil); err != nil {
372	t.Fatalf("SyncExec: %v", err)
373	}
374	if _, err := os.Stat(path); !os.IsNotExist(err) {
375	t.Fatalf("config file survived manifest deletion, stat err = %v", err)
376	}
377	if len(bounced) != 1 {
378	t.Fatalf("bounced = %v, want the stale-config instance bounced once", bounced)
379	}
380	}
381
382	// TestSyncExecMassRemovalGuardRefusesWhenEveryGroupResolvesAbsent covers
383	// F4's misconfiguration guard: two distinct apps' groups both resolve
384	// "manifest absent" in the same pass (the shape a mangled registry account
385	// column produces — release.RouteAccount falls back to "ops" for every
386	// route at once, so every group's bucket key 404s together). Both
387	// existing config files must survive, and neither instance is bounced —
388	// an unguarded pass would otherwise wipe every config file on the box in
389	// one sweep from what might be a registry fault, not a real deletion.
390	func TestSyncExecMassRemovalGuardRefusesWhenEveryGroupResolvesAbsent(t *testing.T) {
391	ctx, s, reg, dd, priv, _ := execKit(t)
392	pk, _ := ParsePrivateKey(priv, "test")
393	// Neither app ever had SetEnv called: both groups' manifest objects are
394	// genuinely absent from the bucket.
395	routes := []registry.Route{
396	{Host: "a.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/a.sock"},
397	{Host: "b.example.com", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Socket: "/run/carlos/b.sock"},
398	}
399	for _, rt := range routes {
400	if err := reg.Upsert(rt); err != nil {
401	t.Fatalf("Upsert %s: %v", rt.Host, err)
402	}
403	}
404	pathA := filepath.Join(dd, "a.example.com.config.env")
405	pathB := filepath.Join(dd, "b.example.com.config.env")
406	if err := os.WriteFile(pathA, []byte("OLD=1\n"), 0o600); err != nil {
407	t.Fatalf("stage a: %v", err)
408	}
409	if err := os.WriteFile(pathB, []byte("OLD=2\n"), 0o600); err != nil {
410	t.Fatalf("stage b: %v", err)
411	}
412	var bounced []string
413	if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error {
414	bounced = append(bounced, h)
415	return nil
416	}, "", nil); err != nil {
417	t.Fatalf("SyncExec: %v", err)
418	}
419	if _, err := os.Stat(pathA); err != nil {
420	t.Fatalf("a's config file was removed despite the mass-removal guard: %v", err)
421	}
422	if _, err := os.Stat(pathB); err != nil {
423	t.Fatalf("b's config file was removed despite the mass-removal guard: %v", err)
424	}
425	if len(bounced) != 0 {
426	t.Fatalf("bounced = %v, want none — the guard must hold back both the removal and the bounce", bounced)
427	}
428	}
429
430	// TestSyncExecMassRemovalGuardDoesNotBlockNormalConvergence covers the
431	// guard's other edge: with two apps in the pass but only ONE absent, that
432	// one's file is still removed normally — the guard only fires when EVERY
433	// group is absent, so a real single-app deletion on a multi-app box must
434	// keep converging.
435	func TestSyncExecMassRemovalGuardDoesNotBlockNormalConvergence(t *testing.T) {
436	ctx, s, reg, dd, priv, _ := execKit(t)
437	pk, _ := ParsePrivateKey(priv, "test")
438	if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil {
439	t.Fatalf("SetEnv: %v", err)
440	}
441	// titogo's manifest is never set: absent, but eleven's is present, so
442	// this pass is NOT unanimous.
443	routes := []registry.Route{
444	{Host: "a.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/a.sock"},
445	{Host: "b.example.com", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Socket: "/run/carlos/b.sock"},
446	}
447	for _, rt := range routes {
448	if err := reg.Upsert(rt); err != nil {
449	t.Fatalf("Upsert %s: %v", rt.Host, err)
450	}
451	}
452	pathA := filepath.Join(dd, "a.example.com.config.env")
453	pathB := filepath.Join(dd, "b.example.com.config.env")
454	if err := os.WriteFile(pathB, []byte("OLD=2\n"), 0o600); err != nil {
455	t.Fatalf("stage b: %v", err)
456	}
457	var bounced []string
458	if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error {
459	bounced = append(bounced, h)
460	return nil
461	}, "", nil); err != nil {
462	t.Fatalf("SyncExec: %v", err)
463	}
464	if _, err := os.Stat(pathA); err != nil {
465	t.Fatalf("a's config file wasn't written: %v", err)
466	}
467	if _, err := os.Stat(pathB); !os.IsNotExist(err) {
468	t.Fatalf("b's config file survived its manifest's absence, stat err = %v", err)
469	}
470	// a's file is a fresh write (its manifest has real content, unlike
471	// b's), so it bounces too — both changes converge normally.
472	if !slices.Contains(bounced, "a.example.com") || !slices.Contains(bounced, "b.example.com") || len(bounced) != 2 {
473	t.Fatalf("bounced = %v, want exactly [a.example.com b.example.com] (order not guaranteed)", bounced)
474	}
475	}
476
477	// TestSyncExecRendersEachRoutesBoundEnvironment is the exec-path twin of
478	// sync.go's TestSyncRendersEachRoutesBoundEnvironment: two instances of ONE
479	// app, one bound to a named environment and one unbound, must land
480	// DIFFERENT files in the same pass — the bound one gets the default bundle
481	// overlaid by its environment (environment wins per key), the unbound one
482	// gets the default bundle alone. Grouping on (acct, app) alone would render
483	// whichever route came first into both.
484	func TestSyncExecRendersEachRoutesBoundEnvironment(t *testing.T) {
485	ctx, s, reg, dd, priv, _ := execKit(t)
486	pk, err := ParsePrivateKey(priv, "test")
487	if err != nil {
488	t.Fatalf("ParsePrivateKey: %v", err)
489	}
490	if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{
491	"LOG_LEVEL": "info",
492	"BASE_ONLY": "1",
493	}); err != nil {
494	t.Fatalf("SetEnv: %v", err)
495	}
496	if err := SaveEnvironment(ctx, s, "bab", "eleven", "demo", map[string]Var{
497	"LOG_LEVEL": {Kind: "env", Value: "debug"}, // overrides the default
498	"DEMO_ONLY": {Kind: "env", Value: "yes"}, // adds
499	}, nil); err != nil {
500	t.Fatalf("SaveEnvironment: %v", err)
501	}
502	routes := []registry.Route{
503	{Host: "demo.example.com", Kind: "instance", App: "eleven", Account: "bab",
504	Status: "ready", Socket: "/run/carlos/demo.sock", Environment: "demo"},
505	{Host: "plain.example.com", Kind: "instance", App: "eleven", Account: "bab",
506	Status: "ready", Socket: "/run/carlos/plain.sock"},
507	}
508	for _, rt := range routes {
509	if err := reg.Upsert(rt); err != nil {
510	t.Fatalf("Upsert %s: %v", rt.Host, err)
511	}
512	}
513	if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil {
514	t.Fatalf("SyncExec: %v", err)
515	}
516	want := map[string]string{
517	"demo.example.com": "BASE_ONLY=1\nDEMO_ONLY=yes\nLOG_LEVEL=debug\n",
518	"plain.example.com": "BASE_ONLY=1\nLOG_LEVEL=info\n",
519	}
520	for host, w := range want {
521	got, err := os.ReadFile(filepath.Join(dd, host+".config.env"))
522	if err != nil {
523	t.Fatalf("read %s config env: %v", host, err)
524	}
525	if string(got) != w {
526	t.Fatalf("%s config env = %q, want %q", host, got, w)
527	}
528	}
529	}
530
531	// TestSyncExecLeavesFileUntouchedWhenBoundEnvironmentIsMissing pins the
532	// distinction that matters most on this path: a bound environment that is
533	// not in the bucket is an ERROR, never absence. Absence removes the config
534	// file here (sync owns the namespace) — so folding the two together would
535	// strip a live instance's whole config, and bounce it onto the stripped
536	// file, because someone deleted a bundle. The file must survive untouched
537	// and the route must not be bounced.
538	func TestSyncExecLeavesFileUntouchedWhenBoundEnvironmentIsMissing(t *testing.T) {
539	ctx, s, reg, dd, priv, _ := execKit(t)
540	pk, _ := ParsePrivateKey(priv, "test")
541	if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "info"}); err != nil {
542	t.Fatalf("SetEnv: %v", err)
543	}
544	if err := reg.Upsert(registry.Route{
545	Host: "ghost.example.com", Kind: "instance", App: "eleven", Account: "bab",
546	Status: "ready", Socket: "/run/carlos/g.sock", Environment: "never-created",
547	}); err != nil {
548	t.Fatalf("Upsert: %v", err)
549	}
550	// The last good render, from before the bundle went missing.
551	path := filepath.Join(dd, "ghost.example.com.config.env")
552	const lastGood = "LOG_LEVEL=trace\n"
553	if err := os.WriteFile(path, []byte(lastGood), 0o600); err != nil {
554	t.Fatalf("stage: %v", err)
555	}
556	var bounced []string
557	if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error {
558	bounced = append(bounced, h)
559	return nil
560	}, "", nil); err != nil {
561	t.Fatalf("SyncExec: %v", err)
562	}
563	got, err := os.ReadFile(path)
564	if err != nil {
565	t.Fatalf("config file removed on an unresolved environment: %v", err)
566	}
567	if string(got) != lastGood {
568	t.Fatalf("config env = %q, want it left exactly as-is (%q)", got, lastGood)
569	}
570	if len(bounced) != 0 {
571	t.Fatalf("bounced = %v, want none — an unresolved route is not a changed route", bounced)
572	}
573	}
574
575	// TestSyncExecOverlayOnlyAppDropsStaleKeys pins the deliberate divergence
576	// from the unit path: SyncExec has NO lostKeys/defaultMissing guard, and
577	// must not grow one.
578	//
579	// This is the exact shape sync.go REFUSES (2026-08-02 review, CRITICAL 1):
580	// a route bound to an environment whose app has no default config.json at
581	// all, so the merged view is the overlay bundle ALONE, dropping a key the
582	// file on disk already holds. On the unit path that refusal is right —
583	// /etc/carlos/apps/<host>.env may have been hand-staged by an operator
584	// before the app's config was ever imported into the bucket, so those keys
585	// are not sync's to delete, and the documented migration step (create a
586	// small override, bind the routes) would otherwise wipe them as root.
587	//
588	// Here it is wrong. Every byte of a .config.env was rendered by a previous
589	// pass of SyncExec from a manifest — the namespace is sync-owned by
590	// construction, nothing hand-staged can be in it — so a key the merged
591	// view no longer contains is stale config that was removed from the
592	// bucket, and dropping it IS the convergence. The instance's own
593	// <host>.env, the one hand-staged file in this directory, is a separate
594	// file layered on top at spawn and never read or written by this sweep.
595	//
596	// Import the guard and this test fails, which is the point.
597	func TestSyncExecOverlayOnlyAppDropsStaleKeys(t *testing.T) {
598	ctx, s, reg, dd, priv, _ := execKit(t)
599	pk, _ := ParsePrivateKey(priv, "test")
600	// No SetEnv for eleven: the app has NO default bundle, only an
601	// environment overlay. STALE is in neither.
602	if err := SaveEnvironment(ctx, s, "bab", "eleven", "demo", map[string]Var{
603	"KEPT": {Kind: "env", Value: "1"},
604	}, nil); err != nil {
605	t.Fatalf("SaveEnvironment: %v", err)
606	}
607	if err := reg.Upsert(registry.Route{
608	Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab",
609	Status: "ready", Socket: "/run/carlos/inst.sock", Environment: "demo",
610	}); err != nil {
611	t.Fatalf("Upsert: %v", err)
612	}
613	// A file an earlier pass rendered, back when STALE was still in the
614	// bundle. Sync wrote it, so sync gets to drop it.
615	path := filepath.Join(dd, "inst.example.com.config.env")
616	if err := os.WriteFile(path, []byte("KEPT=1\nSTALE=old\n"), 0o600); err != nil {
617	t.Fatalf("stage: %v", err)
618	}
619	var bounced []string
620	if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error {
621	bounced = append(bounced, h)
622	return nil
623	}, "", nil); err != nil {
624	t.Fatalf("SyncExec: %v", err)
625	}
626	got, err := os.ReadFile(path)
627	if err != nil {
628	t.Fatalf("read config env: %v", err)
629	}
630	if string(got) != "KEPT=1\n" {
631	t.Fatalf("config env = %q, want the overlay ALONE (KEPT=1) — a key-loss guard must not be imported onto this path", got)
632	}
633	if len(bounced) != 1 || bounced[0] != "inst.example.com" {
634	t.Fatalf("bounced = %v, want exactly one bounce for the dropped key", bounced)
635	}
636	}
637
638	// TestSyncExecMassRemovalGuardIgnoresErroredGroups: an app whose bound
639	// environment is missing is in neither the absent nor the present column,
640	// so it must not sit in the guard's denominator and disarm it. Two
641	// genuinely-absent groups plus one errored group is still "every resolvable
642	// group is absent" — the guard fires, and the errored route's file is left
643	// alone by the error path regardless.
644	func TestSyncExecMassRemovalGuardIgnoresErroredGroups(t *testing.T) {
645	ctx, s, reg, dd, priv, _ := execKit(t)
646	pk, _ := ParsePrivateKey(priv, "test")
647	// keymail has a default bundle but its bound environment is missing:
648	// that group resolves to an error, not to absence.
649	if err := SetEnv(ctx, s, "bab", "keymail", map[string]string{"K": "1"}); err != nil {
650	t.Fatalf("SetEnv: %v", err)
651	}
652	routes := []registry.Route{
653	{Host: "a.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/a.sock"},
654	{Host: "b.example.com", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Socket: "/run/carlos/b.sock"},
655	{Host: "c.example.com", Kind: "instance", App: "keymail", Account: "bab", Status: "ready", Socket: "/run/carlos/c.sock", Environment: "gone"},
656	}
657	for _, rt := range routes {
658	if err := reg.Upsert(rt); err != nil {
659	t.Fatalf("Upsert %s: %v", rt.Host, err)
660	}
661	}
662	for _, h := range []string{"a.example.com", "b.example.com", "c.example.com"} {
663	if err := os.WriteFile(filepath.Join(dd, h+".config.env"), []byte("OLD=1\n"), 0o600); err != nil {
664	t.Fatalf("stage %s: %v", h, err)
665	}
666	}
667	var bounced []string
668	if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error {
669	bounced = append(bounced, h)
670	return nil
671	}, "", nil); err != nil {
672	t.Fatalf("SyncExec: %v", err)
673	}
674	for _, h := range []string{"a.example.com", "b.example.com", "c.example.com"} {
675	if _, err := os.Stat(filepath.Join(dd, h+".config.env")); err != nil {
676	t.Fatalf("%s: config file removed despite the mass-removal guard: %v", h, err)
677	}
678	}
679	if len(bounced) != 0 {
680	t.Fatalf("bounced = %v, want none", bounced)
681	}
682	}
683
684	func TestSyncExecNeverTouchesInstanceEnvOrOutOfScopeRoutes(t *testing.T) {
685	ctx, s, reg, dd, priv, _ := execKit(t)
686	pk, _ := ParsePrivateKey(priv, "test")
687	if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"A": "1"}); err != nil {
688	t.Fatalf("SetEnv: %v", err)
689	}
690	// The instance's own env file must never be read, changed, or reaped.
691	own := filepath.Join(dd, "inst.example.com.env")
692	if err := os.WriteFile(own, []byte("CARLOS_ADMIN_TOKEN=tok\n"), 0o600); err != nil {
693	t.Fatalf("stage own env: %v", err)
694	}
695	routes := []registry.Route{
696	{Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/inst.sock"},
697	{Host: "unit.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/u.sock", Backing: registry.BackingUnit},
698	{Host: "proxy.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Addr: "127.0.0.1:8444"},
699	{Host: "alias.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/a.sock", Platform: true},
700	{Host: "pending.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "provisioning", Socket: "/run/carlos/p.sock"},
701	}
702	for _, rt := range routes {
703	if err := reg.Upsert(rt); err != nil {
704	t.Fatalf("Upsert %s: %v", rt.Host, err)
705	}
706	}
707	if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil {
708	t.Fatalf("SyncExec: %v", err)
709	}
710	if got, _ := os.ReadFile(own); string(got) != "CARLOS_ADMIN_TOKEN=tok\n" {
711	t.Fatalf("instance env file was touched: %q", got)
712	}
713	if _, err := os.Stat(filepath.Join(dd, "inst.example.com.config.env")); err != nil {
714	t.Fatalf("in-scope route got no config file: %v", err)
715	}
716	for _, h := range []string{"unit.example.com", "proxy.example.com", "alias.example.com", "pending.example.com"} {
717	if _, err := os.Stat(filepath.Join(dd, h+".config.env")); !os.IsNotExist(err) {
718	t.Fatalf("out-of-scope route %s got a config file, stat err = %v", h, err)
719	}
720	}
721	}
722
723	func TestSyncExecReapsOnlyItsOwnNamespace(t *testing.T) {
724	ctx, s, reg, dd, priv, _ := execKit(t)
725	pk, _ := ParsePrivateKey(priv, "test")
726	if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"X": "1"}); err != nil {
727	t.Fatalf("SetEnv: %v", err)
728	}
729	if err := reg.Upsert(registry.Route{
730	Host: "live.example.com", Kind: "instance", App: "eleven", Account: "bab",
731	Status: "ready", Socket: "/run/carlos/l.sock",
732	}); err != nil {
733	t.Fatalf("Upsert: %v", err)
734	}
735	files := map[string]bool{ // name -> should survive
736	"live.example.com.config.env": true, // has a route
737	"gone.example.com.config.env": false, // no route: reaped
738	"gone.example.com.config.env.tmp": false, // stale tmp: reaped
739	"gone.example.com.env": true, // NOT ours: instance file, never reaped
740	"gone.example.com-sidecar.env": true, // NOT ours: sidecar file, never reaped
741	}
742	for name := range files {
743	if err := os.WriteFile(filepath.Join(dd, name), []byte("X=1\n"), 0o600); err != nil {
744	t.Fatalf("stage %s: %v", name, err)
745	}
746	}
747	if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil {
748	t.Fatalf("SyncExec: %v", err)
749	}
750	for name, keep := range files {
751	_, err := os.Stat(filepath.Join(dd, name))
752	if keep && err != nil {
753	t.Fatalf("%s: want kept, got %v", name, err)
754	}
755	if !keep && !os.IsNotExist(err) {
756	t.Fatalf("%s: want reaped, stat err = %v", name, err)
757	}
758	}
759	}
760
761	// TestSyncExecReapsOnlyItsOwnNamespaceWithAddrShapeInTheMix is the reap
762	// test's issue #18 twin: an opted-in addr route's own file survives reap
763	// (it has a route row, same as any in-scope route), and a stray
764	// *.config.env with NO route row at all is reaped regardless of which
765	// exec shape produced it in some earlier pass — reap keys purely on
766	// (namespace suffix, route existence), never on shape or opt-in.
767	func TestSyncExecReapsOnlyItsOwnNamespaceWithAddrShapeInTheMix(t *testing.T) {
768	ctx, s, reg, dd, priv, _ := execKit(t)
769	pk, _ := ParsePrivateKey(priv, "test")
770	if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"X": "1"}); err != nil {
771	t.Fatalf("SetEnv: %v", err)
772	}
773	if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil {
774	t.Fatalf("SetExecDelivery: %v", err)
775	}
776	if err := reg.Upsert(registry.Route{
777	Host: "live.go.tito.io", Kind: "instance", App: "titogo", Account: "bab",
778	Status: "ready", Addr: "127.0.0.1:8444",
779	}); err != nil {
780	t.Fatalf("Upsert: %v", err)
781	}
782	files := map[string]bool{ // name -> should survive
783	"live.go.tito.io.config.env": true, // opted-in addr route, has a route row
784	"gone.go.tito.io.config.env": false, // no route row at all: reaped, whatever shape wrote it
785	}
786	for name := range files {
787	if err := os.WriteFile(filepath.Join(dd, name), []byte("X=1\n"), 0o600); err != nil {
788	t.Fatalf("stage %s: %v", name, err)
789	}
790	}
791	var bounced []string
792	if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error {
793	bounced = append(bounced, h)
794	return nil
795	}, "", nil); err != nil {
796	t.Fatalf("SyncExec: %v", err)
797	}
798	for name, keep := range files {
799	_, err := os.Stat(filepath.Join(dd, name))
800	if keep && err != nil {
801	t.Fatalf("%s: want kept, got %v", name, err)
802	}
803	if !keep && !os.IsNotExist(err) {
804	t.Fatalf("%s: want reaped, stat err = %v", name, err)
805	}
806	}
807	// The reap pass itself must never bounce — reap's log line says "no
808	// route on this box", never "bounce"; and the addr route's own
809	// convergence (a fresh, unchanged-content write above) must not
810	// bounce either.
811	if len(bounced) != 0 {
812	t.Fatalf("bounced = %v, want none", bounced)
813	}
814	}
815
816	func TestSyncExecEmptyRegistryRemovesNothing(t *testing.T) {
817	ctx, s, reg, dd, priv, _ := execKit(t)
818	pk, _ := ParsePrivateKey(priv, "test")
819	path := filepath.Join(dd, "orphan.example.com.config.env")
820	if err := os.WriteFile(path, []byte("X=1\n"), 0o600); err != nil {
821	t.Fatalf("stage: %v", err)
822	}
823	if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil {
824	t.Fatalf("SyncExec: %v", err)
825	}
826	if _, err := os.Stat(path); err != nil {
827	t.Fatalf("empty registry removed a config file: %v", err)
828	}
829	}
830
831	func TestSyncExecRefusesNonBareHostname(t *testing.T) {
832	_, _, _, dd, priv, _ := execKit(t) // routes injected directly via the per-route helper below
833	pk, _ := ParsePrivateKey(priv, "test")
834	m := &Manifest{Schema: 1, App: "eleven", Vars: map[string]Var{"A": {Kind: "env", Value: "1"}}}
835	for _, host := range []string{"../escape", "a/b", "", ".", ".."} {
836	syncExecHost(registry.Route{Host: host}, m, "apps/bab/eleven/config.json", pk, dd, func(string) error { return nil }, "", "", nil)
837	}
838	ents, err := os.ReadDir(dd)
839	if err != nil {
840	t.Fatalf("read dataDir: %v", err)
841	}
842	if len(ents) != 0 {
843	t.Fatalf("dataDir = %v, want nothing written for non-bare hostnames", ents)
844	}
845	}
846
847	// TestSyncExecOptedInAddrRouteMaterializesFile is issue #18's core case:
848	// a TCP-addr route (Socket == "", Addr != "") whose app opted in via
849	// ExecDeliveryFile in its default bundle gets the identical
850	// <host>.config.env a socket-exec route would — same rendering, same
851	// layering guarantees — even though carlos neither spawned nor owns the
852	// process behind it.
853	func TestSyncExecOptedInAddrRouteMaterializesFile(t *testing.T) {
854	ctx, s, reg, dd, priv, _ := execKit(t)
855	pk, err := ParsePrivateKey(priv, "test")
856	if err != nil {
857	t.Fatalf("ParsePrivateKey: %v", err)
858	}
859	if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"TITOGO_DEMO_BANNER": "1"}); err != nil {
860	t.Fatalf("SetEnv: %v", err)
861	}
862	if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil {
863	t.Fatalf("SetExecDelivery: %v", err)
864	}
865	if err := reg.Upsert(registry.Route{
866	Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab",
867	Status: "ready", Addr: "127.0.0.1:8444",
868	}); err != nil {
869	t.Fatalf("Upsert: %v", err)
870	}
871	if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil {
872	t.Fatalf("SyncExec: %v", err)
873	}
874	got, err := os.ReadFile(filepath.Join(dd, "demo.go.tito.io.config.env"))
875	if err != nil {
876	t.Fatalf("opted-in TCP-addr route got no config file: %v", err)
877	}
878	want := execAddrConfigHeader + "TITOGO_DEMO_BANNER=1\n"
879	if string(got) != want {
880	t.Fatalf("config env = %q, want %q (issue #18's required header, then the vars)", got, want)
881	}
882
883	// Idempotence must survive the header (2026-08-03 re-review, IMPORTANT
884	// 7): the up-to-date comparison has to include the header bytes, or an
885	// unchanged pass rewrites this secrets-bearing root-owned file every
886	// tick forever. The socket shape has this pin; before this assertion,
887	// moving the header prepend after the comparison left the whole suite
888	// green. mtime is the observable: a second pass over unchanged config
889	// must not touch the file.
890	path := filepath.Join(dd, "demo.go.tito.io.config.env")
891	before, err := os.Stat(path)
892	if err != nil {
893	t.Fatal(err)
894	}
895	time.Sleep(10 * time.Millisecond) // ensure a rewrite would move mtime
896	if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil {
897	t.Fatalf("SyncExec (second pass): %v", err)
898	}
899	after, err := os.Stat(path)
900	if err != nil {
901	t.Fatal(err)
902	}
903	if !after.ModTime().Equal(before.ModTime()) {
904	t.Fatalf("an unchanged second pass rewrote the config file (mtime %v -> %v)", before.ModTime(), after.ModTime())
905	}
906	}
907
908	// TestSyncExecOptedInMultiAddrRouteMaterializesFile pins the HA registry
909	// spelling of the externally managed TCP route. Addrs replaced Addr for HA
910	// routes, including one-upstream routes, and must enter the same opt-in path:
911	// carlos writes their file but never claims to own or bounce the process.
912	func TestSyncExecOptedInMultiAddrRouteMaterializesFile(t *testing.T) {
913	ctx, s, reg, dd, priv, _ := execKit(t)
914	pk, err := ParsePrivateKey(priv, "test")
915	if err != nil {
916	t.Fatalf("ParsePrivateKey: %v", err)
917	}
918	if err := SetEnv(ctx, s, "bab", "woodstar", map[string]string{"PROVISION_TOKEN": "sealed-value"}); err != nil {
919	t.Fatalf("SetEnv: %v", err)
920	}
921	if err := SetExecDelivery(ctx, s, "bab", "woodstar", ExecDeliveryFile); err != nil {
922	t.Fatalf("SetExecDelivery: %v", err)
923	}
924	if err := reg.Upsert(registry.Route{
925	Host: "woodstar.example.com", Kind: "instance", App: "woodstar", Account: "bab",
926	Status: "ready", Addrs: "127.0.0.1:8451",
927	}); err != nil {
928	t.Fatalf("Upsert: %v", err)
929	}
930	bounces := 0
931	if err := SyncExec(ctx, reg, s, pk, dd, func(string) error {
932	bounces++
933	return nil
934	}, "", nil); err != nil {
935	t.Fatalf("SyncExec: %v", err)
936	}
937	got, err := os.ReadFile(filepath.Join(dd, "woodstar.example.com.config.env"))
938	if err != nil {
939	t.Fatalf("opted-in multi-addr route got no config file: %v", err)
940	}
941	want := execAddrConfigHeader + "PROVISION_TOKEN=sealed-value\n"
942	if string(got) != want {
943	t.Fatalf("config env = %q, want %q", got, want)
944	}
945	if bounces != 0 {
946	t.Fatalf("multi-addr route bounced %d times, want zero: its supervisor owns restarts", bounces)
947	}
948	}
949
950	// TestSyncExecUnoptedAddrRouteIsSkipped pins the un-opted case: the exact
951	// same TCP-addr route shape, but its app never set exec_delivery, is
952	// skipped exactly as before issue #18's third case existed — no file, no
953	// error, nothing to converge or remove.
954	func TestSyncExecUnoptedAddrRouteIsSkipped(t *testing.T) {
955	ctx, s, reg, dd, priv, _ := execKit(t)
956	pk, err := ParsePrivateKey(priv, "test")
957	if err != nil {
958	t.Fatalf("ParsePrivateKey: %v", err)
959	}
960	if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"TITOGO_DEMO_BANNER": "1"}); err != nil {
961	t.Fatalf("SetEnv: %v", err)
962	}
963	// Deliberately no SetExecDelivery call.
964	if err := reg.Upsert(registry.Route{
965	Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab",
966	Status: "ready", Addr: "127.0.0.1:8444",
967	}); err != nil {
968	t.Fatalf("Upsert: %v", err)
969	}
970	if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil {
971	t.Fatalf("SyncExec: %v", err)
972	}
973	if _, err := os.Stat(filepath.Join(dd, "demo.go.tito.io.config.env")); !os.IsNotExist(err) {
974	t.Fatalf("un-opted TCP-addr route got a config file, stat err = %v", err)
975	}
976	}
977
978	// TestSyncExecUnrecognizedExecDeliveryValueIgnoredNotFatal covers additive
979	// discipline (AGENTS.md): a manifest a NEWER carlos wrote, carrying an
980	// exec_delivery value this binary doesn't understand, must not brick this
981	// box's sync — the whole pass must still succeed, the unknown value must
982	// be treated as "not opted in" (not a fatal error, not silently treated
983	// as "file"), and OTHER apps' routes in the same pass must converge
984	// normally.
985	func TestSyncExecUnrecognizedExecDeliveryValueIgnoredNotFatal(t *testing.T) {
986	ctx, s, reg, dd, priv, _ := execKit(t)
987	pk, err := ParsePrivateKey(priv, "test")
988	if err != nil {
989	t.Fatalf("ParsePrivateKey: %v", err)
990	}
991	// Written directly, bypassing SetExecDelivery's set-time refusal — the
992	// shape of a manifest a newer binary produced.
993	raw := []byte(`{"schema":1,"app":"titogo","vars":{"K":{"kind":"env","value":"1"}},"exec_delivery":"grpc"}`)
994	if _, err := s.Put(ctx, Key("bab", "titogo"), raw, ""); err != nil {
995	t.Fatalf("Put: %v", err)
996	}
997	// A second, ordinary socket-exec app in the same pass, to prove the
998	// unknown value doesn't abort or otherwise disturb the rest of the
999	// sweep.
1000	if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil {
1001	t.Fatalf("SetEnv: %v", err)
1002	}
1003	routes := []registry.Route{
1004	{Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Addr: "127.0.0.1:8444"},
1005	{Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/inst.sock"},
1006	}
1007	for _, rt := range routes {
1008	if err := reg.Upsert(rt); err != nil {
1009	t.Fatalf("Upsert %s: %v", rt.Host, err)
1010	}
1011	}
1012	var syncErr error
1013	out := captureStdout(t, func() {
1014	syncErr = SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil)
1015	})
1016	if syncErr != nil {
1017	t.Fatalf("SyncExec: %v", syncErr)
1018	}
1019	if _, err := os.Stat(filepath.Join(dd, "demo.go.tito.io.config.env")); !os.IsNotExist(err) {
1020	t.Fatalf("unrecognized exec_delivery value was treated as opted-in, stat err = %v", err)
1021	}
1022	if _, err := os.Stat(filepath.Join(dd, "inst.example.com.config.env")); err != nil {
1023	t.Fatalf("eleven's ordinary socket route did not converge despite titogo's unrecognized value: %v", err)
1024	}
1025	// MINOR 5 (2026-08-03 review): the whole safety story for this case is
1026	// "ignored LOUDLY" — pin the loudness itself, not just the tolerance,
1027	// so a future edit that quietly dropped the fmt.Printf would fail a
1028	// test instead of only being noticed by someone reading a log by hand.
1029	if !strings.Contains(out, "titogo") || !strings.Contains(out, "grpc") {
1030	t.Fatalf("stdout = %q, want a line naming the app (titogo) and the unrecognized value (grpc)", out)
1031	}
1032	}
1033
1034	// TestSyncExecOptedInAddrRouteFetchFailureLogsAndLeavesFileAsIs is
1035	// IMPORTANT 1 from the 2026-08-03 review: a transient default-bundle read
1036	// failure against an ALREADY-opted-in app must not go silent. Before this
1037	// fix, checkOptedIn folded def.err != nil into the same "not opted in,
1038	// say nothing" branch as a genuinely never-configured app — so an opted-in
1039	// app whose bucket blips became indistinguishable, in the log, from an
1040	// app that was never opted in at all, on the exact path this feature
1041	// exists to make observable. The safety property (the file survives
1042	// byte-identical) was already correct; only the silence was the bug.
1043	func TestSyncExecOptedInAddrRouteFetchFailureLogsAndLeavesFileAsIs(t *testing.T) {
1044	ctx, s, reg, dd, priv, _ := execKit(t)
1045	pk, err := ParsePrivateKey(priv, "test")
1046	if err != nil {
1047	t.Fatalf("ParsePrivateKey: %v", err)
1048	}
1049	if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"TITOGO_DEMO_BANNER": "1"}); err != nil {
1050	t.Fatalf("SetEnv: %v", err)
1051	}
1052	if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil {
1053	t.Fatalf("SetExecDelivery: %v", err)
1054	}
1055	if err := reg.Upsert(registry.Route{
1056	Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab",
1057	Status: "ready", Addr: "127.0.0.1:8444",
1058	}); err != nil {
1059	t.Fatalf("Upsert: %v", err)
1060	}
1061	// First pass materializes normally — this is the file a transient
1062	// blip must leave exactly as it is.
1063	if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil {
1064	t.Fatalf("SyncExec (initial): %v", err)
1065	}
1066	path := filepath.Join(dd, "demo.go.tito.io.config.env")
1067	original, err := os.ReadFile(path)
1068	if err != nil {
1069	t.Fatalf("read initial config env: %v", err)
1070	}
1071
1072	blipErr := errors.New("s3: transient 503 SlowDown")
1073	failing := &failingKeyStore{Store: s, failKey: Key("bab", "titogo"), err: blipErr}
1074	var bounced []string
1075	bounce := func(h string) error { bounced = append(bounced, h); return nil }
1076	var syncErr error
1077	out := captureStdout(t, func() {
1078	syncErr = SyncExec(ctx, reg, failing, pk, dd, bounce, "", nil)
1079	})
1080	if syncErr != nil {
1081	t.Fatalf("SyncExec (blip): %v", syncErr)
1082	}
1083	got, err := os.ReadFile(path)
1084	if err != nil {
1085	t.Fatalf("read config env after the blip: %v", err)
1086	}
1087	if string(got) != string(original) {
1088	t.Fatalf("config env changed across a transient fetch failure: got %q, want the original %q untouched", got, original)
1089	}
1090	if len(bounced) != 0 {
1091	t.Fatalf("bounced = %v, want none — a fetch failure is not a change", bounced)
1092	}
1093	// The fix: this must no longer be silent. Name the app and the
1094	// underlying error, the same "unresolved ... left as-is" contract the
1095	// socket path's resolution loop already gives an ordinary fetch
1096	// failure.
1097	if !strings.Contains(out, "titogo") {
1098	t.Fatalf("stdout = %q, want a line naming the app (titogo)", out)
1099	}
1100	if !strings.Contains(out, "SlowDown") {
1101	t.Fatalf("stdout = %q, want the underlying fetch error surfaced", out)
1102	}
1103	// The mutual exclusion the ruling depends on (2026-08-03 re-review,
1104	// IMPORTANT 6): a transient blip must print ONLY the unresolved line —
1105	// never the stale-secrets notice, whose remedy is "delete this file by
1106	// hand". Dropping !c.known from that notice's guard would turn every
1107	// bucket blip into an instruction to rm a live production file
1108	// mid-incident, and before this assertion the whole suite stayed green
1109	// under exactly that mutation.
1110	if strings.Contains(out, "delete it by hand") {
1111	t.Fatalf("stdout = %q — a transient fetch failure produced the stale-file removal advice", out)
1112	}
1113	}
1114
1115	// TestSyncExecOptedInAddrRouteIsNeverBounced is the structural pin the
1116	// issue's design comment demands: carlos does not own an app-supervised
1117	// process, so an opted-in TCP-addr route's bounce callback must not even
1118	// be invoked — not "invoked and then skipped by a flag", genuinely never
1119	// called. The recorder below would append on any call at all; it must
1120	// stay empty across a pass that DOES change the route's file content
1121	// (proving the file write path ran) and a second pass where content is
1122	// unchanged (proving the still-quiet path also never bounces).
1123	func TestSyncExecOptedInAddrRouteIsNeverBounced(t *testing.T) {
1124	ctx, s, reg, dd, priv, _ := execKit(t)
1125	pk, err := ParsePrivateKey(priv, "test")
1126	if err != nil {
1127	t.Fatalf("ParsePrivateKey: %v", err)
1128	}
1129	if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"TITOGO_DEMO_BANNER": "1"}); err != nil {
1130	t.Fatalf("SetEnv: %v", err)
1131	}
1132	if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil {
1133	t.Fatalf("SetExecDelivery: %v", err)
1134	}
1135	if err := reg.Upsert(registry.Route{
1136	Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab",
1137	Status: "ready", Addr: "127.0.0.1:8444",
1138	}); err != nil {
1139	t.Fatalf("Upsert: %v", err)
1140	}
1141	var bounced []string
1142	bounce := func(h string) error {
1143	bounced = append(bounced, h)
1144	return nil
1145	}
1146	if err := SyncExec(ctx, reg, s, pk, dd, bounce, "", nil); err != nil {
1147	t.Fatalf("SyncExec (initial write): %v", err)
1148	}
1149	if len(bounced) != 0 {
1150	t.Fatalf("bounced = %v, want none — carlos does not own this process", bounced)
1151	}
1152	// Change the config so the file's content changes on the next pass —
1153	// the socket-exec equivalent of this WOULD bounce.
1154	if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"TITOGO_DEMO_BANNER": "2"}); err != nil {
1155	t.Fatalf("SetEnv (change): %v", err)
1156	}
1157	if err := SyncExec(ctx, reg, s, pk, dd, bounce, "", nil); err != nil {
1158	t.Fatalf("SyncExec (changed content): %v", err)
1159	}
1160	if len(bounced) != 0 {
1161	t.Fatalf("bounced = %v after a content change, want none", bounced)
1162	}
1163	got, err := os.ReadFile(filepath.Join(dd, "demo.go.tito.io.config.env"))
1164	if err != nil {
1165	t.Fatalf("read config env: %v", err)
1166	}
1167	want := execAddrConfigHeader + "TITOGO_DEMO_BANNER=2\n"
1168	if string(got) != want {
1169	t.Fatalf("config env = %q, want %q — the changed content, header included — the write path must still run", got, want)
1170	}
1171	}
1172
1173	// TestSyncExecSocketRouteStillBouncesAlongsideOptedInAddrRoute proves the
1174	// addr shape's no-bounce rule is scoped to addr routes ONLY: a socket-exec
1175	// route for a DIFFERENT app in the same pass must still bounce on change,
1176	// unaffected by the addr route sharing the sweep.
1177	func TestSyncExecSocketRouteStillBouncesAlongsideOptedInAddrRoute(t *testing.T) {
1178	ctx, s, reg, dd, priv, _ := execKit(t)
1179	pk, err := ParsePrivateKey(priv, "test")
1180	if err != nil {
1181	t.Fatalf("ParsePrivateKey: %v", err)
1182	}
1183	if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"A": "1"}); err != nil {
1184	t.Fatalf("SetEnv: %v", err)
1185	}
1186	if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil {
1187	t.Fatalf("SetExecDelivery: %v", err)
1188	}
1189	if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"B": "1"}); err != nil {
1190	t.Fatalf("SetEnv: %v", err)
1191	}
1192	routes := []registry.Route{
1193	{Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Addr: "127.0.0.1:8444"},
1194	{Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/inst.sock"},
1195	}
1196	for _, rt := range routes {
1197	if err := reg.Upsert(rt); err != nil {
1198	t.Fatalf("Upsert %s: %v", rt.Host, err)
1199	}
1200	}
1201	var bounced []string
1202	bounce := func(h string) error {
1203	bounced = append(bounced, h)
1204	return nil
1205	}
1206	if err := SyncExec(ctx, reg, s, pk, dd, bounce, "", nil); err != nil {
1207	t.Fatalf("SyncExec: %v", err)
1208	}
1209	if !slices.Contains(bounced, "inst.example.com") {
1210	t.Fatalf("bounced = %v, want the socket-exec route bounced (unaffected by the addr route sharing the pass)", bounced)
1211	}
1212	if slices.Contains(bounced, "demo.go.tito.io") {
1213	t.Fatalf("bounced = %v, the opted-in addr route must never be bounced", bounced)
1214	}
1215	if len(bounced) != 1 {
1216	t.Fatalf("bounced = %v, want exactly one bounce", bounced)
1217	}
1218	}
1219
1220	// TestSyncExecOptedInAddrRouteHonorsBoundEnvironment: environments-
1221	// awareness applies identically to the addr shape — a route bound to a
1222	// named environment resolves via LoadMerged (default overlaid by the
1223	// environment, environment winning per key), exactly like a socket-exec
1224	// route bound the same way.
1225	func TestSyncExecOptedInAddrRouteHonorsBoundEnvironment(t *testing.T) {
1226	ctx, s, reg, dd, priv, _ := execKit(t)
1227	pk, err := ParsePrivateKey(priv, "test")
1228	if err != nil {
1229	t.Fatalf("ParsePrivateKey: %v", err)
1230	}
1231	if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{
1232	"LOG_LEVEL": "info",
1233	"BASE_ONLY": "1",
1234	}); err != nil {
1235	t.Fatalf("SetEnv: %v", err)
1236	}
1237	if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil {
1238	t.Fatalf("SetExecDelivery: %v", err)
1239	}
1240	if err := SaveEnvironment(ctx, s, "bab", "titogo", "demo", map[string]Var{
1241	"LOG_LEVEL": {Kind: "env", Value: "debug"},
1242	"DEMO_ONLY": {Kind: "env", Value: "yes"},
1243	}, nil); err != nil {
1244	t.Fatalf("SaveEnvironment: %v", err)
1245	}
1246	if err := reg.Upsert(registry.Route{
1247	Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab",
1248	Status: "ready", Addr: "127.0.0.1:8444", Environment: "demo",
1249	}); err != nil {
1250	t.Fatalf("Upsert: %v", err)
1251	}
1252	if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil {
1253	t.Fatalf("SyncExec: %v", err)
1254	}
1255	got, err := os.ReadFile(filepath.Join(dd, "demo.go.tito.io.config.env"))
1256	if err != nil {
1257	t.Fatalf("read config env: %v", err)
1258	}
1259	want := execAddrConfigHeader + "BASE_ONLY=1\nDEMO_ONLY=yes\nLOG_LEVEL=debug\n"
1260	if string(got) != want {
1261	t.Fatalf("config env = %q, want %q", got, want)
1262	}
1263	}
1264
1265	// TestSyncExecOptedInAddrAppWithNoVarsConvergesToEmptyFile: opting in
1266	// (SetExecDelivery alone, no SetEnv) creates the default bundle with an
1267	// empty Vars map — found=true, so this is the "configured, then nothing
1268	// set" state, not "absent". The route must get an EMPTY file, not have
1269	// none written at all and not have the mass-removal guard's absent
1270	// bucket confused by it. This is also the reason
1271	// "opted-in AND the group resolves absent" is unreachable by
1272	// construction: checkOptedIn requires the default bundle to exist
1273	// (found=true) before an addr route is ever added to resolved, and a
1274	// present-but-empty bundle is exactly resolution{m: <empty manifest>},
1275	// never resolution{} (nil, absent).
1276	func TestSyncExecOptedInAddrAppWithNoVarsConvergesToEmptyFile(t *testing.T) {
1277	ctx, s, reg, dd, priv, _ := execKit(t)
1278	pk, err := ParsePrivateKey(priv, "test")
1279	if err != nil {
1280	t.Fatalf("ParsePrivateKey: %v", err)
1281	}
1282	if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil {
1283	t.Fatalf("SetExecDelivery: %v", err)
1284	}
1285	if err := reg.Upsert(registry.Route{
1286	Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab",
1287	Status: "ready", Addr: "127.0.0.1:8444",
1288	}); err != nil {
1289	t.Fatalf("Upsert: %v", err)
1290	}
1291	if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil {
1292	t.Fatalf("SyncExec: %v", err)
1293	}
1294	got, err := os.ReadFile(filepath.Join(dd, "demo.go.tito.io.config.env"))
1295	if err != nil {
1296	t.Fatalf("opted-in-but-unconfigured app's route got no file at all (want the header alone, not none): %v", err)
1297	}
1298	// No vars, but never a truly empty file: the header (issue #18) is
1299	// unconditional for this shape, whether or not the app has any vars
1300	// set yet.
1301	if string(got) != execAddrConfigHeader {
1302	t.Fatalf("config env = %q, want just the header (%q)", got, execAddrConfigHeader)
1303	}
1304	}
1305
1306	// TestSyncExecMassRemovalGuardUnaffectedByOptedInAddrAppWithContent
1307	// exercises the mass-removal guard sharing its EXACT code path with the
1308	// addr shape (no addr-specific carve-out): an opted-in addr app WITH
1309	// content sits in the pass alongside a socket app that is genuinely
1310	// never configured. Only the socket app's group is absent — the guard
1311	// must not fire (only one of two eligible groups is absent, same as
1312	// TestSyncExecMassRemovalGuardDoesNotBlockNormalConvergence pins for two
1313	// socket apps), so the socket app's stale file is removed normally while
1314	// the addr app's file converges with its content, unaffected by sharing
1315	// the pass.
1316	func TestSyncExecMassRemovalGuardUnaffectedByOptedInAddrAppWithContent(t *testing.T) {
1317	ctx, s, reg, dd, priv, _ := execKit(t)
1318	pk, err := ParsePrivateKey(priv, "test")
1319	if err != nil {
1320	t.Fatalf("ParsePrivateKey: %v", err)
1321	}
1322	if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"A": "1"}); err != nil {
1323	t.Fatalf("SetEnv: %v", err)
1324	}
1325	if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil {
1326	t.Fatalf("SetExecDelivery: %v", err)
1327	}
1328	// eleven is never configured at all: genuinely absent.
1329	routes := []registry.Route{
1330	{Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Addr: "127.0.0.1:8444"},
1331	{Host: "b.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/b.sock"},
1332	}
1333	for _, rt := range routes {
1334	if err := reg.Upsert(rt); err != nil {
1335	t.Fatalf("Upsert %s: %v", rt.Host, err)
1336	}
1337	}
1338	pathEleven := filepath.Join(dd, "b.example.com.config.env")
1339	if err := os.WriteFile(pathEleven, []byte("OLD=2\n"), 0o600); err != nil {
1340	t.Fatalf("stage eleven: %v", err)
1341	}
1342	var bounced []string
1343	if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error {
1344	bounced = append(bounced, h)
1345	return nil
1346	}, "", nil); err != nil {
1347	t.Fatalf("SyncExec: %v", err)
1348	}
1349	got, err := os.ReadFile(filepath.Join(dd, "demo.go.tito.io.config.env"))
1350	if err != nil {
1351	t.Fatalf("titogo's config file wasn't written: %v", err)
1352	}
1353	want := execAddrConfigHeader + "A=1\n"
1354	if string(got) != want {
1355	t.Fatalf("titogo config env = %q, want %q", got, want)
1356	}
1357	if _, err := os.Stat(pathEleven); !os.IsNotExist(err) {
1358	t.Fatalf("eleven's config file survived its manifest's genuine absence, stat err = %v (guard incorrectly disarmed convergence, or incorrectly held it back)", err)
1359	}
1360	// eleven's file is a removal (bounceLogged fires); titogo's is a
1361	// fresh write, socket-shape rules would bounce it too, but titogo is
1362	// the addr shape — it must never appear here.
1363	if !slices.Contains(bounced, "b.example.com") {
1364	t.Fatalf("bounced = %v, want eleven's removal bounce", bounced)
1365	}
1366	if slices.Contains(bounced, "demo.go.tito.io") {
1367	t.Fatalf("bounced = %v, the opted-in addr route must never be bounced", bounced)
1368	}
1369	if len(bounced) != 1 {
1370	t.Fatalf("bounced = %v, want exactly one bounce", bounced)
1371	}
1372	}
1373
1374	// TestSyncExecOptOutLeavesStaleFileButWarnsEveryPass is IMPORTANT 2 from
1375	// the 2026-08-03 review's ruling: turning exec_delivery off does NOT
1376	// delete an already-materialized <host>.config.env (deleting it would
1377	// change what the app's own supervisor layers in on its next respawn, at
1378	// a moment carlos does not control and cannot announce — worse than a
1379	// stale file, and consistent with every other ineligibility transition in
1380	// this file). But the file holds DECRYPTED secrets and nothing will ever
1381	// update it again once opted out, so the silence itself was the bug:
1382	// every pass must name the stale file and the manual remedy.
1383	func TestSyncExecOptOutLeavesStaleFileButWarnsEveryPass(t *testing.T) {
1384	ctx, s, reg, dd, priv, _ := execKit(t)
1385	pk, err := ParsePrivateKey(priv, "test")
1386	if err != nil {
1387	t.Fatalf("ParsePrivateKey: %v", err)
1388	}
1389	if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"TITOGO_DEMO_BANNER": "1"}); err != nil {
1390	t.Fatalf("SetEnv: %v", err)
1391	}
1392	if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil {
1393	t.Fatalf("SetExecDelivery: %v", err)
1394	}
1395	if err := reg.Upsert(registry.Route{
1396	Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab",
1397	Status: "ready", Addr: "127.0.0.1:8444",
1398	}); err != nil {
1399	t.Fatalf("Upsert: %v", err)
1400	}
1401	if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil {
1402	t.Fatalf("SyncExec (opted in): %v", err)
1403	}
1404	path := filepath.Join(dd, "demo.go.tito.io.config.env")
1405	staged, err := os.ReadFile(path)
1406	if err != nil {
1407	t.Fatalf("read staged config env: %v", err)
1408	}
1409
1410	if err := SetExecDelivery(ctx, s, "bab", "titogo", ""); err != nil {
1411	t.Fatalf("SetExecDelivery (off): %v", err)
1412	}
1413	var bounced []string
1414	bounce := func(h string) error { bounced = append(bounced, h); return nil }
1415	var syncErr error
1416	out := captureStdout(t, func() {
1417	syncErr = SyncExec(ctx, reg, s, pk, dd, bounce, "", nil)
1418	})
1419	if syncErr != nil {
1420	t.Fatalf("SyncExec (opted out): %v", syncErr)
1421	}
1422	got, err := os.ReadFile(path)
1423	if err != nil {
1424	t.Fatalf("stale config env was deleted on opt-out, want it left in place: %v", err)
1425	}
1426	if string(got) != string(staged) {
1427	t.Fatalf("stale config env changed on opt-out: got %q, want the untouched original %q", got, staged)
1428	}
1429	if len(bounced) != 0 {
1430	t.Fatalf("bounced = %v, want none", bounced)
1431	}
1432	if !strings.Contains(out, "demo.go.tito.io") || !strings.Contains(out, path) {
1433	t.Fatalf("stdout = %q, want a line naming the host and the stale file's path", out)
1434	}
1435	if !strings.Contains(strings.ToLower(out), "secret") {
1436	t.Fatalf("stdout = %q, want the warning to say the file holds secrets", out)
1437	}
1438
1439	// The warning is PER PASS, not one-shot: a second pass with nothing
1440	// changed must repeat it, since nothing about the risk (stale
1441	// plaintext secrets, forever) has gone away.
1442	out2 := captureStdout(t, func() {
1443	syncErr = SyncExec(ctx, reg, s, pk, dd, bounce, "", nil)
1444	})
1445	if syncErr != nil {
1446	t.Fatalf("SyncExec (opted out, second pass): %v", syncErr)
1447	}
1448	if !strings.Contains(out2, "demo.go.tito.io") {
1449	t.Fatalf("second pass stdout = %q, want the warning repeated", out2)
1450	}
1451	if len(bounced) != 0 {
1452	t.Fatalf("bounced = %v after the second pass, want none", bounced)
1453	}
1454	}
1455
1456	// TestSyncExecNoOptOutWarningWhenNeverOptedIn: the stale-file warning
1457	// fires only when a file actually exists — an app that was simply NEVER
1458	// opted in must stay exactly as quiet as it was before issue #18 existed,
1459	// not gain a new per-pass line for a file it never had.
1460	func TestSyncExecNoOptOutWarningWhenNeverOptedIn(t *testing.T) {
1461	ctx, s, reg, dd, priv, _ := execKit(t)
1462	pk, err := ParsePrivateKey(priv, "test")
1463	if err != nil {
1464	t.Fatalf("ParsePrivateKey: %v", err)
1465	}
1466	if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"K": "1"}); err != nil {
1467	t.Fatalf("SetEnv: %v", err)
1468	}
1469	// No SetExecDelivery call at all.
1470	if err := reg.Upsert(registry.Route{
1471	Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab",
1472	Status: "ready", Addr: "127.0.0.1:8444",
1473	}); err != nil {
1474	t.Fatalf("Upsert: %v", err)
1475	}
1476	var syncErr error
1477	out := captureStdout(t, func() {
1478	syncErr = SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil)
1479	})
1480	if syncErr != nil {
1481	t.Fatalf("SyncExec: %v", syncErr)
1482	}
1483	if strings.Contains(out, "demo.go.tito.io") {
1484	t.Fatalf("stdout = %q, want no mention of a route that never had a file", out)
1485	}
1486	}
1487
1488	// TestSyncExecAddrFileCarriesHeaderSocketFileDoesNot is IMPORTANT 3 from
1489	// the 2026-08-03 review: the issue #18 design comment's contract line —
1490	// "the file's header comment says so" — pinned directly. The addr shape's
1491	// file must carry it; the socket shape's file (same function, same pass)
1492	// must NOT, so an existing fleet's socket-exec files never churn or bounce
1493	// on upgrade (see execAddrConfigHeader's doc comment for the full reason).
1494	func TestSyncExecAddrFileCarriesHeaderSocketFileDoesNot(t *testing.T) {
1495	ctx, s, reg, dd, priv, _ := execKit(t)
1496	pk, err := ParsePrivateKey(priv, "test")
1497	if err != nil {
1498	t.Fatalf("ParsePrivateKey: %v", err)
1499	}
1500	if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"A": "1"}); err != nil {
1501	t.Fatalf("SetEnv: %v", err)
1502	}
1503	if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil {
1504	t.Fatalf("SetExecDelivery: %v", err)
1505	}
1506	if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"B": "1"}); err != nil {
1507	t.Fatalf("SetEnv: %v", err)
1508	}
1509	routes := []registry.Route{
1510	{Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab", Status: "ready", Addr: "127.0.0.1:8444"},
1511	{Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/inst.sock"},
1512	}
1513	for _, rt := range routes {
1514	if err := reg.Upsert(rt); err != nil {
1515	t.Fatalf("Upsert %s: %v", rt.Host, err)
1516	}
1517	}
1518	if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil {
1519	t.Fatalf("SyncExec: %v", err)
1520	}
1521	addrGot, err := os.ReadFile(filepath.Join(dd, "demo.go.tito.io.config.env"))
1522	if err != nil {
1523	t.Fatalf("read addr config env: %v", err)
1524	}
1525	if !strings.HasPrefix(string(addrGot), "#") {
1526	t.Fatalf("addr config env = %q, want it to start with a header comment", addrGot)
1527	}
1528	if !strings.Contains(string(addrGot), "carlos") {
1529	t.Fatalf("addr config env = %q, want the header to say carlos wrote it", addrGot)
1530	}
1531	if !strings.Contains(strings.ToLower(string(addrGot)), "never") || !strings.Contains(strings.ToLower(string(addrGot)), "restart") {
1532	t.Fatalf("addr config env = %q, want the header to say carlos never restarts this process", addrGot)
1533	}
1534	if !strings.Contains(strings.ToLower(string(addrGot)), "supervisor") {
1535	t.Fatalf("addr config env = %q, want the header to say the app's supervisor owns restarts", addrGot)
1536	}
1537	socketGot, err := os.ReadFile(filepath.Join(dd, "inst.example.com.config.env"))
1538	if err != nil {
1539	t.Fatalf("read socket config env: %v", err)
1540	}
1541	if want := "B=1\n"; string(socketGot) != want {
1542	t.Fatalf("socket config env = %q, want exactly %q — no header on this shape, ever", socketGot, want)
1543	}
1544	}
1545
1546	// TestSyncExecConvergesToEmptyFileWhenAllVarsUnset covers the lifecycle
1547	// F8 flagged as missing: a manifest object that still EXISTS but has had
1548	// every var unset must converge its route to an EMPTY file, not have the
1549	// file removed — removal is reserved for a manifest object that is
1550	// entirely ABSENT from the bucket (see syncExecHost's m == nil branch).
1551	// Blurring those two states would make "unset every var" behave like
1552	// deleting the manifest, which it structurally is not.
1553	func TestSyncExecConvergesToEmptyFileWhenAllVarsUnset(t *testing.T) {
1554	ctx, s, reg, dd, priv, _ := execKit(t)
1555	pk, err := ParsePrivateKey(priv, "test")
1556	if err != nil {
1557	t.Fatalf("ParsePrivateKey: %v", err)
1558	}
1559	if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil {
1560	t.Fatalf("SetEnv: %v", err)
1561	}
1562	if err := reg.Upsert(registry.Route{
1563	Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab",
1564	Status: "ready", Socket: "/run/carlos/inst.sock",
1565	}); err != nil {
1566	t.Fatalf("Upsert: %v", err)
1567	}
1568	path := filepath.Join(dd, "inst.example.com.config.env")
1569
1570	var bounced []string
1571	bounce := func(h string) error { bounced = append(bounced, h); return nil }
1572	if err := SyncExec(ctx, reg, s, pk, dd, bounce, "", nil); err != nil {
1573	t.Fatalf("SyncExec (set): %v", err)
1574	}
1575	got, err := os.ReadFile(path)
1576	if err != nil {
1577	t.Fatalf("read config env: %v", err)
1578	}
1579	if string(got) != "LOG_LEVEL=debug\n" {
1580	t.Fatalf("config env = %q, want LOG_LEVEL=debug", got)
1581	}
1582
1583	// Unset every var — the manifest object itself still exists (Unset
1584	// rewrites it with an empty Vars map; it does not delete the object).
1585	if _, err := Unset(ctx, s, "bab", "eleven", []string{"LOG_LEVEL"}); err != nil {
1586	t.Fatalf("Unset (all vars): %v", err)
1587	}
1588	bounced = nil
1589	if err := SyncExec(ctx, reg, s, pk, dd, bounce, "", nil); err != nil {
1590	t.Fatalf("SyncExec (unset): %v", err)
1591	}
1592	got, err = os.ReadFile(path)
1593	if err != nil {
1594	t.Fatalf("config file removed instead of converged to empty: %v", err)
1595	}
1596	if string(got) != "" {
1597	t.Fatalf("config env = %q, want empty", got)
1598	}
1599	if len(bounced) != 1 || bounced[0] != "inst.example.com" {
1600	t.Fatalf("bounced = %v, want exactly one bounce for the change to empty", bounced)
1601	}
1602	}
1603
1604	// TestReapSparesLiveRoutesInstanceEnvWithConfigSuffix covers the PR #16
1605	// deferred namespace-collision hazard: host "x.config"'s hand-staged
1606	// instance env is named "x.config.env" (<host>.env for host "x.config"),
1607	// which is byte-identical to what host "x"'s SYNC-OWNED file would be
1608	// named. With no route "x" at all, the naive reap trims configEnvSuffix off
1609	// "x.config.env", sees host "x", finds it not live, and deletes what is
1610	// actually route "x.config"'s own instance env — another owner's file.
1611	func TestReapSparesLiveRoutesInstanceEnvWithConfigSuffix(t *testing.T) {
1612	ctx, s, reg, dd, priv, _ := execKit(t)
1613	pk, _ := ParsePrivateKey(priv, "test")
1614	path := filepath.Join(dd, "x.config.env")
1615	sentinel := []byte("SENTINEL=owned-by-x-config\n")
1616	if err := os.WriteFile(path, sentinel, 0o600); err != nil {
1617	t.Fatalf("stage: %v", err)
1618	}
1619	// A live route named "x.config" and, critically, no route "x" at all.
1620	if err := reg.Upsert(registry.Route{
1621	Host: "x.config", Kind: "instance", App: "eleven", Account: "bab", Status: "ready",
1622	}); err != nil {
1623	t.Fatalf("Upsert: %v", err)
1624	}
1625	out := captureStdout(t, func() {
1626	if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil {
1627	t.Fatalf("SyncExec: %v", err)
1628	}
1629	})
1630	// Pin the guard's own text, not just the file's survival — a passing
1631	// file-survival check alone wouldn't distinguish "the guard fired" from
1632	// "reap never ran at all".
1633	if !strings.Contains(out, "not reaped") || !strings.Contains(out, "claims this name as its instance env") {
1634	t.Fatalf("stdout = %q, want the guard's own refusal text", out)
1635	}
1636	got, err := os.ReadFile(path)
1637	if err != nil {
1638	t.Fatalf("x.config's instance env was reaped: %v", err)
1639	}
1640	if !bytes.Equal(got, sentinel) {
1641	t.Fatalf("x.config's instance env bytes changed: got %q, want %q", got, sentinel)
1642	}
1643	}
1644
1645	// TestSyncExecRefusesConfigNamespaceCollision covers the other half of the
1646	// same hazard: route "x" IS live (an ordinary socket-exec route) alongside
1647	// route "x.config". Converging route x would write/rename into
1648	// "x.config.env" — route x.config's hand-staged instance env, not route x's
1649	// own namespace. Route x must be refused, loudly, and route x.config's OWN
1650	// config file ("x.config.config.env") must still converge normally: the
1651	// guard protects one file without collaterally blocking the unrelated
1652	// route that happens to produce the colliding name.
1653	func TestSyncExecRefusesConfigNamespaceCollision(t *testing.T) {
1654	ctx, s, reg, dd, priv, _ := execKit(t)
1655	pk, err := ParsePrivateKey(priv, "test")
1656	if err != nil {
1657	t.Fatalf("ParsePrivateKey: %v", err)
1658	}
1659	if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"X": "1"}); err != nil {
1660	t.Fatalf("SetEnv (x's app): %v", err)
1661	}
1662	if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"Y": "2"}); err != nil {
1663	t.Fatalf("SetEnv (x.config's app): %v", err)
1664	}
1665	if err := reg.Upsert(registry.Route{
1666	Host: "x", Kind: "instance", App: "eleven", Account: "bab", Status: "ready",
1667	Socket: "/run/carlos/x.sock",
1668	}); err != nil {
1669	t.Fatalf("Upsert x: %v", err)
1670	}
1671	if err := reg.Upsert(registry.Route{
1672	Host: "x.config", Kind: "instance", App: "titogo", Account: "bab", Status: "ready",
1673	Socket: "/run/carlos/xc.sock",
1674	}); err != nil {
1675	t.Fatalf("Upsert x.config: %v", err)
1676	}
1677	// x.config's hand-staged instance env, pre-staged with sentinel bytes:
1678	// its name ("x.config.env") is exactly what route x's own sync-owned
1679	// config file would be named.
1680	collidingPath := filepath.Join(dd, "x.config.env")
1681	sentinel := []byte("SENTINEL=owned-by-x-config\n")
1682	if err := os.WriteFile(collidingPath, sentinel, 0o600); err != nil {
1683	t.Fatalf("stage: %v", err)
1684	}
1685	var out string
1686	var bounced []string
1687	bounce := func(h string) error { bounced = append(bounced, h); return nil }
1688	out = captureStdout(t, func() {
1689	if err := SyncExec(ctx, reg, s, pk, dd, bounce, "", nil); err != nil {
1690	t.Fatalf("SyncExec: %v", err)
1691	}
1692	})
1693	got, err := os.ReadFile(collidingPath)
1694	if err != nil {
1695	t.Fatalf("read colliding file: %v", err)
1696	}
1697	if !bytes.Equal(got, sentinel) {
1698	t.Fatalf("x.config's instance env was overwritten by route x's convergence: got %q, want %q", got, sentinel)
1699	}
1700	if slices.Contains(bounced, "x") {
1701	t.Fatalf("bounced = %v, want route x never bounced (it was refused, not converged)", bounced)
1702	}
1703	// "x.config" alone is a vacuous assertion here — route x.config's own
1704	// convergence prints " x.config: config updated" regardless of whether
1705	// the guard fired at all. Pin the guard's OWN distinctive text instead.
1706	if !strings.Contains(out, "not converging") || !strings.Contains(out, "refusing to write into another owner's namespace") {
1707	t.Fatalf("stdout = %q, want the guard's own refusal text", out)
1708	}
1709	// Route x.config's OWN config file is unrelated to the guard and must
1710	// converge exactly as it would with no collision in play.
1711	xConfigOwn, err := os.ReadFile(filepath.Join(dd, "x.config.config.env"))
1712	if err != nil {
1713	t.Fatalf("x.config's own config file did not converge: %v", err)
1714	}
1715	if string(xConfigOwn) != "Y=2\n" {
1716	t.Fatalf("x.config's own config file = %q, want Y=2", xConfigOwn)
1717	}
1718	}
1719
1720	// TestSyncExecFetchFailureLeavesOtherGroupsConverging is the M4
1721	// cross-group half TestSyncExecOptedInAddrRouteFetchFailureLogsAndLeavesFileAsIs
1722	// doesn't cover (that test has exactly one group in the pass): one app's
1723	// manifest fetch fails while a DIFFERENT app's group, in the SAME pass,
1724	// converges normally. A fetch failure must stay scoped to its own group —
1725	// it must neither remove/leave-stale the failing group's file (error is not
1726	// absence) NOR abort the sweep for every other group on the box.
1727	func TestSyncExecFetchFailureLeavesOtherGroupsConverging(t *testing.T) {
1728	ctx, s, reg, dd, priv, _ := execKit(t)
1729	pk, err := ParsePrivateKey(priv, "test")
1730	if err != nil {
1731	t.Fatalf("ParsePrivateKey: %v", err)
1732	}
1733	if err := SetEnv(ctx, s, "bab", "appa", map[string]string{"A": "1"}); err != nil {
1734	t.Fatalf("SetEnv appa: %v", err)
1735	}
1736	if err := SetEnv(ctx, s, "bab", "appb", map[string]string{"B": "2"}); err != nil {
1737	t.Fatalf("SetEnv appb: %v", err)
1738	}
1739	if err := reg.Upsert(registry.Route{
1740	Host: "hosta", Kind: "instance", App: "appa", Account: "bab", Status: "ready",
1741	Socket: "/run/carlos/a.sock",
1742	}); err != nil {
1743	t.Fatalf("Upsert hosta: %v", err)
1744	}
1745	if err := reg.Upsert(registry.Route{
1746	Host: "hostb", Kind: "instance", App: "appb", Account: "bab", Status: "ready",
1747	Socket: "/run/carlos/b.sock",
1748	}); err != nil {
1749	t.Fatalf("Upsert hostb: %v", err)
1750	}
1751	stalePath := filepath.Join(dd, "hosta.config.env")
1752	stale := []byte("STALE=1\n")
1753	if err := os.WriteFile(stalePath, stale, 0o600); err != nil {
1754	t.Fatalf("stage: %v", err)
1755	}
1756	blipErr := errors.New("s3: transient 503 SlowDown")
1757	failing := &failingKeyStore{Store: s, failKey: Key("bab", "appa"), err: blipErr}
1758	var bounced []string
1759	bounce := func(h string) error { bounced = append(bounced, h); return nil }
1760	if err := SyncExec(ctx, reg, failing, pk, dd, bounce, "", nil); err != nil {
1761	t.Fatalf("SyncExec: %v", err)
1762	}
1763	got, err := os.ReadFile(stalePath)
1764	if err != nil {
1765	t.Fatalf("read hosta config: %v", err)
1766	}
1767	if !bytes.Equal(got, stale) {
1768	t.Fatalf("hosta's config changed despite its own group's fetch failure: got %q, want %q", got, stale)
1769	}
1770	gotB, err := os.ReadFile(filepath.Join(dd, "hostb.config.env"))
1771	if err != nil {
1772	t.Fatalf("hostb's config didn't converge in the same pass: %v", err)
1773	}
1774	if string(gotB) != "B=2\n" {
1775	t.Fatalf("hostb config = %q, want B=2", gotB)
1776	}
1777	if !slices.Contains(bounced, "hostb") || slices.Contains(bounced, "hosta") {
1778	t.Fatalf("bounced = %v, want exactly [hostb] — appa's route left untouched, appb's route converged", bounced)
1779	}
1780	}
1781
1782	// TestExecConfigApps pins the set the agent's fast generation loop polls:
1783	// exactly the apps SyncExec could deliver config to, by route SHAPE — and
1784	// nothing else. The opt-in half of the TCP-addr case is deliberately NOT
1785	// consulted here (that would cost a bucket read per app, the very thing the
1786	// fast loop exists to avoid), so an un-opted addr route is in this list;
1787	// see ExecConfigApps' doc comment.
1788	func TestExecConfigApps(t *testing.T) {
1789	routes := []registry.Route{
1790	// In: the socket-exec shape, unconditionally syncable.
1791	{Host: "sock.example.com", Kind: "instance", App: "eleven", Account: "bab",
1792	Status: "ready", Socket: "/run/carlos/sock.sock"},
1793	// In: the TCP-addr shape (issue #18), opt-in unknown and unread.
1794	{Host: "addr.example.com", Kind: "instance", App: "titogo", Account: "zed",
1795	Status: "ready", Addr: "127.0.0.1:9000"},
1796	// In: the HA spelling is the same externally managed TCP shape.
1797	{Host: "multi-addr.example.com", Kind: "instance", App: "woodstar", Account: "zed",
1798	Status: "ready", Addrs: "127.0.0.1:8451"},
1799	// In, and deduped with sock.example.com: one app, two routes, one
1800	// generation object between them.
1801	{Host: "sock2.example.com", Kind: "instance", App: "eleven", Account: "bab",
1802	Status: "", Socket: "/run/carlos/sock2.sock"},
1803	// In under release.OpsAccount: a route whose Account is unusable as
1804	// a bucket segment reads as the ops account everywhere else, and
1805	// must here too or the loop watches a key nothing writes.
1806	{Host: "legacy.example.com", Kind: "instance", App: "kass", Account: "",
1807	Status: "ready", Socket: "/run/carlos/legacy.sock"},
1808	// Out: unit-backed routes belong to the root adopt path.
1809	{Host: "unit.example.com", Kind: "instance", App: "unitapp", Account: "bab",
1810	Status: "ready", Socket: "/run/carlos/unit.sock", Backing: registry.BackingUnit},
1811	// Out: platform routes are not app config.
1812	{Host: "plat.example.com", Kind: "instance", App: "platapp", Account: "bab",
1813	Status: "ready", Socket: "/run/carlos/plat.sock", Platform: true},
1814	// Out: a draining route is on its way out; nothing to converge.
1815	{Host: "drain.example.com", Kind: "instance", App: "drainapp", Account: "bab",
1816	Status: "draining", Socket: "/run/carlos/drain.sock"},
1817	// Out: no app means no config manifest to key on.
1818	{Host: "noapp.example.com", Kind: "instance", App: "", Account: "bab",
1819	Status: "ready", Socket: "/run/carlos/noapp.sock"},
1820	// Out: neither socket nor addr — no upstream either predicate matches.
1821	{Host: "bare.example.com", Kind: "instance", App: "bareapp", Account: "bab",
1822	Status: "ready"},
1823	// Out: not an instance route at all.
1824	{Host: "proxy.example.com", Kind: "proxy", App: "proxyapp", Account: "bab",
1825	Status: "ready", Socket: "/run/carlos/proxy.sock"},
1826	}
1827	got := ExecConfigApps(routes)
1828	want := [][2]string{
1829	{"bab", "eleven"},
1830	{"bab", "kass"},
1831	{"zed", "titogo"},
1832	{"zed", "woodstar"},
1833	}
1834	if !slices.Equal(got, want) {
1835	t.Fatalf("ExecConfigApps = %v, want %v (distinct, sorted by account then app)", got, want)
1836	}
1837	}
1838
1839	func TestExecConfigAppsNoRoutes(t *testing.T) {
1840	if got := ExecConfigApps(nil); len(got) != 0 {
1841	t.Fatalf("ExecConfigApps(nil) = %v, want empty", got)
1842	}
1843	}
1844
1845	// TestUnitConfigApps is TestExecConfigApps' mirror for the other half of the
1846	// fast loop's poll set: the apps whose config only the ROOT adopt pass can
1847	// deliver. Same fixture shapes, opposite inclusion on Backing — and the
1848	// both-shapes app below is the case that matters most, because an app can
1849	// hold a socket-exec route and a unit route at once and must appear in BOTH
1850	// functions' outputs (the loop then syncs exec in-process AND asks for a
1851	// root pass off one generation read).
1852	func TestUnitConfigApps(t *testing.T) {
1853	routes := []registry.Route{
1854	// In: the carlos-app@ unit shape the root adopt pass renders.
1855	{Host: "unit.example.com", Kind: "instance", App: "unitapp", Account: "bab",
1856	Status: "ready", Socket: "/run/carlos/unit.sock", Backing: registry.BackingUnit},
1857	// In, and deduped: one app, two unit routes, one generation object.
1858	{Host: "unit2.example.com", Kind: "instance", App: "unitapp", Account: "bab",
1859	Status: "", Socket: "/run/carlos/unit2.sock", Backing: registry.BackingUnit},
1860	// In under release.OpsAccount, exactly as ExecConfigApps resolves it.
1861	{Host: "legacy.example.com", Kind: "instance", App: "kass", Account: "",
1862	Status: "ready", Socket: "/run/carlos/legacy.sock", Backing: registry.BackingUnit},
1863	// In BOTH sets: this app's unit route is here, its exec route is in
1864	// ExecConfigApps (asserted below).
1865	{Host: "mixed-unit.example.com", Kind: "instance", App: "mixed", Account: "zed",
1866	Status: "ready", Socket: "/run/carlos/mixed-unit.sock", Backing: registry.BackingUnit},
1867	{Host: "mixed-exec.example.com", Kind: "instance", App: "mixed", Account: "zed",
1868	Status: "ready", Socket: "/run/carlos/mixed-exec.sock"},
1869	// Out: exec-backed only — the agent delivers this one itself.
1870	{Host: "sock.example.com", Kind: "instance", App: "eleven", Account: "bab",
1871	Status: "ready", Socket: "/run/carlos/sock.sock"},
1872	// Out: platform routes are not app config.
1873	{Host: "plat.example.com", Kind: "instance", App: "platapp", Account: "bab",
1874	Status: "ready", Socket: "/run/carlos/plat.sock", Platform: true,
1875	Backing: registry.BackingUnit},
1876	// Out: a draining route is on its way out; nothing to converge.
1877	{Host: "drain.example.com", Kind: "instance", App: "drainapp", Account: "bab",
1878	Status: "draining", Socket: "/run/carlos/drain.sock", Backing: registry.BackingUnit},
1879	// Out: no app means no config manifest to key on.
1880	{Host: "noapp.example.com", Kind: "instance", App: "", Account: "bab",
1881	Status: "ready", Socket: "/run/carlos/noapp.sock", Backing: registry.BackingUnit},
1882	// Out: not an instance route at all.
1883	{Host: "proxy.example.com", Kind: "proxy", App: "proxyapp", Account: "bab",
1884	Status: "ready", Socket: "/run/carlos/proxy.sock", Backing: registry.BackingUnit},
1885	}
1886	got := UnitConfigApps(routes)
1887	want := [][2]string{
1888	{"bab", "kass"},
1889	{"bab", "unitapp"},
1890	{"zed", "mixed"},
1891	}
1892	if !slices.Equal(got, want) {
1893	t.Fatalf("UnitConfigApps = %v, want %v (distinct, sorted by account then app)", got, want)
1894	}
1895	if gotExec := ExecConfigApps(routes); !slices.Contains(gotExec, [2]string{"zed", "mixed"}) {
1896	t.Fatalf("ExecConfigApps = %v, want it to also hold {zed mixed} — an app with both route shapes is in both sets", gotExec)
1897	}
1898	}
1899
1900	func TestUnitConfigAppsNoRoutes(t *testing.T) {
1901	if got := UnitConfigApps(nil); len(got) != 0 {
1902	t.Fatalf("UnitConfigApps(nil) = %v, want empty", got)
1903	}
1904	}
1905
1906	// --- issue #195: the opt-in resolves from the route's bound bundle ---
1907
1908	// TestSyncExecOptInResolvesPerEnvironment is issue #195's matrix: which
1909	// hosts get a <host>.config.env, for each combination of the app-level
1910	// flag and the bound bundle's own. Two TCP-addr exec routes of ONE app,
1911	// one bound to "relay" and one unbound — titogo's exact shape, where
1912	// turning the app-level flag on to deliver the relay's sealed Apple Wallet
1913	// material wrote a file for every router route too.
1914	func TestSyncExecOptInResolvesPerEnvironment(t *testing.T) {
1915	for _, tc := range []struct {
1916	name string
1917	appLevel string // "" (never set) | ExecDeliveryFile
1918	envLevel string // absent | silent | file | bogus
1919	wantRelayFile bool
1920	wantUnboundFile bool
1921	}{
1922	{"both off", "", "silent", false, false},
1923	{"app on, env silent — inherit on", ExecDeliveryFile, "silent", true, true},
1924	{"app off, env on — the narrow case #195 exists for", "", "file", true, false},
1925	{"app on, env on", ExecDeliveryFile, "file", true, true},
1926	{"app off, env bundle absent", "", "absent", false, false},
1927	// Bound to a bundle that does not exist: the opt-in inherits "on"
1928	// from the app default, but the route never reaches a write anyway
1929	// — LoadMerged refuses a bound-but-absent bundle, so the group
1930	// resolves to an ERROR and the file is left exactly as it is. The
1931	// unbound router is untouched by that and still gets its file.
1932	{"app on, env bundle absent — resolution errors before delivery", ExecDeliveryFile, "absent", false, true},
1933	{"app on, env states a value this binary doesn't know — that statement decides", ExecDeliveryFile, "bogus", false, true},
1934	} {
1935	t.Run(tc.name, func(t *testing.T) {
1936	ctx, s, reg, dd, priv, _ := execKit(t)
1937	pk, err := ParsePrivateKey(priv, "test")
1938	if err != nil {
1939	t.Fatalf("ParsePrivateKey: %v", err)
1940	}
1941	// The default bundle always exists and always has a var, so the
1942	// only thing under test is WHETHER a file is written.
1943	if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"SHARED": "1"}); err != nil {
1944	t.Fatalf("SetEnv: %v", err)
1945	}
1946	if tc.appLevel != "" {
1947	if err := SetExecDelivery(ctx, s, "bab", "titogo", tc.appLevel); err != nil {
1948	t.Fatalf("SetExecDelivery: %v", err)
1949	}
1950	}
1951	switch tc.envLevel {
1952	case "absent":
1953	// Nothing written: the route is bound to a bundle that does
1954	// not exist.
1955	case "bogus":
1956	// Written past the setter's validation on purpose, and
1957	// without SaveEnvironment first (a create, not an update):
1958	// this is what an OLDER binary sees when a NEWER one
1959	// invents a value, and it must not brick the pass.
1960	raw := []byte(`{"schema":1,"app":"titogo","vars":{"SEALED":{"kind":"env","value":"2"}},"exec_delivery":"grpc"}`)
1961	if _, err := s.Put(ctx, EnvironmentKey("bab", "titogo", "relay"), raw, ""); err != nil {
1962	t.Fatalf("Put: %v", err)
1963	}
1964	default:
1965	if err := SaveEnvironment(ctx, s, "bab", "titogo", "relay",
1966	map[string]Var{"SEALED": {Kind: "env", Value: "2"}}, nil); err != nil {
1967	t.Fatalf("SaveEnvironment: %v", err)
1968	}
1969	if tc.envLevel == "file" {
1970	if err := SetExecDeliveryAt(ctx, s, "bab", "titogo", "relay", ExecDeliveryFile); err != nil {
1971	t.Fatalf("SetExecDeliveryAt: %v", err)
1972	}
1973	}
1974	}
1975	for _, rt := range []registry.Route{
1976	{Host: "connect.go.tito.io", Kind: "instance", App: "titogo", Account: "bab",
1977	Status: "ready", Addr: "127.0.0.1:9001", Environment: "relay"},
1978	{Host: "router-a.go.tito.io", Kind: "instance", App: "titogo", Account: "bab",
1979	Status: "ready", Addr: "127.0.0.1:9002"},
1980	} {
1981	if err := reg.Upsert(rt); err != nil {
1982	t.Fatalf("Upsert %s: %v", rt.Host, err)
1983	}
1984	}
1985	if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil {
1986	t.Fatalf("SyncExec: %v", err)
1987	}
1988	assertConfigFile(t, dd, "connect.go.tito.io", tc.wantRelayFile)
1989	assertConfigFile(t, dd, "router-a.go.tito.io", tc.wantUnboundFile)
1990	})
1991	}
1992	}
1993
1994	// assertConfigFile asserts a route's <host>.config.env exists (or not).
1995	func assertConfigFile(t *testing.T, dir, host string, want bool) {
1996	t.Helper()
1997	_, err := os.Stat(filepath.Join(dir, host+configEnvSuffix))
1998	switch {
1999	case want && err != nil:
2000	t.Fatalf("%s: want a config file, got %v", host, err)
2001	case !want && err == nil:
2002	t.Fatalf("%s: want NO config file, one was written", host)
2003	}
2004	}
2005
2006	// TestSyncExecEnvScopedOptInDeliversTheMergedBundle: the flag decides
2007	// WHETHER a file is written, never what is in it. A route opted in through
2008	// its own bundle still gets the default bundle overlaid by that bundle —
2009	// the same merge LoadMerged has always produced.
2010	func TestSyncExecEnvScopedOptInDeliversTheMergedBundle(t *testing.T) {
2011	ctx, s, reg, dd, priv, _ := execKit(t)
2012	pk, err := ParsePrivateKey(priv, "test")
2013	if err != nil {
2014	t.Fatalf("ParsePrivateKey: %v", err)
2015	}
2016	if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"SHARED": "1", "OVERRIDDEN": "default"}); err != nil {
2017	t.Fatalf("SetEnv: %v", err)
2018	}
2019	if err := SaveEnvironment(ctx, s, "bab", "titogo", "relay", map[string]Var{
2020	"OVERRIDDEN": {Kind: "env", Value: "relay"},
2021	"WALLET": {Kind: "env", Value: "sealed"},
2022	}, nil); err != nil {
2023	t.Fatalf("SaveEnvironment: %v", err)
2024	}
2025	if err := SetExecDeliveryAt(ctx, s, "bab", "titogo", "relay", ExecDeliveryFile); err != nil {
2026	t.Fatalf("SetExecDeliveryAt: %v", err)
2027	}
2028	if err := reg.Upsert(registry.Route{
2029	Host: "connect.go.tito.io", Kind: "instance", App: "titogo", Account: "bab",
2030	Status: "ready", Addr: "127.0.0.1:9001", Environment: "relay",
2031	}); err != nil {
2032	t.Fatalf("Upsert: %v", err)
2033	}
2034	if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil {
2035	t.Fatalf("SyncExec: %v", err)
2036	}
2037	got, err := os.ReadFile(filepath.Join(dd, "connect.go.tito.io"+configEnvSuffix))
2038	if err != nil {
2039	t.Fatalf("read config file: %v", err)
2040	}
2041	want := execAddrConfigHeader + "OVERRIDDEN=relay\nSHARED=1\nWALLET=sealed\n"
2042	if string(got) != want {
2043	t.Fatalf("config env = %q, want %q (the merged bundle, unchanged by where the flag lives)", got, want)
2044	}
2045	// exec_delivery itself must never be rendered into the file: it is a
2046	// delivery flag, not a config var.
2047	if strings.Contains(string(got), "exec_delivery") || strings.Contains(string(got), "EXEC_DELIVERY") {
2048	t.Fatalf("the delivery flag leaked into the rendered file: %q", got)
2049	}
2050	}
2051
2052	// TestSyncExecEnvBundleFetchErrorLeavesFileAlone: an unreadable
2053	// environment bundle is NOT a determination that the route is un-opted.
2054	// The route's existing file must survive it byte-identical — the same
2055	// contract a default-bundle fetch failure already has, extended to the
2056	// second bundle the opt-in now consults.
2057	func TestSyncExecEnvBundleFetchErrorLeavesFileAlone(t *testing.T) {
2058	ctx, s, reg, dd, priv, _ := execKit(t)
2059	pk, err := ParsePrivateKey(priv, "test")
2060	if err != nil {
2061	t.Fatalf("ParsePrivateKey: %v", err)
2062	}
2063	if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"SHARED": "1"}); err != nil {
2064	t.Fatalf("SetEnv: %v", err)
2065	}
2066	if err := SaveEnvironment(ctx, s, "bab", "titogo", "relay",
2067	map[string]Var{"WALLET": {Kind: "env", Value: "sealed"}}, nil); err != nil {
2068	t.Fatalf("SaveEnvironment: %v", err)
2069	}
2070	if err := SetExecDeliveryAt(ctx, s, "bab", "titogo", "relay", ExecDeliveryFile); err != nil {
2071	t.Fatalf("SetExecDeliveryAt: %v", err)
2072	}
2073	if err := reg.Upsert(registry.Route{
2074	Host: "connect.go.tito.io", Kind: "instance", App: "titogo", Account: "bab",
2075	Status: "ready", Addr: "127.0.0.1:9001", Environment: "relay",
2076	}); err != nil {
2077	t.Fatalf("Upsert: %v", err)
2078	}
2079	if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "", nil); err != nil {
2080	t.Fatalf("SyncExec: %v", err)
2081	}
2082	path := filepath.Join(dd, "connect.go.tito.io"+configEnvSuffix)
2083	before, err := os.ReadFile(path)
2084	if err != nil {
2085	t.Fatalf("first pass wrote no file: %v", err)
2086	}
2087
2088	blip := &failingKeyStore{Store: s, failKey: EnvironmentKey("bab", "titogo", "relay"),
2089	err: errors.New("503 SlowDown")}
2090	out := captureStdout(t, func() {
2091	if err := SyncExec(ctx, reg, blip, pk, dd, func(string) error { return nil }, "", nil); err != nil {
2092	t.Fatalf("SyncExec (blip): %v", err)
2093	}
2094	})
2095	after, err := os.ReadFile(path)
2096	if err != nil {
2097	t.Fatalf("a bundle fetch blip removed a live route's config file: %v", err)
2098	}
2099	if !bytes.Equal(before, after) {
2100	t.Fatalf("config file changed on a fetch blip: %q -> %q", before, after)
2101	}
2102	if !strings.Contains(out, "unresolved") || !strings.Contains(out, "relay") {
2103	t.Fatalf("a fetch blip against the bound bundle must say so loudly and name the environment; got %q", out)
2104	}
2105	}
2106
2107	// ---- Task 11: sidecar config delivery -----------------------------------
2108
2109	// TestSidecarGetsItsConfigEnvFile: execRouteBase is the shape predicate both
2110	// exec eligibility cases narrow. A sidecar row (no Socket, no Addr) must
2111	// pass it — this is the direct replacement for the hand-staged
2112	// <host>-sidecar.env, and the whole reason the file path existed.
2113	func TestSidecarGetsItsConfigEnvFile(t *testing.T) {
2114	rt := registry.Route{Host: "bot.eleven.bdm.sidecar.internal", Kind: "sidecar",
2115	App: "eleven", Status: "ready"}
2116	if !execRouteBase(rt) {
2117	t.Fatal("a sidecar must receive config delivery — this is the direct replacement for the hand-staged <host>-sidecar.env, and the whole reason the file path existed")
2118	}
2119	}
2120
2121	// TestSidecarConfigStillExcludesPlatformAndUnitRoutes: widening the Kind
2122	// test must not loosen any of execRouteBase's other clauses.
2123	func TestSidecarConfigStillExcludesPlatformAndUnitRoutes(t *testing.T) {
2124	base := registry.Route{Host: "bot.x.sidecar.internal", Kind: "sidecar", App: "eleven", Status: "ready"}
2125	plat := base
2126	plat.Platform = true
2127	if execRouteBase(plat) {
2128	t.Fatal("a Platform alias is still excluded")
2129	}
2130	unit := base
2131	unit.Backing = registry.BackingUnit
2132	if execRouteBase(unit) {
2133	t.Fatal("unit-backed routes are still excluded")
2134	}
2135	}
2136
2137	// TestSyncExecWritesConfigFileForASidecarRoute is the end-to-end proof that
2138	// execRouteBase's widening actually reaches the file: a sidecar route
2139	// carries neither Socket nor Addr (sidecarRouteFor's contract — nothing
2140	// dials a sidecar), so execRouteBase alone is not sufficient to bring it
2141	// into SyncExec's resolution loop; inScope must recognize the shape too.
2142	// StartWorker (backend_exec.go's instanceEnv) reads exactly this file at
2143	// spawn, so a sidecar that never gets it here never gets its env at all.
2144	func TestSyncExecWritesConfigFileForASidecarRoute(t *testing.T) {
2145	ctx, s, reg, dd, priv, _ := execKit(t)
2146	pk, err := ParsePrivateKey(priv, "test")
2147	if err != nil {
2148	t.Fatalf("ParsePrivateKey: %v", err)
2149	}
2150	if err := SetEnv(ctx, s, "bdm", "eleven", map[string]string{"BOT_TOKEN": "shh"}); err != nil {
2151	t.Fatalf("SetEnv: %v", err)
2152	}
2153	if err := reg.Upsert(registry.Route{
2154	Host: "bot.eleven.bdm.sidecar.internal", Kind: "sidecar", App: "eleven", Account: "bdm",
2155	Status: "ready",
2156	}); err != nil {
2157	t.Fatalf("Upsert: %v", err)
2158	}
2159	if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "oncarlos.test", nil); err != nil {
2160	t.Fatalf("SyncExec: %v", err)
2161	}
2162	got, err := os.ReadFile(filepath.Join(dd, "bot.eleven.bdm.sidecar.internal.config.env"))
2163	if err != nil {
2164	t.Fatalf("sidecar never got a config.env file: %v", err)
2165	}
2166	if string(got) != "BOT_TOKEN=shh\n" {
2167	t.Fatalf("config env = %q, want BOT_TOKEN=shh", got)
2168	}
2169	}
2170
2171	// TestSyncExecSidecarRouteIsNeverBounced pins the safety decision behind
2172	// sidecarShape in inScope: Bounce alone cannot stand in for a sidecar's
2173	// wake (nothing dials a worker the way a request dials an instance — see
2174	// inScope's doc comment), so an automatic bounce on every ordinary
2175	// config-sync pass would risk stranding an always-on sidecar or leaving a
2176	// hibernating one silent until its schedule next comes due. The bounce
2177	// callback must never be invoked for a sidecar route, on either the
2178	// initial write or a changed-content pass — restarting one on purpose is
2179	// `carlos restart -app X` (host/restart.go), which pairs the stop with an
2180	// explicit wake.
2181	func TestSyncExecSidecarRouteIsNeverBounced(t *testing.T) {
2182	ctx, s, reg, dd, priv, _ := execKit(t)
2183	pk, err := ParsePrivateKey(priv, "test")
2184	if err != nil {
2185	t.Fatalf("ParsePrivateKey: %v", err)
2186	}
2187	if err := SetEnv(ctx, s, "bdm", "eleven", map[string]string{"BOT_TOKEN": "one"}); err != nil {
2188	t.Fatalf("SetEnv: %v", err)
2189	}
2190	if err := reg.Upsert(registry.Route{
2191	Host: "bot.eleven.bdm.sidecar.internal", Kind: "sidecar", App: "eleven", Account: "bdm",
2192	Status: "ready",
2193	}); err != nil {
2194	t.Fatalf("Upsert: %v", err)
2195	}
2196	var bounced []string
2197	bounce := func(h string) error { bounced = append(bounced, h); return nil }
2198	if err := SyncExec(ctx, reg, s, pk, dd, bounce, "oncarlos.test", nil); err != nil {
2199	t.Fatalf("SyncExec (initial write): %v", err)
2200	}
2201	if len(bounced) != 0 {
2202	t.Fatalf("bounced = %v, want none — Bounce alone cannot wake a worker back up", bounced)
2203	}
2204	if err := SetEnv(ctx, s, "bdm", "eleven", map[string]string{"BOT_TOKEN": "two"}); err != nil {
2205	t.Fatalf("SetEnv (change): %v", err)
2206	}
2207	if err := SyncExec(ctx, reg, s, pk, dd, bounce, "oncarlos.test", nil); err != nil {
2208	t.Fatalf("SyncExec (changed content): %v", err)
2209	}
2210	if len(bounced) != 0 {
2211	t.Fatalf("bounced = %v after a content change, want none", bounced)
2212	}
2213	got, err := os.ReadFile(filepath.Join(dd, "bot.eleven.bdm.sidecar.internal.config.env"))
2214	if err != nil {
2215	t.Fatalf("read config env: %v", err)
2216	}
2217	if string(got) != "BOT_TOKEN=two\n" {
2218	t.Fatalf("config env = %q, want BOT_TOKEN=two — the changed content, no header — the write path must still run", got)
2219	}
2220	}
2221
2222	// TestExecConfigAppsIncludesSidecars (fix round 1, coordinator ruling): a
2223	// sidecar route must be in scope for the fast generation-poll loop exactly
2224	// as an instance is. Left unwidened, a sidecar-only app's config delivered
2225	// by a direct-bucket write — never bumping control/serial, exactly what the
2226	// pinfra scripts do — would fall back to the hourly floor while an instance
2227	// next to it converges within the fast loop's cadence: a predicate widened
2228	// in SyncExec's inScope and not in this sibling, where the symptom is not
2229	// an error but work happening late.
2230	func TestExecConfigAppsIncludesSidecars(t *testing.T) {
2231	routes := []registry.Route{
2232	// In, and the whole point of this test: a SIDECAR-ONLY app — no
2233	// socket, no addr route at all — must still be polled. An earlier
2234	// version of this test paired the sidecar with an ordinary instance
2235	// of the same app, which made the assertion pass whether or not
2236	// ExecConfigApps actually looked at the sidecar route at all.
2237	{Host: "bot.eleven.bdm.sidecar.internal", Kind: "sidecar", App: "eleven", Account: "bdm", Status: "ready"},
2238	// In BOTH via one dedup: a second app with a sidecar AND a
2239	// socket-exec instance must still report as ONE (account, app) pair.
2240	{Host: "bot.mixed.bdm.sidecar.internal", Kind: "sidecar", App: "mixed", Account: "bdm", Status: "ready"},
2241	{Host: "inst.mixed.example.com", Kind: "instance", App: "mixed", Account: "bdm",
2242	Status: "ready", Socket: "/run/carlos/inst.sock"},
2243	// Out, same exclusions execRouteBase already gives an instance:
2244	// Platform and a non-ready Status.
2245	{Host: "plat.bot.sidecar.internal", Kind: "sidecar", App: "platapp", Account: "bdm",
2246	Status: "ready", Platform: true},
2247	{Host: "creating.bot.sidecar.internal", Kind: "sidecar", App: "creatingapp", Account: "bdm",
2248	Status: "creating"},
2249	}
2250	got := ExecConfigApps(routes)
2251	want := [][2]string{{"bdm", "eleven"}, {"bdm", "mixed"}}
2252	if !slices.Equal(got, want) {
2253	t.Fatalf("ExecConfigApps = %v, want %v — a sidecar-only app must be polled by the fast loop too, and a mixed-shape app must dedupe to one entry", got, want)
2254	}
2255	}
2256
2257	// The #344 guard, from the 2026-08-28 incident: the domains reconcile
2258	// mis-stamped a route's account, the config sync keyed the manifest under
2259	// the stamped account, found nothing, and DELETED the tenant's delivered
2260	// config — secrets included — bouncing the instance into a crash loop. An
2261	// absent manifest may remove the file only when the lookup account agrees
2262	// with the account the host itself names; a platform host's <sqid> label
2263	// is not a moving part, the row's account column is.
2264	func TestSyncExecHoldsConfigWhenHostNamesADifferentAccount(t *testing.T) {
2265	ctx, s, reg, dd, priv, _ := execKit(t)
2266	pk, _ := ParsePrivateKey(priv, "test")
2267	if err := reg.Upsert(registry.Route{
2268	Host: "birthday-alarm.bde.oncarlos.test", Kind: "instance", App: "birthday-alarm",
2269	Account: "bab", // the mis-stamp
2270	Status: "ready", Socket: "/run/carlos/birthday.sock",
2271	}); err != nil {
2272	t.Fatalf("Upsert: %v", err)
2273	}
2274	// A second, healthy group so the mass-removal guard (which needs EVERY
2275	// group absent) stays out of the way — this guard is per-route.
2276	if err := reg.Upsert(registry.Route{
2277	Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab",
2278	Status: "ready", Socket: "/run/carlos/inst.sock",
2279	}); err != nil {
2280	t.Fatalf("Upsert: %v", err)
2281	}
2282	if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"A": "1"}); err != nil {
2283	t.Fatalf("SetEnv: %v", err)
2284	}
2285	path := filepath.Join(dd, "birthday-alarm.bde.oncarlos.test.config.env")
2286	if err := os.WriteFile(path, []byte("BIRTHDAYALARM_INSTANCE_KEY=deadbeef\n"), 0o600); err != nil {
2287	t.Fatalf("stage: %v", err)
2288	}
2289	var bounced []string
2290	if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error {
2291	bounced = append(bounced, h)
2292	return nil
2293	}, "oncarlos.test", nil); err != nil {
2294	t.Fatalf("SyncExec: %v", err)
2295	}
2296	if _, err := os.Stat(path); err != nil {
2297	t.Fatalf("delivered config removed under a mis-stamped account: %v", err)
2298	}
2299	for _, h := range bounced {
2300	if h == "birthday-alarm.bde.oncarlos.test" {
2301	t.Fatalf("held route was bounced: %v", bounced)
2302	}
2303	}
2304	// And the guard steps aside when the row and the host agree: the same
2305	// absent manifest under the RIGHT account is a real deletion.
2306	if err := reg.Upsert(registry.Route{
2307	Host: "birthday-alarm.bab.oncarlos.test", Kind: "instance", App: "birthday-alarm",
2308	Account: "bab", Status: "ready", Socket: "/run/carlos/birthday2.sock",
2309	}); err != nil {
2310	t.Fatalf("Upsert: %v", err)
2311	}
2312	agreed := filepath.Join(dd, "birthday-alarm.bab.oncarlos.test.config.env")
2313	if err := os.WriteFile(agreed, []byte("OLD=1\n"), 0o600); err != nil {
2314	t.Fatalf("stage: %v", err)
2315	}
2316	if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "oncarlos.test", nil); err != nil {
2317	t.Fatalf("SyncExec: %v", err)
2318	}
2319	if _, err := os.Stat(agreed); !os.IsNotExist(err) {
2320	t.Fatalf("agreeing host's stale file survived, stat err = %v", err)
2321	}
2322	}
2323
2324	// The #344 guard is NOT a removal branch, and an overlay does not clear it
2325	// (task 7 fix round 1 — the brief's own reasoning, "a route with an overlay
2326	// never reaches a removal branch anyway," does not transfer to this guard).
2327	// A mis-stamp that makes the group's default-bundle READ land under the
2328	// wrong account prefix makes the instance layer's own WRITE
2329	// (ApplyInstance) land under that same wrong prefix too, so an overlay
2330	// resolving here under the mis-stamped account is exactly what a
2331	// mis-stamped fleet produces — not evidence the account column is safe to
2332	// act on. Sibling of TestSyncExecHoldsConfigWhenHostNamesADifferentAccount,
2333	// with an overlay added to the held route.
2334	func TestSyncExecHoldsConfigWhenHostNamesADifferentAccountEvenWithAnOverlay(t *testing.T) {
2335	ctx, s, reg, dd, priv, _ := execKit(t)
2336	pk, err := ParsePrivateKey(priv, "test")
2337	if err != nil {
2338	t.Fatalf("ParsePrivateKey: %v", err)
2339	}
2340	if err := reg.Upsert(registry.Route{
2341	Host: "birthday-alarm.bde.oncarlos.test", Kind: "instance", App: "birthday-alarm",
2342	Account: "bab", // the mis-stamp
2343	Status: "ready", Socket: "/run/carlos/birthday.sock",
2344	}); err != nil {
2345	t.Fatalf("Upsert: %v", err)
2346	}
2347	// The overlay lands at apps/bab/birthday-alarm/instances-config/ — the
2348	// SAME mis-stamped prefix the group's absent default bundle was looked
2349	// up under, which is the whole point: a mangled account column mangles
2350	// both.
2351	if _, _, err := ApplyInstance(ctx, s, "bab", "birthday-alarm", "birthday-alarm.bde.oncarlos.test",
2352	map[string]Var{"TENANT": {Kind: "env", Value: "mis-stamped"}}, nil); err != nil {
2353	t.Fatalf("ApplyInstance: %v", err)
2354	}
2355	// A second, healthy group so the mass-removal guard (which needs EVERY
2356	// group absent) stays out of the way — this guard is per-route.
2357	if err := reg.Upsert(registry.Route{
2358	Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab",
2359	Status: "ready", Socket: "/run/carlos/inst.sock",
2360	}); err != nil {
2361	t.Fatalf("Upsert: %v", err)
2362	}
2363	if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"A": "1"}); err != nil {
2364	t.Fatalf("SetEnv: %v", err)
2365	}
2366	path := filepath.Join(dd, "birthday-alarm.bde.oncarlos.test.config.env")
2367	if err := os.WriteFile(path, []byte("BIRTHDAYALARM_INSTANCE_KEY=deadbeef\n"), 0o600); err != nil {
2368	t.Fatalf("stage: %v", err)
2369	}
2370	var bounced []string
2371	out := captureStdout(t, func() {
2372	if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error {
2373	bounced = append(bounced, h)
2374	return nil
2375	}, "oncarlos.test", nil); err != nil {
2376	t.Fatalf("SyncExec: %v", err)
2377	}
2378	})
2379	cur, err := os.ReadFile(path)
2380	if err != nil || string(cur) != "BIRTHDAYALARM_INSTANCE_KEY=deadbeef\n" {
2381	t.Fatalf("delivered config was replaced by the overlay under a mis-stamped account: %q %v", cur, err)
2382	}
2383	for _, h := range bounced {
2384	if h == "birthday-alarm.bde.oncarlos.test" {
2385	t.Fatalf("held route was bounced: %v", bounced)
2386	}
2387	}
2388	if !strings.Contains(out, "birthday-alarm.bde.oncarlos.test") || !strings.Contains(out, "#344") {
2389	t.Fatalf("the #344 hold did not fire for a route that has an overlay:\n%s", out)
2390	}
2391	}
2392
2393	// TestSyncExecWritesConfigAckForSocketExecRoute is the review's finding 1
2394	// (2026-09-07): syncExecHost DOES bounce a socket-exec route (bounce is
2395	// non-nil for it, only a tcp-exec-file/addr route gets nil), so its ack
2396	// must carry that bounce's own result — this pins Restarted true and
2397	// RelaySuffixes naming what was actually sealed, the same way syncHost's
2398	// unit-path ack does.
2399	func TestSyncExecWritesConfigAckForSocketExecRoute(t *testing.T) {
2400	ctx, s, reg, dd, _, _ := execKit(t)
2401	priv, pub := syncRelayTestKeys(t)
2402
2403	rec := &relay.Record{Enabled: true, Incarnation: "inc00003", Regions: []string{"eu-west-1"}}
2404	masters := []relay.RegionMaster{{Region: "eu-west-1", Kid: "k1", Master: []byte("0123456789abcdef0123456789abcdef")}}
2405	wantSuffixes := sealRelayConfig(t, ctx, s, pub, "bab", "eleven", rec, masters)
2406
2407	if err := reg.Upsert(registry.Route{
2408	Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab",
2409	Status: "ready", Socket: "/run/carlos/inst.sock",
2410	}); err != nil {
2411	t.Fatalf("Upsert: %v", err)
2412	}
2413
2414	var acks []ConfigAck
2415	ack := func(a ConfigAck) { acks = append(acks, a) }
2416	bounce := func(h string) error { return nil }
2417	if err := SyncExec(ctx, reg, s, priv, dd, bounce, "", ack); err != nil {
2418	t.Fatalf("SyncExec: %v", err)
2419	}
2420
2421	if len(acks) != 1 {
2422	t.Fatalf("acks = %d, want 1: %+v", len(acks), acks)
2423	}
2424	got := acks[0]
2425	if got.Shape != "socket-exec" {
2426	t.Fatalf("ack.Shape = %q, want socket-exec", got.Shape)
2427	}
2428	if !got.Restarted {
2429	t.Fatalf("ack.Restarted = false, want true — bounce succeeded and IS this shape's restart (spec §5 step 5)")
2430	}
2431	if got.RestartError != "" {
2432	t.Fatalf("ack.RestartError = %q, want empty", got.RestartError)
2433	}
2434	if got.RelaySuffixes["eu-west-1"] != wantSuffixes["eu-west-1"] {
2435	t.Fatalf("ack.RelaySuffixes[eu-west-1] = %q, want %q", got.RelaySuffixes["eu-west-1"], wantSuffixes["eu-west-1"])
2436	}
2437	}
2438
2439	// TestSyncExecWritesConfigAckForTCPExecFileRoute is the addr shape's other
2440	// half of the same ack path: bounce is nil for it (this platform never
2441	// restarts a process it does not own), so Restarted/RestartError must stay
2442	// false/empty even though the file itself landed and named real
2443	// identities.
2444	func TestSyncExecWritesConfigAckForTCPExecFileRoute(t *testing.T) {
2445	ctx, s, reg, dd, _, _ := execKit(t)
2446	priv, pub := syncRelayTestKeys(t)
2447
2448	if err := SetExecDelivery(ctx, s, "bab", "titogo", ExecDeliveryFile); err != nil {
2449	t.Fatalf("SetExecDelivery: %v", err)
2450	}
2451	rec := &relay.Record{Enabled: true, Incarnation: "inc00004", Regions: []string{"eu-west-1"}}
2452	masters := []relay.RegionMaster{{Region: "eu-west-1", Kid: "k1", Master: []byte("fedcba9876543210fedcba9876543210")}}
2453	wantSuffixes := sealRelayConfig(t, ctx, s, pub, "bab", "titogo", rec, masters)
2454
2455	if err := reg.Upsert(registry.Route{
2456	Host: "demo.go.tito.io", Kind: "instance", App: "titogo", Account: "bab",
2457	Status: "ready", Addr: "127.0.0.1:8444",
2458	}); err != nil {
2459	t.Fatalf("Upsert: %v", err)
2460	}
2461
2462	var acks []ConfigAck
2463	ack := func(a ConfigAck) { acks = append(acks, a) }
2464	if err := SyncExec(ctx, reg, s, priv, dd, func(string) error { return nil }, "", ack); err != nil {
2465	t.Fatalf("SyncExec: %v", err)
2466	}
2467
2468	if len(acks) != 1 {
2469	t.Fatalf("acks = %d, want 1: %+v", len(acks), acks)
2470	}
2471	got := acks[0]
2472	if got.Shape != "tcp-exec-file" {
2473	t.Fatalf("ack.Shape = %q, want tcp-exec-file", got.Shape)
2474	}
2475	if got.Restarted {
2476	t.Fatalf("ack.Restarted = true, want false — this platform never restarts this shape's process")
2477	}
2478	if got.RestartError != "" {
2479	t.Fatalf("ack.RestartError = %q, want empty", got.RestartError)
2480	}
2481	if got.RelaySuffixes["eu-west-1"] != wantSuffixes["eu-west-1"] {
2482	t.Fatalf("ack.RelaySuffixes[eu-west-1] = %q, want %q", got.RelaySuffixes["eu-west-1"], wantSuffixes["eu-west-1"])
2483	}
2484	}
2485
2486	// With overlap swaps armed, config sync reports each delivered revision to
2487	// the scheduler before its (no-op) restart, and never for a shape the
2488	// platform does not restart (zero-downtime spec §8.1).
2489	func TestSyncExecReportsTheDeliveredRevision(t *testing.T) {
2490	ctx, s, reg, dd, priv, _ := execKit(t)
2491	pk, err := ParsePrivateKey(priv, "test")
2492	if err != nil {
2493	t.Fatal(err)
2494	}
2495	if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil {
2496	t.Fatal(err)
2497	}
2498	for _, rt := range []registry.Route{
2499	{Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/inst.sock"},
2500	{Host: "bot.eleven.bab.sidecar.internal", Kind: "sidecar", App: "eleven", Account: "bab", Status: "ready"},
2501	} {
2502	if err := reg.Upsert(rt); err != nil {
2503	t.Fatal(err)
2504	}
2505	}
2506	var order, revs []string
2507	cb := Callbacks{
2508	Restart: func(h string) error { order = append(order, "restart:"+h); return nil },
2509	Delivered: func(h, rev string) { order = append(order, "delivered:"+h); revs = append(revs, rev) },
2510	}
2511	if err := SyncExecWithCallbacks(ctx, reg, s, pk, dd, "", cb); err != nil {
2512	t.Fatal(err)
2513	}
2514	want := []string{"delivered:inst.example.com", "restart:inst.example.com"}
2515	if !slices.Equal(order, want) {
2516	t.Fatalf("order = %v, want %v (no report for the sidecar)", order, want)
2517	}
2518	if !strings.HasPrefix(revs[0], "sha256:") {
2519	t.Fatalf("rev = %q, want the rendered identity", revs[0])
2520	}
2521	order = nil
2522	if err := SyncExecWithCallbacks(ctx, reg, s, pk, dd, "", cb); err != nil {
2523	t.Fatal(err)
2524	}
2525	if len(order) != 0 {
2526	t.Fatalf("an unchanged file reported %v", order)
2527	}
2528	if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "info"}); err != nil {
2529	t.Fatal(err)
2530	}
2531	if err := SyncExecWithCallbacks(ctx, reg, s, pk, dd, "", cb); err != nil {
2532	t.Fatal(err)
2533	}
2534	if len(revs) != 2 || revs[1] == revs[0] {
2535	t.Fatalf("revs = %v, want a new revision for new content", revs)
2536	}
2537	}
2538
2539	// Final review F7 (1) and (2): with Managed set, the pass splits as the
2540	// scheduler does. A managed socket route gets Restart and is reported to
2541	// Delivered. A socket route Managed says no to is restarted by Unmanaged,
2542	// as config sync restarts it with the flag off, is not reported, and its
2543	// ack says it was restarted. A managed native sidecar is reported though
2544	// nothing here restarts it (the scheduler's config attempt does); an
2545	// unmanaged one is neither reported nor restarted, as before.
2546	func TestSyncExecSplitsDeliveryByManagement(t *testing.T) {
2547	ctx, s, reg, dd, priv, _ := execKit(t)
2548	pk, err := ParsePrivateKey(priv, "test")
2549	if err != nil {
2550	t.Fatal(err)
2551	}
2552	if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil {
2553	t.Fatal(err)
2554	}
2555	const (
2556	mInst = "m.example.com"
2557	uInst = "u.example.com"
2558	mBot = "bot.eleven.bab.sidecar.internal"
2559	uBot = "bot2.eleven.bab.sidecar.internal"
2560	)
2561	for _, rt := range []registry.Route{
2562	{Host: mInst, Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/m.sock"},
2563	{Host: uInst, Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/u.sock"},
2564	{Host: mBot, Kind: "sidecar", App: "eleven", Account: "bab", Status: "ready"},
2565	{Host: uBot, Kind: "sidecar", App: "eleven", Account: "bab", Status: "ready"},
2566	} {
2567	if err := reg.Upsert(rt); err != nil {
2568	t.Fatal(err)
2569	}
2570	}
2571	calls := map[string][]string{}
2572	acks := map[string]ConfigAck{}
2573	cb := Callbacks{
2574	Restart: func(h string) error { calls[h] = append(calls[h], "restart"); return nil },
2575	Unmanaged: func(h string) error { calls[h] = append(calls[h], "unmanaged"); return nil },
2576	Delivered: func(h, _ string) { calls[h] = append(calls[h], "delivered") },
2577	Ack: func(a ConfigAck) { acks[a.Host] = a },
2578	Managed: func(rt registry.Route) bool { return rt.Host == mInst || rt.Host == mBot },
2579	}
2580	if err := SyncExecWithCallbacks(ctx, reg, s, pk, dd, "", cb); err != nil {
2581	t.Fatal(err)
2582	}
2583	want := map[string][]string{
2584	mInst: {"delivered", "restart"},
2585	uInst: {"unmanaged"},
2586	mBot: {"delivered"},
2587	}
2588	if !reflect.DeepEqual(calls, want) {
2589	t.Fatalf("calls = %v, want %v", calls, want)
2590	}
2591	if a := acks[uInst]; !a.Restarted || a.RestartError != "" {
2592	t.Fatalf("unmanaged ack = %+v, want today's: restarted", a)
2593	}
2594	if a := acks[mBot]; a.Restarted || a.RestartError != "" {
2595	t.Fatalf("managed sidecar ack = %+v: nothing here restarted it", a)
2596	}
2597
2598	// Managed nil: every socket route gets Restart and Delivered, and no
2599	// sidecar is reported, exactly as before.
2600	if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "info"}); err != nil {
2601	t.Fatal(err)
2602	}
2603	calls = map[string][]string{}
2604	cb.Managed = nil
2605	if err := SyncExecWithCallbacks(ctx, reg, s, pk, dd, "", cb); err != nil {
2606	t.Fatal(err)
2607	}
2608	want = map[string][]string{
2609	mInst: {"delivered", "restart"},
2610	uInst: {"delivered", "restart"},
2611	}
2612	if !reflect.DeepEqual(calls, want) {
2613	t.Fatalf("with Managed nil, calls = %v, want %v", calls, want)
2614	}
2615	}
2616
2617	// Final review F7 (3): a restart that answers ErrDeliveredOnWake acks the
2618	// revision as delivered on wake (Hibernating), the unit pass's marker, with
2619	// no restart claimed and no error, on both the write and the removal paths.
2620	func TestSyncExecAcksDeliveredOnWake(t *testing.T) {
2621	ctx, s, reg, dd, priv, _ := execKit(t)
2622	pk, err := ParsePrivateKey(priv, "test")
2623	if err != nil {
2624	t.Fatal(err)
2625	}
2626	if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil {
2627	t.Fatal(err)
2628	}
2629	host := "asleep.example.com"
2630	if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", App: "eleven", Account: "bab", Status: "ready",
2631	Socket: "/run/carlos/asleep.sock", Hibernate: true}); err != nil {
2632	t.Fatal(err)
2633	}
2634	var acks []ConfigAck
2635	restarts := 0
2636	cb := Callbacks{
2637	Restart: func(string) error { restarts++; return ErrDeliveredOnWake },
2638	Ack: func(a ConfigAck) { acks = append(acks, a) },
2639	}
2640	if err := SyncExecWithCallbacks(ctx, reg, s, pk, dd, "", cb); err != nil {
2641	t.Fatal(err)
2642	}
2643	if restarts != 1 || len(acks) != 1 {
2644	t.Fatalf("restarts %d, acks %+v", restarts, acks)
2645	}
2646	if a := acks[0]; !a.Hibernating || a.Restarted || a.RestartError != "" {
2647	t.Fatalf("ack = %+v, want delivered on wake: Hibernating, not restarted, no error", a)
2648	}
2649	}
2650
2651	// keyReadStore records every key and prefix a pass reads through it.
2652	type keyReadStore struct {
2653	release.Store
2654	mu sync.Mutex
2655	read []string
2656	}
2657
2658	func (s *keyReadStore) note(k string) { s.mu.Lock(); s.read = append(s.read, k); s.mu.Unlock() }
2659
2660	func (s *keyReadStore) Get(ctx context.Context, key string) ([]byte, string, error) {
2661	s.note(key)
2662	return s.Store.Get(ctx, key)
2663	}
2664
2665	func (s *keyReadStore) List(ctx context.Context, prefix string) ([]string, error) {
2666	s.note(prefix)
2667	return s.Store.List(ctx, prefix)
2668	}
2669
2670	func (s *keyReadStore) ListDirs(ctx context.Context, prefix string) ([]string, error) {
2671	s.note(prefix)
2672	return s.Store.ListDirs(ctx, prefix)
2673	}
2674
2675	// TestSyncExecWithRouteFilter pins the scoped config pass the agent runs
2676	// when the change index says one app moved (S3 change-index design §4.5):
2677	// a route the filter leaves out is not in scope, so nothing is read for it,
2678	// its file is left exactly as it is (neither rewritten nor reaped), and the
2679	// routes the filter keeps converge as on a full pass.
2680	func TestSyncExecWithRouteFilter(t *testing.T) {
2681	t.Parallel()
2682	ctx, base, reg, dd, priv, _ := execKit(t)
2683	pk, err := ParsePrivateKey(priv, "test")
2684	if err != nil {
2685	t.Fatalf("ParsePrivateKey: %v", err)
2686	}
2687	for _, app := range []string{"shop", "cafe"} {
2688	if err := SetEnv(ctx, base, "bab", app, map[string]string{"APP": app}); err != nil {
2689	t.Fatalf("SetEnv %s: %v", app, err)
2690	}
2691	}
2692	if err := SetExecDelivery(ctx, base, "bab", "cafe", ExecDeliveryFile); err != nil {
2693	t.Fatalf("SetExecDelivery: %v", err)
2694	}
2695	for _, rt := range []registry.Route{
2696	{Host: "shop.example.com", Kind: "instance", App: "shop", Account: "bab", Status: "ready", Socket: "/run/carlos/shop.sock"},
2697	{Host: "cafe.example.com", Kind: "instance", App: "cafe", Account: "bab", Status: "ready", Socket: "/run/carlos/cafe.sock"},
2698	// The addr shape: its opt-in is read before the resolution loop.
2699	{Host: "cafe-addr.example.com", Kind: "instance", App: "cafe", Account: "bab", Status: "ready", Addr: "127.0.0.1:8444"},
2700	} {
2701	if err := reg.Upsert(rt); err != nil {
2702	t.Fatalf("Upsert %s: %v", rt.Host, err)
2703	}
2704	}
2705	stale := filepath.Join(dd, "cafe.example.com.config.env")
2706	if err := os.WriteFile(stale, []byte("OLD=1\n"), 0o600); err != nil {
2707	t.Fatal(err)
2708	}
2709	s := &keyReadStore{Store: base}
2710	keep := WithRouteFilter(func(rt registry.Route) bool { return rt.App != "cafe" })
2711	var bounced []string
2712	if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error { bounced = append(bounced, h); return nil }, "", nil, keep); err != nil {
2713	t.Fatalf("SyncExec: %v", err)
2714	}
2715	var shopRead bool
2716	for _, k := range s.read {
2717	if strings.HasPrefix(k, "apps/bab/cafe/") {
2718	t.Errorf("the filtered pass read %s, under an app the filter left out", k)
2719	}
2720	shopRead = shopRead || strings.HasPrefix(k, "apps/bab/shop/")
2721	}
2722	if !shopRead {
2723	t.Fatalf("the pass read nothing for shop (%v): the witness pinned nothing", s.read)
2724	}
2725	if got, err := os.ReadFile(filepath.Join(dd, "shop.example.com.config.env")); err != nil || string(got) != "APP=shop\n" {
2726	t.Fatalf("shop's file = %q, %v; want APP=shop", got, err)
2727	}
2728	if got, err := os.ReadFile(stale); err != nil || string(got) != "OLD=1\n" {
2729	t.Fatalf("cafe's file = %q, %v; a route the filter leaves out keeps its file as it is", got, err)
2730	}
2731	if _, err := os.Stat(filepath.Join(dd, "cafe-addr.example.com.config.env")); !os.IsNotExist(err) {
2732	t.Fatalf("cafe's addr route got a file from a pass that left it out: %v", err)
2733	}
2734	if !slices.Equal(bounced, []string{"shop.example.com"}) {
2735	t.Fatalf("bounced %v, want only shop.example.com", bounced)
2736	}
2737	}
2738