carlos / platform Public

Download this file

1	package host
2
3	import (
4	"bytes"
5	"context"
6	"crypto/ed25519"
7	"crypto/sha256"
8	"encoding/hex"
9	"encoding/json"
10	"errors"
11	"fmt"
12	"os"
13	"path/filepath"
14	"slices"
15	"strings"
16	"testing"
17	"time"
18
19	"github.com/carlosframework/platform/internal/moves"
20	"github.com/carlosframework/platform/internal/provision"
21	"github.com/carlosframework/platform/internal/registry"
22	"github.com/carlosframework/platform/internal/release"
23	)
24
25	// applyFixture is a fake box: temp opt/data/etc roots, a DirStore with a
26	// signed pointer, and a sysOps that records every call and answers like a
27	// healthy systemd. The candidate "binary" is answered by the fake run.
28	type applyFixture struct {
29	t *testing.T
30	a *systemApplier
31	store *release.DirStore
32	priv ed25519.PrivateKey
33	pub ed25519.PublicKey
34	calls []string // "systemctl is-active carlos-edge", "run <base> version", …
35	answer map[string]string // "version:<path>" → stdout; "plan:<path>" → JSON plan
36	now time.Time
37	edgeUp bool
38	nrestarts string
39	activeEnter string
40	probeErr error
41	edgeStartedThisBoot bool
42	}
43
44	func newApplyFixture(t *testing.T, channel, build string) *applyFixture {
45	t.Helper()
46	pub, priv, err := ed25519.GenerateKey(nil)
47	if err != nil {
48	t.Fatal(err)
49	}
50	f := &applyFixture{t: t, store: release.NewDirStore(t.TempDir()), priv: priv, answer: map[string]string{},
51	now: time.Date(2026, 9, 6, 11, 0, 0, 0, time.UTC), edgeUp: true, nrestarts: "0", activeEnter: "1000", edgeStartedThisBoot: true}
52	f.pub = pub
53	f.a = &systemApplier{
54	openStore: func(context.Context) (release.Store, ed25519.PublicKey, error) { return f.store, f.pub, nil },
55	channel: channel,
56	optDir: t.TempDir(), dataDir: t.TempDir(), etcDir: t.TempDir(), build: build,
57	platform: "linux-arm64",
58	}
59	f.a.ops = sysOps{
60	systemctl: func(args ...string) (string, error) {
61	f.calls = append(f.calls, "systemctl "+strings.Join(args, " "))
62	switch {
63	case args[0] == "is-active" && args[1] == systemEdgeUnit:
64	if f.edgeUp {
65	return "active\n", nil
66	}
67	return "failed\n", errors.New("inactive")
68	case args[0] == "show":
69	return f.nrestarts + "\n" + f.activeEnter + "\n", nil
70	}
71	return "", nil
72	},
73	run: func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
74	f.calls = append(f.calls, "run "+filepath.Base(name)+" "+strings.Join(args, " "))
75	if len(args) == 1 && args[0] == "version" {
76	if v, ok := f.answer["version:"+name]; ok {
77	return v, nil
78	}
79	// With no scripted answer the "binary" speaks for its own
80	// bytes ("#!/bin/sh\necho carlos vN"), so a rename over it
81	// changes what it reports — which is what the install does.
82	if b, err := os.ReadFile(name); err == nil {
83	if _, rest, ok := strings.Cut(string(b), "echo "); ok {
84	return strings.SplitN(rest, "\n", 2)[0] + "\n", nil
85	}
86	}
87	return "", errors.New("no such binary")
88	}
89	if len(args) >= 3 && args[0] == "bootstrap" && args[1] == "-root" {
90	plan, ok := f.answer["plan:"+name]
91	if !ok {
92	return "", errors.New("bootstrap failed")
93	}
94	units := strings.Split(f.answer["units:"+name], " ")
95	dir := filepath.Join(args[2], "etc", "systemd", "system")
96	os.MkdirAll(dir, 0o755)
97	for _, u := range units {
98	if u != "" {
99	os.WriteFile(filepath.Join(dir, u), []byte("[Unit]\n# "+filepath.Base(name)+"\n"), 0o644)
100	}
101	}
102	os.MkdirAll(filepath.Join(args[2], "etc", "sysusers.d"), 0o755)
103	os.WriteFile(filepath.Join(args[2], "etc", "sysusers.d", "carlos.conf"), []byte("g carlos-sockets -\n"), 0o644)
104	os.MkdirAll(filepath.Join(args[2], "etc", "tmpfiles.d"), 0o755)
105	os.WriteFile(filepath.Join(args[2], "etc", "tmpfiles.d", "carlos.conf"), []byte("d /run/carlos\n"), 0o644)
106	os.MkdirAll(filepath.Join(args[2], "data"), 0o755)
107	os.WriteFile(filepath.Join(args[2], "data", "registry.db"), nil, 0o640)
108	return plan, nil
109	}
110	// The two config appliers the install shells out to.
111	if b := filepath.Base(name); b == "systemd-sysusers" || b == "systemd-tmpfiles" {
112	return "", nil
113	}
114	return "", errors.New("unexpected command")
115	},
116	now: func() time.Time { return f.now },
117	sleep: func(d time.Duration) { f.now = f.now.Add(d) },
118	probeEdge: func() error { return f.probeErr },
119	edgeStartedThisBoot: func() bool { return f.edgeStartedThisBoot },
120	logf: t.Logf,
121	}
122	// The layout guard opens the registry only for a candidate that
123	// predates the layout, and closes it; an empty one in the fake data dir
124	// is a flat box.
125	f.a.openRegistry = func() (*registry.Registry, error) {
126	return registry.Open(filepath.Join(f.a.dataDir, "registry.db"))
127	}
128	os.MkdirAll(systemRootDir(f.a.optDir), 0o755)
129	os.MkdirAll(filepath.Join(f.a.etcDir, "systemd", "system"), 0o755)
130	// The on-disk binary answers `version` with build.
131	os.WriteFile(systemBinaryPath(f.a.optDir), []byte("#!/bin/sh\necho carlos "+build+"\n"), 0o755)
132	f.answer["version:"+systemBinaryPath(f.a.optDir)] = "carlos " + build + "\n"
133	f.answer["plan:"+systemBinaryPath(f.a.optDir)] = `{"enabled":["carlos-edge.service","carlos-adopt.timer","carlos-adopt.path","carlos-stage.path","carlos-restart.path","carlos-system-update.path","carlos-system-recover.service"],"inert":["carlos-restore-verify.timer"]}`
134	f.answer["units:"+systemBinaryPath(f.a.optDir)] = "carlos-edge.service carlos-adopt.timer carlos-adopt.path carlos-stage.path carlos-restart.path carlos-system-update.path carlos-system-update.service carlos-system-recover.service carlos-instance-control.service carlos-old-only.timer"
135	f.a.participation = updateParticipationFixture(t)
136	return f
137	}
138
139	// promote signs a pointer for f.a.channel naming version with body as the
140	// linux-arm64 artifact, stages body into the agent's cache, and writes the
141	// request the agent would have written. Returns the request.
142	func (f *applyFixture) promote(version string, body []byte, noop bool) systemRequest {
143	f.t.Helper()
144	sum := sha256.Sum256(body)
145	rel := release.SystemRelease{Version: version, Artifacts: map[string]release.SystemArtifact{
146	"linux-arm64": {Key: release.SystemArtifactKey(version, "linux-arm64"), SHA256: hex.EncodeToString(sum[:])}}}
147	f.store.Put(context.Background(), rel.Artifacts["linux-arm64"].Key, body, "")
148	p, err := release.PromoteSystem(context.Background(), f.store, release.RootSigner(f.priv), "router", f.a.channel, rel, release.SystemPromoteOptions{})
149	if err != nil {
150	f.t.Fatal(err)
151	}
152	req := systemRequest{Version: version, PromotedAt: p.PromotedAt, NoOp: noop, SeenAt: f.now}
153	os.MkdirAll(systemDataDir(f.a.dataDir), 0o755)
154	writeJSONDurable(systemRequestPath(f.a.dataDir), req, 0o644)
155	if !noop {
156	os.MkdirAll(filepath.Dir(systemCachePath(f.a.dataDir, version)), 0o755)
157	os.WriteFile(systemCachePath(f.a.dataDir, version), body, 0o644)
158	}
159	return req
160	}
161
162	func (f *applyFixture) result() *systemResult { r, _ := readSystemResult(f.a.optDir); return r }
163
164	func (f *applyFixture) called(prefix string) bool {
165	for _, c := range f.calls {
166	if strings.HasPrefix(c, prefix) {
167	return true
168	}
169	}
170	return false
171	}
172
173	func TestApplyNoRequestIsNothingToDo(t *testing.T) {
174	f := newApplyFixture(t, "stable", "v1")
175	if err := f.a.Run(context.Background(), false); err != nil {
176	t.Fatal(err)
177	}
178	if f.result() != nil || len(f.calls) != 0 {
179	t.Fatalf("result=%+v calls=%v", f.result(), f.calls)
180	}
181	}
182
183	func TestApplyLockKeepsTwoRunsApart(t *testing.T) {
184	f := newApplyFixture(t, "stable", "v1")
185	f.promote("v2", []byte("x"), false)
186	// Hold the lock as another process would.
187	held, err := os.OpenFile(f.a.lockPath(), os.O_CREATE|os.O_RDWR, 0o600)
188	if err != nil {
189	t.Fatal(err)
190	}
191	defer held.Close()
192	if err := flockExclusiveNonblock(held); err != nil {
193	t.Fatal(err)
194	}
195	if err := f.a.Run(context.Background(), false); err != nil {
196	t.Fatalf("a locked-out path run must exit 0 quietly: %v", err)
197	}
198	if f.result() != nil {
199	t.Fatal("a locked-out run wrote a result")
200	}
201	}
202
203	func TestApplyRequestIsAHintNotAnInstruction(t *testing.T) {
204	f := newApplyFixture(t, "stable", "v1")
205	// A symlinked parent under /data.
206	os.MkdirAll(filepath.Join(f.a.dataDir, "elsewhere"), 0o755)
207	os.Symlink(filepath.Join(f.a.dataDir, "elsewhere"), filepath.Join(f.a.dataDir, "system"))
208	os.MkdirAll(filepath.Join(f.a.dataDir, "elsewhere", "router"), 0o755)
209	os.WriteFile(filepath.Join(f.a.dataDir, "elsewhere", "router", "requested.json"), []byte(`{"version":"v2"}`), 0o644)
210	if _, err := f.a.readRequest(); err == nil {
211	t.Fatal("followed a symlinked parent under /data")
212	}
213	os.Remove(filepath.Join(f.a.dataDir, "system"))
214	// A symlinked request file.
215	os.MkdirAll(systemDataDir(f.a.dataDir), 0o755)
216	os.WriteFile(filepath.Join(f.a.dataDir, "real.json"), []byte(`{"version":"v2"}`), 0o644)
217	os.Symlink(filepath.Join(f.a.dataDir, "real.json"), systemRequestPath(f.a.dataDir))
218	if _, err := f.a.readRequest(); err == nil {
219	t.Fatal("followed a symlinked request")
220	}
221	os.Remove(systemRequestPath(f.a.dataDir))
222	// A traversal version.
223	writeJSONDurable(systemRequestPath(f.a.dataDir), systemRequest{Version: "../../etc/passwd", PromotedAt: f.now}, 0o644)
224	if _, err := f.a.readRequest(); err == nil {
225	t.Fatal("accepted a traversal version")
226	}
227	// Oversized.
228	os.WriteFile(systemRequestPath(f.a.dataDir), []byte(`{"version":"`+strings.Repeat("a", 5000)+`"}`), 0o644)
229	if _, err := f.a.readRequest(); err == nil {
230	t.Fatal("accepted a 5 KiB request")
231	}
232	}
233
234	func TestApplyChannelComesFromRootNotTheRequest(t *testing.T) {
235	// The box is on stable; a valid CANARY pointer exists and the agent
236	// requests its version. Root reads control/system/router/stable.json
237	// (missing) and refuses — a signed canary promote does not authorise a
238	// stable box.
239	f := newApplyFixture(t, "stable", "v1")
240	f.a.channel = "canary"
241	req := f.promote("v2", []byte("x"), false) // signs the CANARY pointer
242	f.a.channel = "stable"
243	if err := f.a.Run(context.Background(), false); err != nil {
244	t.Fatal(err)
245	}
246	r := f.result()
247	if r == nil || r.Outcome != systemOutcomeRefused || !r.matches("stable", "v2", req.PromotedAt) {
248	t.Fatalf("result = %+v", r)
249	}
250	if _, err := os.Stat(systemBinaryPath(f.a.optDir) + ".prev"); err == nil {
251	t.Fatal("touched the box")
252	}
253	}
254
255	func TestApplyStaleTupleIsRefusedAgainstTheRequestsTuple(t *testing.T) {
256	t.Parallel()
257	f := newApplyFixture(t, "stable", "v1")
258	old := f.promote("v2", []byte("x"), false)
259	time.Sleep(1100 * time.Millisecond)
260	f.promote("v3", []byte("y"), false)
261	// Put the OLD request back: the agent's spread started under v2.
262	writeJSONDurable(systemRequestPath(f.a.dataDir), old, 0o644)
263	if err := f.a.Run(context.Background(), false); err != nil {
264	t.Fatal(err)
265	}
266	r := f.result()
267	if r == nil || r.Outcome != systemOutcomeRefused || r.Version != "v2" || !r.PromotedAt.Equal(old.PromotedAt) {
268	t.Fatalf("result = %+v (must name the REQUEST's tuple so v3 is not suppressed)", r)
269	}
270	}
271
272	func TestApplyReplayGuardIsPerChannelAndEqualIsAllowed(t *testing.T) {
273	f := newApplyFixture(t, "stable", "v1")
274	req := f.promote("v2", []byte("x"), true)
275	// Older than the watermark → refused, seen unchanged.
276	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "seen.json"), systemSeen{"stable": req.PromotedAt.Add(time.Hour), "canary": time.Time{}}, 0o644)
277	if err := f.a.Run(context.Background(), false); err != nil {
278	t.Fatal(err)
279	}
280	if r := f.result(); r == nil || r.Outcome != systemOutcomeRefused || !strings.Contains(r.Reason, "replay") {
281	t.Fatalf("result = %+v", r)
282	}
283	// Equal → allowed (a retry of the same promote).
284	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "seen.json"), systemSeen{"stable": req.PromotedAt}, 0o644)
285	if err := f.a.Run(context.Background(), false); err != nil {
286	t.Fatal(err)
287	}
288	if r := f.result(); r == nil || r.Outcome != systemOutcomeNoted {
289	t.Fatalf("result = %+v", r)
290	}
291	// Another channel's newer watermark does not block this channel.
292	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "seen.json"), systemSeen{"canary": req.PromotedAt.Add(time.Hour)}, 0o644)
293	if err := f.a.Run(context.Background(), false); err != nil {
294	t.Fatal(err)
295	}
296	if r := f.result(); r == nil || r.Outcome != systemOutcomeNoted {
297	t.Fatalf("result = %+v", r)
298	}
299	seen, _ := readSystemSeen(f.a.optDir)
300	if !seen["stable"].Equal(req.PromotedAt) {
301	t.Fatalf("seen not advanced: %v", seen)
302	}
303	}
304
305	func TestApplyNoOpNotesAndTouchesNothing(t *testing.T) {
306	f := newApplyFixture(t, "stable", "v2")
307	f.promote("v2", []byte("x"), true)
308	if err := f.a.Run(context.Background(), false); err != nil {
309	t.Fatal(err)
310	}
311	if r := f.result(); r == nil || r.Outcome != systemOutcomeNoted {
312	t.Fatalf("result = %+v", r)
313	}
314	if _, err := os.Stat(f.a.stagingDir()); err == nil {
315	t.Fatal("a no-op created staging/")
316	}
317	if len(f.calls) != 0 {
318	t.Fatalf("a no-op ran commands: %v", f.calls)
319	}
320	}
321
322	func TestApplyStoreErrorIsDeferredNotRefused(t *testing.T) {
323	f := newApplyFixture(t, "stable", "v1")
324	req := f.promote("v2", []byte("x"), false)
325	f.a.openStore = func(context.Context) (release.Store, ed25519.PublicKey, error) {
326	return unreadableStore{f.store}, f.pub, nil
327	}
328	if err := f.a.Run(context.Background(), false); err != nil {
329	t.Fatal(err)
330	}
331	if r := f.result(); r == nil || r.Outcome != systemOutcomeDeferred || !r.matches("stable", "v2", req.PromotedAt) {
332	t.Fatalf("result = %+v", r)
333	}
334	}
335
336	// unreadableStore is the bucket-outage fixture for reads: the door is
337	// restarting, so the pointer cannot be fetched at all.
338	type unreadableStore struct{ release.Store }
339
340	func (unreadableStore) Get(context.Context, string) ([]byte, string, error) {
341	return nil, "", errors.New("door restarting")
342	}
343
344	func TestApplyStagesTheCandidateFromARootOwnedCopy(t *testing.T) {
345	f := newApplyFixture(t, "stable", "v1")
346	body := []byte("#!/bin/sh\necho carlos v2\n")
347	f.promote("v2", body, false)
348	f.answer["version:"+filepath.Join(f.a.stagingDir(), "carlos")] = "carlos v2\n"
349	p, err := f.a.verifyPointer(context.Background())
350	if err != nil {
351	t.Fatal(err)
352	}
353	req, _ := f.a.readRequest()
354	staged, err := f.a.stageCandidate(context.Background(), p, req)
355	if err != nil {
356	t.Fatal(err)
357	}
358	st, _ := os.Stat(staged)
359	if st.Mode().Perm() != 0o755 {
360	t.Fatalf("staged mode = %v after verification", st.Mode().Perm())
361	}
362	if dst, _ := os.Stat(f.a.stagingDir()); dst.Mode().Perm() != 0o700 {
363	t.Fatalf("staging dir mode = %v", dst.Mode().Perm())
364	}
365	// Swapping the cache after staging changes nothing about the copy.
366	os.WriteFile(systemCachePath(f.a.dataDir, "v2"), []byte("evil"), 0o644)
367	got, _ := os.ReadFile(staged)
368	if string(got) != string(body) {
369	t.Fatal("staged copy follows the cache")
370	}
371	// A sha mismatch is refused BEFORE the copy is made executable and
372	// staging/ is cleared.
373	f2 := newApplyFixture(t, "stable", "v1")
374	f2.promote("v2", body, false)
375	os.WriteFile(systemCachePath(f2.a.dataDir, "v2"), []byte("evil"), 0o644)
376	p2, _ := f2.a.verifyPointer(context.Background())
377	req2, _ := f2.a.readRequest()
378	if _, err := f2.a.stageCandidate(context.Background(), p2, req2); err == nil || !strings.Contains(err.Error(), "sha256") {
379	t.Fatalf("err = %v", err)
380	}
381	if st, err := os.Stat(filepath.Join(f2.a.stagingDir(), "carlos")); err == nil && st.Mode().Perm() != 0o600 {
382	t.Fatalf("an unverified copy was made executable: mode = %v", st.Mode().Perm())
383	}
384	// A candidate that lies about its version is refused.
385	f3 := newApplyFixture(t, "stable", "v1")
386	f3.promote("v2", body, false)
387	f3.answer["version:"+filepath.Join(f3.a.stagingDir(), "carlos")] = "carlos v9\n"
388	p3, _ := f3.a.verifyPointer(context.Background())
389	req3, _ := f3.a.readRequest()
390	if _, err := f3.a.stageCandidate(context.Background(), p3, req3); err == nil || !strings.Contains(err.Error(), "reports") {
391	t.Fatalf("err = %v", err)
392	}
393	// A symlinked cache file is refused.
394	f4 := newApplyFixture(t, "stable", "v1")
395	f4.promote("v2", body, false)
396	os.Remove(systemCachePath(f4.a.dataDir, "v2"))
397	os.WriteFile(filepath.Join(f4.a.dataDir, "real"), body, 0o644)
398	os.Symlink(filepath.Join(f4.a.dataDir, "real"), systemCachePath(f4.a.dataDir, "v2"))
399	p4, _ := f4.a.verifyPointer(context.Background())
400	req4, _ := f4.a.readRequest()
401	if _, err := f4.a.stageCandidate(context.Background(), p4, req4); err == nil {
402	t.Fatal("followed a symlinked cache file")
403	}
404	}
405
406	// applyHappy sets up a v1 box with a v2 candidate whose fake bootstrap
407	// plan adds carlos-new-only.path and drops carlos-old-only.timer.
408	func applyHappy(t *testing.T) (*applyFixture, systemRequest) {
409	return applyHappyVersion(t, "v2")
410	}
411
412	// applyHappyVersion is applyHappy with the candidate's version named, for
413	// the guards that place a version against release.LayoutFloor.
414	func applyHappyVersion(t *testing.T, version string) (*applyFixture, systemRequest) {
415	f := newApplyFixture(t, "stable", "v1")
416	body := []byte("#!/bin/sh\necho carlos " + version + "\n")
417	req := f.promote(version, body, false)
418	staged := filepath.Join(f.a.stagingDir(), "carlos")
419	f.answer["version:"+staged] = "carlos " + version + "\n"
420	f.answer["plan:"+staged] = `{"enabled":["carlos-edge.service","carlos-adopt.timer","carlos-adopt.path","carlos-stage.path","carlos-restart.path","carlos-system-update.path","carlos-system-recover.service","carlos-new-only.path"],"inert":["carlos-restore-verify.timer"]}`
421	f.answer["units:"+staged] = "carlos-edge.service carlos-adopt.timer carlos-adopt.path carlos-stage.path carlos-restart.path carlos-system-update.path carlos-system-update.service carlos-system-recover.service carlos-instance-control.service carlos-new-only.path"
422	// The unit v1 owns and v2 drops is on the box, so removing it is a
423	// thing that can be seen to happen — or not to.
424	os.WriteFile(filepath.Join(f.a.etcDir, "systemd", "system", "carlos-old-only.timer"), []byte("[Unit]\n# v1\n"), 0o644)
425	// .prev, once made, answers as v1 and stages v1's units.
426	prev := systemPrevPath(f.a.optDir)
427	f.answer["version:"+prev] = "carlos v1\n"
428	f.answer["plan:"+prev] = f.answer["plan:"+systemBinaryPath(f.a.optDir)]
429	f.answer["units:"+prev] = f.answer["units:"+systemBinaryPath(f.a.optDir)]
430	// The on-disk binary answers from its own bytes, so it says v1 until
431	// the rename and v2 after it — a scripted answer would make the run a
432	// no-op before it started.
433	delete(f.answer, "version:"+systemBinaryPath(f.a.optDir))
434	return f, req
435	}
436
437	func TestApplyInstallsGatesArmsAndRecords(t *testing.T) {
438	f, req := applyHappy(t)
439	if err := f.a.Run(context.Background(), false); err != nil {
440	t.Fatal(err)
441	}
442	r := f.result()
443	if r == nil || r.Outcome != systemOutcomeApplied || !r.matches("stable", "v2", req.PromotedAt) {
444	t.Fatalf("result = %+v", r)
445	}
446	ap, _ := readSystemApplied(f.a.optDir)
447	if ap == nil || ap.Version != "v2" || ap.Channel != "stable" {
448	t.Fatalf("applied = %+v", ap)
449	}
450	if j, _ := readSystemJournal(f.a.optDir); j != nil {
451	t.Fatalf("journal left open: %+v", j)
452	}
453	if _, err := os.Stat(f.a.stagingDir()); err == nil {
454	t.Fatal("staging/ left behind")
455	}
456	bin, _ := os.ReadFile(systemBinaryPath(f.a.optDir))
457	if !strings.Contains(string(bin), "carlos v2") {
458	t.Fatal("binary not swapped")
459	}
460	prev, _ := os.ReadFile(systemPrevPath(f.a.optDir))
461	if !strings.Contains(string(prev), "carlos v1") {
462	t.Fatal(".prev is not the old binary")
463	}
464	units := filepath.Join(f.a.etcDir, "systemd", "system")
465	if _, err := os.Stat(filepath.Join(units, "carlos-new-only.path")); err != nil {
466	t.Error("new unit not installed")
467	}
468	if _, err := os.Stat(filepath.Join(units, "carlos-old-only.timer")); err == nil {
469	t.Error("removed unit still installed")
470	}
471	if _, err := os.Stat(filepath.Join(f.a.etcDir, "sysusers.d", "carlos.conf")); err != nil {
472	t.Error("sysusers not installed")
473	}
474	if _, err := os.Stat(filepath.Join(f.a.etcDir, "tmpfiles.d", "carlos.conf")); err != nil {
475	t.Error("tmpfiles not installed")
476	}
477	if _, err := os.Stat(filepath.Join(f.a.etcDir, "data", "registry.db")); err == nil {
478	t.Error("staging data/ copied")
479	}
480	// Order of the systemctl calls is the spec.
481	joined := strings.Join(f.calls, "\n")
482	order := []string{
483	"systemctl stop carlos-adopt.timer carlos-adopt.path carlos-stage.path carlos-restart.path carlos-adopt.service carlos-config-sync.service carlos-stage.service carlos-restart.service",
484	"systemctl disable --now carlos-old-only.timer",
485	"run systemd-sysusers",
486	"run systemd-tmpfiles --create",
487	"systemctl daemon-reload",
488	"systemctl reset-failed carlos-edge",
489	"systemctl restart carlos-edge",
490	"systemctl is-active carlos-edge",
491	"systemctl daemon-reload",
492	"systemctl enable carlos-edge.service carlos-adopt.timer carlos-adopt.path carlos-stage.path carlos-restart.path carlos-system-update.path carlos-system-recover.service carlos-new-only.path",
493	"systemctl start carlos-adopt.timer carlos-adopt.path carlos-stage.path carlos-restart.path carlos-system-update.path carlos-new-only.path",
494	}
495	at := 0
496	for _, want := range order {
497	i := strings.Index(joined[at:], want)
498	if i < 0 {
499	t.Fatalf("call order broken: %q not found after position %d in:\n%s", want, at, joined)
500	}
501	at += i + len(want)
502	}
503	for _, c := range f.calls {
504	if strings.HasPrefix(c, "systemctl start ") && strings.Contains(c, ".service") {
505	t.Fatalf("apply started a service: %s", c)
506	}
507	}
508	if strings.Contains(joined, "systemctl enable --now") {
509	t.Fatal("enable --now is never used")
510	}
511	}
512
513	func TestApplyGateSoakCatchesAFlap(t *testing.T) {
514	f, req := applyHappy(t)
515	// Edge comes up, then restarts once during the soak.
516	restarts := 0
517	orig := f.a.ops.systemctl
518	f.a.ops.systemctl = func(args ...string) (string, error) {
519	if args[0] == "show" {
520	restarts++
521	if restarts > 1 {
522	return "1\n2000\n", nil
523	}
524	}
525	return orig(args...)
526	}
527	err := f.a.Run(context.Background(), false)
528	if err == nil {
529	t.Fatal("a flapping edge passed the gate")
530	}
531	r := f.result()
532	if r == nil || !r.matches("stable", "v2", req.PromotedAt) || r.Outcome == systemOutcomeApplied {
533	t.Fatalf("result = %+v", r)
534	}
535	}
536
537	func TestApplyPrevIsRefreshedOnlyWhenTheDiskIsKnownGood(t *testing.T) {
538	// applied.json names v0 but the disk says v1 (a rollback-failed left a
539	// bad v1 in place, .prev is the good v0): .prev must be left alone.
540	f, _ := applyHappy(t)
541	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v0\n"), 0o755)
542	f.answer["version:"+systemPrevPath(f.a.optDir)] = "carlos v0\n"
543	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "applied.json"), systemApplied{Version: "v0", Channel: "stable", At: f.now}, 0o644)
544	_ = f.a.Run(context.Background(), false)
545	prev, _ := os.ReadFile(systemPrevPath(f.a.optDir))
546	if !strings.Contains(string(prev), "carlos v0") {
547	t.Fatal(".prev was overwritten by a binary that never passed a gate")
548	}
549	}
550
551	func TestApplyPrevUnitsComeFromPrevNotFromDisk(t *testing.T) {
552	f, _ := applyHappy(t)
553	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v0\n"), 0o755)
554	f.answer["version:"+systemPrevPath(f.a.optDir)] = "carlos v0\n"
555	f.answer["plan:"+systemPrevPath(f.a.optDir)] = `{"enabled":["carlos-edge.service"],"inert":[]}`
556	f.answer["units:"+systemPrevPath(f.a.optDir)] = "carlos-edge.service carlos-v0-only.timer"
557	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "applied.json"), systemApplied{Version: "v0", Channel: "stable", At: f.now}, 0o644)
558	// Inspect the journal mid-flight by failing the gate.
559	f.edgeUp = false
560	_ = f.a.Run(context.Background(), false)
561	// units_removed must be computed against .prev's set (v0-only.timer),
562	// not the on-disk v1's (old-only.timer).
563	found := false
564	for _, c := range f.calls {
565	if c == "run carlos.prev bootstrap -root "+filepath.Join(f.a.stagingDir(), "prev")+" -plan-json" {
566	found = true
567	}
568	}
569	if !found {
570	t.Fatalf("prev units not staged from carlos.prev:\n%s", strings.Join(f.calls, "\n"))
571	}
572	// And units_removed follows from that set: v0's timer goes, and the
573	// on-disk v1's — a build nothing here is restoring — is left alone.
574	if !f.called("systemctl disable --now carlos-v0-only.timer") {
575	t.Errorf("carlos-v0-only.timer not removed:\n%s", strings.Join(f.calls, "\n"))
576	}
577	if f.called("systemctl disable --now carlos-old-only.timer") {
578	t.Errorf("removed a unit computed against the on-disk build:\n%s", strings.Join(f.calls, "\n"))
579	}
580	}
581
582	func TestApplyPrevIsRefreshedWhenAppliedNamesTheDisk(t *testing.T) {
583	// The ordinary box: applied.json names the binary that is on disk, so
584	// that binary passed a gate here and is the one to fall back to.
585	f, _ := applyHappy(t)
586	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v0\n"), 0o755)
587	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "applied.json"), systemApplied{Version: "v1", Channel: "stable", At: f.now}, 0o644)
588	if err := f.a.Run(context.Background(), false); err != nil {
589	t.Fatal(err)
590	}
591	prev, _ := os.ReadFile(systemPrevPath(f.a.optDir))
592	if !strings.Contains(string(prev), "carlos v1") {
593	t.Fatalf(".prev = %q, want the v1 the box had applied", prev)
594	}
595	}
596
597	func TestApplyPrevIsLeftAloneWhenNothingWasEverApplied(t *testing.T) {
598	// No applied.json, but a .prev exists: the box's history is unknown, so
599	// the untested disk binary does not get to become the fallback.
600	f, _ := applyHappy(t)
601	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v0\n"), 0o755)
602	f.answer["version:"+systemPrevPath(f.a.optDir)] = "carlos v0\n"
603	if err := f.a.Run(context.Background(), false); err != nil {
604	t.Fatal(err)
605	}
606	prev, _ := os.ReadFile(systemPrevPath(f.a.optDir))
607	if !strings.Contains(string(prev), "carlos v0") {
608	t.Fatalf(".prev = %q, want the v0 that was already there", prev)
609	}
610	}
611
612	func TestApplyGateDemandsZeroRestartsNotJustAStillOne(t *testing.T) {
613	// The edge crashed once and then settled: nothing moves across the
614	// soak, but NRestarts is not 0 and the build is not one to keep.
615	f, req := applyHappy(t)
616	f.nrestarts = "1"
617	if err := f.a.Run(context.Background(), false); err == nil {
618	t.Fatal("an edge that had already crash-restarted passed the gate")
619	}
620	r := f.result()
621	if r == nil || !r.matches("stable", "v2", req.PromotedAt) || r.Outcome == systemOutcomeApplied {
622	t.Fatalf("result = %+v", r)
623	}
624	}
625
626	func TestApplyRecordsAnArmingFailureAgainstTheApply(t *testing.T) {
627	// The gate passed, so the roll stands; but `enable` is one batched
628	// call, and a box left unarmed must not look like a clean apply.
629	f, req := applyHappy(t)
630	orig := f.a.ops.systemctl
631	f.a.ops.systemctl = func(args ...string) (string, error) {
632	if args[0] == "enable" {
633	orig(args...)
634	return "", errors.New("Failed to enable unit carlos-new-only.path")
635	}
636	return orig(args...)
637	}
638	if err := f.a.Run(context.Background(), false); err != nil {
639	t.Fatal(err)
640	}
641	r := f.result()
642	if r == nil || r.Outcome != systemOutcomeApplied || !r.matches("stable", "v2", req.PromotedAt) {
643	t.Fatalf("result = %+v", r)
644	}
645	if !strings.Contains(r.Reason, "armed with errors") {
646	t.Fatalf("reason = %q, want the arming failure recorded", r.Reason)
647	}
648	if ap, _ := readSystemApplied(f.a.optDir); ap == nil || ap.Version != "v2" {
649	t.Fatalf("applied = %+v (the gate passed; the roll stands)", ap)
650	}
651	}
652
653	func TestApplyRollsBackWhenTheGateFails(t *testing.T) {
654	f, req := applyHappy(t)
655	// v2 never comes up; v1 does — which is what makes this a rollback and
656	// not a rollback-failed. The fake edge answers for whatever binary is
657	// on disk when the restart is issued.
658	f.edgeUp = false
659	orig := f.a.ops.systemctl
660	f.a.ops.systemctl = func(args ...string) (string, error) {
661	if args[0] == "restart" {
662	if b, _ := os.ReadFile(systemBinaryPath(f.a.optDir)); strings.Contains(string(b), "carlos v1") {
663	f.edgeUp = true
664	}
665	}
666	return orig(args...)
667	}
668	err := f.a.Run(context.Background(), false)
669	if err == nil {
670	t.Fatal("a failed gate must exit non-zero")
671	}
672	r := f.result()
673	if r == nil || r.Outcome != systemOutcomeRolledBack || !r.matches("stable", "v2", req.PromotedAt) || !strings.Contains(r.Reason, "gate") {
674	t.Fatalf("result = %+v", r)
675	}
676	bin, _ := os.ReadFile(systemBinaryPath(f.a.optDir))
677	if !strings.Contains(string(bin), "carlos v1") {
678	t.Fatal("binary not restored")
679	}
680	if _, err := os.Stat(systemPrevPath(f.a.optDir)); err != nil {
681	t.Fatal(".prev consumed by the rollback")
682	}
683	units := filepath.Join(f.a.etcDir, "systemd", "system")
684	if _, err := os.Stat(filepath.Join(units, "carlos-new-only.path")); err == nil {
685	t.Error("added unit survived the rollback")
686	}
687	if _, err := os.Stat(filepath.Join(units, "carlos-old-only.timer")); err != nil {
688	t.Error("removed unit not restored")
689	}
690	if ap, _ := readSystemApplied(f.a.optDir); ap != nil {
691	t.Fatalf("applied.json written by a rollback: %+v", ap)
692	}
693	if j, _ := readSystemJournal(f.a.optDir); j != nil {
694	t.Fatal("journal left open after rollback")
695	}
696	joined := strings.Join(f.calls, "\n")
697	if !strings.Contains(joined, "systemctl disable --now carlos-new-only.path") {
698	t.Error("added unit not disabled")
699	}
700	// The install already staged carlos.prev; the rollback reuses it rather
701	// than re-running a bootstrap on a box that is already unhealthy.
702	if n := strings.Count(joined, "run carlos.prev bootstrap"); n != 1 {
703	t.Errorf("carlos.prev bootstrapped %d times, want 1:\n%s", n, joined)
704	}
705	// reset-failed precedes the rollback's restart too, and the previous
706	// plan is armed afterwards.
707	i := strings.LastIndex(joined, "systemctl reset-failed carlos-edge")
708	k := strings.LastIndex(joined, "systemctl restart carlos-edge")
709	m := strings.LastIndex(joined, "systemctl start carlos-adopt.timer")
710	if !(i < k && k < m) {
711	t.Fatalf("rollback order wrong:\n%s", joined)
712	}
713	}
714
715	// The relay rides the platform binary, so a self-update has to bounce it
716	// onto the new build the way it bounces the edge — after the edge, and with
717	// reset-failed then restart. `restart`, not `try-restart`: an unarmed box is
718	// kept relay-less by the unit's own ConditionPathExists (bootstrap starts
719	// the unit outright and relies on the same thing), while try-restart after a
720	// reset-failed cannot revive a relay that crash-looped at boot, because
721	// reset-failed has just made it inactive and try-restart skips an inactive
722	// unit (relay spec §3).
723	func TestApplyRestartsTheRelayAfterTheEdge(t *testing.T) {
724	f, _ := applyHappy(t)
725	if err := f.a.Run(context.Background(), false); err != nil {
726	t.Fatal(err)
727	}
728	joined := strings.Join(f.calls, "\n")
729	edge := strings.Index(joined, "systemctl restart carlos-edge")
730	reset := strings.Index(joined, "systemctl reset-failed carlos-relay")
731	restart := strings.Index(joined, "systemctl restart carlos-relay")
732	switch {
733	case reset < 0 || restart < 0:
734	t.Fatalf("relay never restarted:\n%s", joined)
735	case !(edge < reset && reset < restart):
736	t.Fatalf("want restart carlos-edge, then reset-failed then restart carlos-relay:\n%s", joined)
737	}
738	// try-restart is the bug this test now guards against: it would leave a
739	// failed relay down until the next update.
740	if strings.Contains(joined, "systemctl try-restart carlos-relay") {
741	t.Errorf("relay try-restarted, which cannot revive a failed unit:\n%s", joined)
742	}
743	}
744
745	// Discussion #256: the root instance-control service execs the platform
746	// binary and stays up for weeks, so an install that leaves it running the
747	// previous build leaves every nested wake failing at namespace setup. It
748	// is restarted after the new units are in place (daemon-reload) and before
749	// the edge, so the new agent never meets an old control service.
750	func TestApplyRestartsTheInstanceControlServiceBeforeTheEdge(t *testing.T) {
751	f, _ := applyHappy(t)
752	if err := f.a.Run(context.Background(), false); err != nil {
753	t.Fatal(err)
754	}
755	joined := strings.Join(f.calls, "\n")
756	reload := strings.Index(joined, "systemctl daemon-reload")
757	reset := strings.Index(joined, "systemctl reset-failed carlos-instance-control.service")
758	restart := strings.Index(joined, "systemctl restart carlos-instance-control.service")
759	edge := strings.Index(joined, "systemctl restart carlos-edge")
760	switch {
761	case reset < 0 || restart < 0:
762	t.Fatalf("instance-control never restarted:\n%s", joined)
763	case !(reload < reset && reset < restart && restart < edge):
764	t.Fatalf("want daemon-reload, then reset-failed and restart carlos-instance-control.service, then restart carlos-edge:\n%s", joined)
765	}
766	if strings.Contains(joined, "systemctl try-restart carlos-instance-control.service") {
767	t.Errorf("try-restart cannot start an inactive (socket-activated, never connected) service:\n%s", joined)
768	}
769	}
770
771	// A control service that will not restart fails the install, and the
772	// rollback restarts it again onto the previous binary (so the restart is
773	// attempted twice over the run), rather than leaving a new edge beside a
774	// control service of unknown vintage.
775	func TestApplyInstanceControlRestartFailureRollsBack(t *testing.T) {
776	f, req := applyHappy(t)
777	orig := f.a.ops.systemctl
778	failed := false
779	f.a.ops.systemctl = func(args ...string) (string, error) {
780	out, err := orig(args...)
781	if args[0] == "restart" && args[1] == systemControlUnit && !failed {
782	failed = true
783	return "", errors.New("Job for carlos-instance-control.service failed")
784	}
785	return out, err
786	}
787	if err := f.a.Run(context.Background(), false); err == nil {
788	t.Fatal("a control service that would not restart passed the install")
789	}
790	r := f.result()
791	if r == nil || r.Outcome != systemOutcomeRolledBack || !r.matches("stable", "v2", req.PromotedAt) || !strings.Contains(r.Reason, "instance-control restart") {
792	t.Fatalf("result = %+v", r)
793	}
794	if n := strings.Count(strings.Join(f.calls, "\n"), "systemctl restart carlos-instance-control.service"); n != 2 {
795	t.Errorf("control service restarted %d times, want 2 (install then rollback):\n%s", n, strings.Join(f.calls, "\n"))
796	}
797	if bin, _ := os.ReadFile(systemBinaryPath(f.a.optDir)); !strings.Contains(string(bin), "carlos v1") {
798	t.Fatal("binary not rolled back")
799	}
800	}
801
802	// The rollback of a failed gate restarts the control service onto the
803	// previous binary too, before the edge, for the same reason the relay is.
804	func TestApplyRollbackRestartsTheInstanceControlService(t *testing.T) {
805	f, _ := applyHappy(t)
806	f.edgeUp = false
807	orig := f.a.ops.systemctl
808	f.a.ops.systemctl = func(args ...string) (string, error) {
809	if args[0] == "restart" && args[1] == systemEdgeUnit {
810	if b, _ := os.ReadFile(systemBinaryPath(f.a.optDir)); strings.Contains(string(b), "carlos v1") {
811	f.edgeUp = true
812	}
813	}
814	return orig(args...)
815	}
816	if err := f.a.Run(context.Background(), false); err == nil {
817	t.Fatal("a failed gate must exit non-zero")
818	}
819	joined := strings.Join(f.calls, "\n")
820	edge := strings.LastIndex(joined, "systemctl restart carlos-edge")
821	ctl := strings.LastIndex(joined, "systemctl restart carlos-instance-control.service")
822	if ctl < 0 || ctl > edge {
823	t.Fatalf("the rollback left the control service on the withdrawn build, or restarted it after the edge:\n%s", joined)
824	}
825	if n := strings.Count(joined, "systemctl restart carlos-instance-control.service"); n != 2 {
826	t.Errorf("control service restarted %d times, want 2 (apply then rollback):\n%s", n, joined)
827	}
828	}
829
830	// A candidate that ships no control service (a downgrade below the
831	// isolated-exec releases) has nothing to restart: installUnits has just
832	// disabled and removed the unit, and a restart would fail the install.
833	func TestApplyDoesNotRestartAControlServiceTheCandidateLacks(t *testing.T) {
834	f, req := applyHappy(t)
835	staged := filepath.Join(f.a.stagingDir(), "carlos")
836	f.answer["units:"+staged] = strings.ReplaceAll(f.answer["units:"+staged], " carlos-instance-control.service", "")
837	if err := f.a.Run(context.Background(), false); err != nil {
838	t.Fatal(err)
839	}
840	if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied || !r.matches("stable", "v2", req.PromotedAt) {
841	t.Fatalf("result = %+v", r)
842	}
843	joined := strings.Join(f.calls, "\n")
844	if !strings.Contains(joined, "systemctl disable --now carlos-instance-control.service") {
845	t.Fatalf("the unit the candidate lacks was not disabled:\n%s", joined)
846	}
847	if strings.Contains(joined, "systemctl restart carlos-instance-control.service") {
848	t.Fatalf("restarted a unit the candidate removed:\n%s", joined)
849	}
850	}
851
852	// A relay that cannot be restarted is not a failed update. The commonest
853	// case is the ordinary one — a box with no masters file, whose unit systemd
854	// skips on its condition and may well answer about — and the edge, the
855	// binary and the gate are all fine.
856	func TestApplyRelayRestartFailureDoesNotFailTheUpdate(t *testing.T) {
857	f, req := applyHappy(t)
858	orig := f.a.ops.systemctl
859	f.a.ops.systemctl = func(args ...string) (string, error) {
860	if args[0] == "restart" && args[1] == systemRelayUnit {
861	orig(args...) // still recorded
862	return "", errors.New("Unit carlos-relay.service is not loaded")
863	}
864	return orig(args...)
865	}
866	if err := f.a.Run(context.Background(), false); err != nil {
867	t.Fatalf("a relay that would not restart failed the whole update: %v", err)
868	}
869	r := f.result()
870	if r == nil || r.Outcome != systemOutcomeApplied || !r.matches("stable", "v2", req.PromotedAt) {
871	t.Fatalf("result = %+v", r)
872	}
873	if !f.called("systemctl restart " + systemRelayUnit) {
874	t.Fatalf("restart never attempted:\n%s", strings.Join(f.calls, "\n"))
875	}
876	}
877
878	// And the rollback bounces it too: the box is going back to the previous
879	// binary, and leaving the relay on the withdrawn one is the same bug the
880	// edge's own rollback restart exists to avoid.
881	func TestApplyRollbackRestartsTheRelay(t *testing.T) {
882	f, _ := applyHappy(t)
883	f.edgeUp = false
884	orig := f.a.ops.systemctl
885	f.a.ops.systemctl = func(args ...string) (string, error) {
886	if args[0] == "restart" {
887	if b, _ := os.ReadFile(systemBinaryPath(f.a.optDir)); strings.Contains(string(b), "carlos v1") {
888	f.edgeUp = true
889	}
890	}
891	return orig(args...)
892	}
893	if err := f.a.Run(context.Background(), false); err == nil {
894	t.Fatal("a failed gate must exit non-zero")
895	}
896	joined := strings.Join(f.calls, "\n")
897	edge := strings.LastIndex(joined, "systemctl restart carlos-edge")
898	relay := strings.LastIndex(joined, "systemctl restart carlos-relay")
899	if relay < 0 || relay < edge {
900	t.Fatalf("the rollback left the relay on the withdrawn build:\n%s", joined)
901	}
902	// Twice over the whole run: once for the apply's restart, once for the
903	// rollback's — not just the apply's, which would pass a LastIndex check
904	// by accident if the rollback did nothing.
905	if n := strings.Count(joined, "systemctl restart carlos-relay"); n != 2 {
906	t.Errorf("relay restarted %d times, want 2 (apply then rollback):\n%s", n, joined)
907	}
908	}
909
910	func TestApplyRollbackFailedLeavesPrevAndAppliedAlone(t *testing.T) {
911	t.Parallel()
912	f, req := applyHappy(t)
913	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "applied.json"), systemApplied{Version: "v1", Channel: "stable", At: f.now}, 0o644)
914	f.edgeUp = false // never comes up, on either binary
915	if err := f.a.Run(context.Background(), false); err == nil {
916	t.Fatal("must exit non-zero")
917	}
918	r := f.result()
919	if r == nil || r.Outcome != systemOutcomeRollbackFailed || !r.matches("stable", "v2", req.PromotedAt) {
920	t.Fatalf("result = %+v", r)
921	}
922	ap, _ := readSystemApplied(f.a.optDir)
923	if ap == nil || ap.Version != "v1" {
924	t.Fatalf("applied = %+v, must still name v1", ap)
925	}
926	prev, _ := os.ReadFile(systemPrevPath(f.a.optDir))
927	if !strings.Contains(string(prev), "carlos v1") {
928	t.Fatal(".prev damaged")
929	}
930	// A second apply of a NEW version must not refresh .prev from the
931	// unproven disk binary. The box a failed rollback leaves behind is the
932	// one that makes that matter: applied.json still names v1, and the v2
933	// that never passed a gate is what is on disk.
934	os.WriteFile(systemBinaryPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v2\n"), 0o755)
935	f.answer["version:"+systemBinaryPath(f.a.optDir)] = "carlos v2\n"
936	os.Remove(systemRequestPath(f.a.dataDir))
937	f.promote("v3", []byte("#!/bin/sh\necho carlos v3\n"), false)
938	f.answer["version:"+filepath.Join(f.a.stagingDir(), "carlos")] = "carlos v3\n"
939	f.answer["plan:"+filepath.Join(f.a.stagingDir(), "carlos")] = f.answer["plan:"+systemPrevPath(f.a.optDir)]
940	f.answer["units:"+filepath.Join(f.a.stagingDir(), "carlos")] = f.answer["units:"+systemPrevPath(f.a.optDir)]
941	_ = f.a.Run(context.Background(), false)
942	prev2, _ := os.ReadFile(systemPrevPath(f.a.optDir))
943	if string(prev2) != string(prev) {
944	t.Fatal(".prev refreshed after rollback-failed")
945	}
946	}
947
948	func TestRecoveryRollsBackFilesOnlyAtBootAndRestartsAtRuntime(t *testing.T) {
949	for _, phase := range []string{systemPhaseInstalling, systemPhaseInstalled, systemPhaseUnits, systemPhaseRestarted} {
950	t.Run(phase, func(t *testing.T) {
951	f, _ := applyHappy(t)
952	// Simulate: v2 installed, journal open at phase, box rebooted.
953	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v1\n"), 0o755)
954	os.WriteFile(systemBinaryPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v2\n"), 0o755)
955	os.WriteFile(filepath.Join(f.a.etcDir, "systemd", "system", "carlos-new-only.path"), []byte("x"), 0o644)
956	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
957	systemJournal{Phase: phase, Target: "v2", PrevBuild: "v1", UnitsAdded: []string{"carlos-new-only.path"}, UnitsRemoved: []string{"carlos-old-only.timer"}, StartedAt: f.now}, 0o600)
958	f.edgeStartedThisBoot = false
959	if err := f.a.Run(context.Background(), true); err != nil {
960	t.Fatal(err)
961	}
962	bin, _ := os.ReadFile(systemBinaryPath(f.a.optDir))
963	if !strings.Contains(string(bin), "carlos v1") {
964	t.Fatal("binary not restored")
965	}
966	if _, err := os.Stat(filepath.Join(f.a.etcDir, "systemd", "system", "carlos-new-only.path")); err == nil {
967	t.Error("added unit survived")
968	}
969	if _, err := os.Stat(filepath.Join(f.a.etcDir, "systemd", "system", "carlos-old-only.timer")); err != nil {
970	t.Error("removed unit not restored")
971	}
972	joined := strings.Join(f.calls, "\n")
973	if strings.Contains(joined, "systemctl restart carlos-edge") || strings.Contains(joined, "systemctl start ") || strings.Contains(joined, "systemctl restart carlos-instance-control") {
974	t.Fatalf("boot recovery restarted or started something:\n%s", joined)
975	}
976	if !strings.Contains(joined, "systemctl enable carlos-edge.service") {
977	t.Fatal("previous plan not enabled for the boot")
978	}
979	if r := f.result(); r == nil || r.Outcome != systemOutcomeRolledBack || !strings.Contains(r.Reason, phase) {
980	t.Fatalf("result = %+v", r)
981	}
982	if j, _ := readSystemJournal(f.a.optDir); j != nil {
983	t.Fatal("journal left open")
984	}
985	})
986	}
987	// Runtime (OnFailure=): same, plus a non-blocking restart and the
988	// previous plan's timers/paths started.
989	f, _ := applyHappy(t)
990	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v1\n"), 0o755)
991	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
992	systemJournal{Phase: systemPhaseUnits, Target: "v2", PrevBuild: "v1", StartedAt: f.now}, 0o600)
993	f.edgeStartedThisBoot = true
994	if err := f.a.Run(context.Background(), true); err != nil {
995	t.Fatal(err)
996	}
997	joined := strings.Join(f.calls, "\n")
998	for _, want := range []string{"systemctl stop carlos-adopt.timer", "systemctl reset-failed carlos-edge", "systemctl restart --no-block carlos-edge", "systemctl start carlos-adopt.timer"} {
999	if !strings.Contains(joined, want) {
1000	t.Errorf("runtime recovery lacks %q:\n%s", want, joined)
1001	}
1002	}
1003	// The control service goes back to the previous binary with the edge
1004	// (#256), and before it; at boot (above) neither is touched.
1005	ctl := strings.Index(joined, "systemctl restart carlos-instance-control.service")
1006	edge := strings.Index(joined, "systemctl restart --no-block carlos-edge")
1007	if ctl < 0 || ctl > edge {
1008	t.Errorf("runtime recovery does not restart the control service before the edge:\n%s", joined)
1009	}
1010	}
1011
1012	func TestRecoveryFinishesForwardFromGated(t *testing.T) {
1013	f, req := applyHappy(t)
1014	os.WriteFile(systemBinaryPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v2\n"), 0o755)
1015	writeJSONDurable(systemRequestPath(f.a.dataDir), req, 0o644)
1016	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
1017	systemJournal{Phase: systemPhaseGated, Target: "v2", PrevBuild: "v1", StartedAt: f.now}, 0o600)
1018	if err := f.a.Run(context.Background(), true); err != nil {
1019	t.Fatal(err)
1020	}
1021	bin, _ := os.ReadFile(systemBinaryPath(f.a.optDir))
1022	if !strings.Contains(string(bin), "carlos v2") {
1023	t.Fatal("a gated apply was undone")
1024	}
1025	ap, _ := readSystemApplied(f.a.optDir)
1026	if ap == nil || ap.Version != "v2" {
1027	t.Fatalf("applied = %+v", ap)
1028	}
1029	if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied {
1030	t.Fatalf("result = %+v", r)
1031	}
1032	}
1033
1034	func TestRecoveryRemovesTheJournalLast(t *testing.T) {
1035	// The restore is done and the result is the last thing before the
1036	// journal goes: a restart that fails after the files are back leaves
1037	// both the journal and no result at all for the next run to redo.
1038	f, _ := applyHappy(t)
1039	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v1\n"), 0o755)
1040	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
1041	systemJournal{Phase: systemPhaseInstalled, Target: "v2", PrevBuild: "v1", StartedAt: f.now}, 0o600)
1042	f.edgeStartedThisBoot = true
1043	orig := f.a.ops.systemctl
1044	f.a.ops.systemctl = func(args ...string) (string, error) {
1045	if args[0] == "restart" {
1046	return "", errors.New("job for carlos-edge.service failed")
1047	}
1048	return orig(args...)
1049	}
1050	if err := f.a.Run(context.Background(), true); err == nil {
1051	t.Fatal("a failed recovery must exit non-zero")
1052	}
1053	if j, _ := readSystemJournal(f.a.optDir); j == nil {
1054	t.Fatal("journal removed although recovery did not complete")
1055	}
1056	if r := f.result(); r != nil {
1057	t.Fatalf("result written before the restart was issued: %+v", r)
1058	}
1059	bin, _ := os.ReadFile(systemBinaryPath(f.a.optDir))
1060	if !strings.Contains(string(bin), "carlos v1") {
1061	t.Fatal("the restore itself did not happen")
1062	}
1063	}
1064
1065	func TestRecoveryKeepsTheJournalWhenTheRestoreFails(t *testing.T) {
1066	// systemd will not reload, so the units are not back: the journal stays
1067	// open and the next run recovers again.
1068	f, _ := applyHappy(t)
1069	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v1\n"), 0o755)
1070	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
1071	systemJournal{Phase: systemPhaseInstalled, Target: "v2", PrevBuild: "v1", StartedAt: f.now}, 0o600)
1072	orig := f.a.ops.systemctl
1073	f.a.ops.systemctl = func(args ...string) (string, error) {
1074	if args[0] == "daemon-reload" {
1075	return "", errors.New("systemd busy")
1076	}
1077	return orig(args...)
1078	}
1079	if err := f.a.Run(context.Background(), true); err == nil {
1080	t.Fatal("a failed recovery must exit non-zero")
1081	}
1082	if j, _ := readSystemJournal(f.a.optDir); j == nil {
1083	t.Fatal("journal removed although recovery did not complete")
1084	}
1085	}
1086
1087	func TestRecoveryAnswersTheJournalsVersionNotALaterRequest(t *testing.T) {
1088	// The agent has moved on to v3 while a v2 apply lay interrupted. The
1089	// result belongs to v2 — v3 has not been tried at all.
1090	f, _ := applyHappy(t)
1091	started := f.now
1092	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v1\n"), 0o755)
1093	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
1094	systemJournal{Phase: systemPhaseUnits, Target: "v2", PrevBuild: "v1", StartedAt: started}, 0o600)
1095	v3 := f.promote("v3", []byte("#!/bin/sh\necho carlos v3\n"), false)
1096	if err := f.a.Run(context.Background(), true); err != nil {
1097	t.Fatal(err)
1098	}
1099	r := f.result()
1100	if r == nil || r.Version != "v2" || !r.PromotedAt.Equal(started) {
1101	t.Fatalf("result = %+v, want the v2 the journal names at %s", r, started)
1102	}
1103	if r.PromotedAt.Equal(v3.PromotedAt) {
1104	t.Fatal("the recovery answered for the request the agent wrote later")
1105	}
1106	}
1107
1108	func TestRecoveryAnswersTheInterruptedPromoteNotTheRePromote(t *testing.T) {
1109	t.Parallel()
1110	// v2 was interrupted; the operator re-promoted the same v2. The
1111	// recovery's result must name the promote it recovered, or the retry
1112	// arrives at a box that has already recorded it as rolled back.
1113	f, first := applyHappy(t)
1114	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v1\n"), 0o755)
1115	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
1116	systemJournal{Phase: systemPhaseUnits, Target: "v2", PromotedAt: first.PromotedAt, PrevBuild: "v1", StartedAt: f.now}, 0o600)
1117	time.Sleep(1100 * time.Millisecond) // promoted_at has second resolution
1118	again := f.promote("v2", []byte("#!/bin/sh\necho carlos v2\n"), false)
1119	if again.PromotedAt.Equal(first.PromotedAt) {
1120	t.Fatal("fixture: the re-promote must move promoted_at")
1121	}
1122	if err := f.a.Run(context.Background(), true); err != nil {
1123	t.Fatal(err)
1124	}
1125	r := f.result()
1126	if r == nil || r.Outcome != systemOutcomeRolledBack || !r.matches("stable", "v2", first.PromotedAt) {
1127	t.Fatalf("result = %+v, want the promote the journal names (%s)", r, first.PromotedAt)
1128	}
1129	// And the re-promote is a tuple of its own: the terminal-tuple guard
1130	// does not refuse it.
1131	if err := f.a.Run(context.Background(), false); err != nil {
1132	t.Fatal(err)
1133	}
1134	if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied || !r.matches("stable", "v2", again.PromotedAt) {
1135	t.Fatalf("result = %+v, the operator's re-promote must be applied", r)
1136	}
1137	}
1138
1139	func TestRecoveryFromGatedRollsBackWhenTheBinaryIsNotTheGatedOne(t *testing.T) {
1140	// The journal says v2 passed its gate, but v1 is on disk: whatever put
1141	// it there, this is a pre-gate state and going forward would record an
1142	// apply of a build that is not installed.
1143	f, _ := applyHappy(t)
1144	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v1\n"), 0o755)
1145	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
1146	systemJournal{Phase: systemPhaseGated, Target: "v2", PrevBuild: "v1", StartedAt: f.now}, 0o600)
1147	if err := f.a.Run(context.Background(), true); err != nil {
1148	t.Fatal(err)
1149	}
1150	if ap, _ := readSystemApplied(f.a.optDir); ap != nil {
1151	t.Fatalf("applied.json written for a build that is not on disk: %+v", ap)
1152	}
1153	if r := f.result(); r == nil || r.Outcome != systemOutcomeRolledBack {
1154	t.Fatalf("result = %+v", r)
1155	}
1156	}
1157
1158	func TestRollbackThatCannotRestoreKeepsTheJournalForTheNextRun(t *testing.T) {
1159	f, _ := applyHappy(t)
1160	f.edgeUp = false
1161	prevDir := filepath.Join(f.a.stagingDir(), "prev")
1162	prevPlan := f.answer["plan:"+systemPrevPath(f.a.optDir)]
1163	stops := 0
1164	orig := f.a.ops.systemctl
1165	f.a.ops.systemctl = func(args ...string) (string, error) {
1166	// The rollback's own quiesce: by then the staging is gone and
1167	// carlos.prev will not bootstrap, so the restore stops after the
1168	// binary and before the units.
1169	if args[0] == "stop" {
1170	if stops++; stops == 2 {
1171	os.RemoveAll(prevDir)
1172	delete(f.answer, "plan:"+systemPrevPath(f.a.optDir))
1173	}
1174	}
1175	return orig(args...)
1176	}
1177	if err := f.a.Run(context.Background(), false); err == nil {
1178	t.Fatal("a failed rollback must exit non-zero")
1179	}
1180	if r := f.result(); r == nil || r.Outcome != systemOutcomeRollbackFailed {
1181	t.Fatalf("result = %+v", r)
1182	}
1183	if j, _ := readSystemJournal(f.a.optDir); j == nil {
1184	t.Fatal("journal removed although the restore never finished")
1185	}
1186	units := filepath.Join(f.a.etcDir, "systemd", "system")
1187	if _, err := os.Stat(filepath.Join(units, "carlos-new-only.path")); err != nil {
1188	t.Fatal("precondition: the failed build's units are what is still installed")
1189	}
1190	// The next run recovers what the rollback could not: the bootstrap
1191	// works again, and the previous build's units go back.
1192	f.answer["plan:"+systemPrevPath(f.a.optDir)] = prevPlan
1193	if err := f.a.Run(context.Background(), true); err != nil {
1194	t.Fatal(err)
1195	}
1196	if _, err := os.Stat(filepath.Join(units, "carlos-new-only.path")); err == nil {
1197	t.Error("added unit survived the recovery")
1198	}
1199	if _, err := os.Stat(filepath.Join(units, "carlos-old-only.timer")); err != nil {
1200	t.Error("removed unit not restored by the recovery")
1201	}
1202	if j, _ := readSystemJournal(f.a.optDir); j != nil {
1203	t.Error("journal left open after a complete recovery")
1204	}
1205	}
1206
1207	func TestARecoveredRunDoesNotThenApplyTheRequest(t *testing.T) {
1208	// The v2 request is still on disk, and the recovery has just rolled v2
1209	// back. Applying it again in the same run would overwrite the result
1210	// the agent is waiting for and restart the edge a second time.
1211	f, _ := applyHappy(t)
1212	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v1\n"), 0o755)
1213	os.WriteFile(systemBinaryPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v2\n"), 0o755)
1214	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
1215	systemJournal{Phase: systemPhaseUnits, Target: "v2", PrevBuild: "v1", StartedAt: f.now}, 0o600)
1216	if err := f.a.Run(context.Background(), false); err != nil {
1217	t.Fatal(err)
1218	}
1219	if r := f.result(); r == nil || r.Outcome != systemOutcomeRolledBack {
1220	t.Fatalf("result = %+v", r)
1221	}
1222	if ap, _ := readSystemApplied(f.a.optDir); ap != nil {
1223	t.Fatalf("the request was applied on top of the recovery: %+v", ap)
1224	}
1225	bin, _ := os.ReadFile(systemBinaryPath(f.a.optDir))
1226	if !strings.Contains(string(bin), "carlos v1") {
1227	t.Fatal("the binary is not the one the recovery restored")
1228	}
1229	}
1230
1231	func TestARolledBackTupleIsNotAppliedAgain(t *testing.T) {
1232	f, req := applyHappy(t)
1233	results := filepath.Join(systemRootDir(f.a.optDir), "result.json")
1234	writeJSONDurable(results, systemResult{Channel: "stable", Version: req.Version, PromotedAt: req.PromotedAt,
1235	Outcome: systemOutcomeRolledBack, Reason: "gate: carlos-edge is failed", At: f.now}, 0o644)
1236	if err := f.a.Run(context.Background(), false); err != nil {
1237	t.Fatal(err)
1238	}
1239	if len(f.calls) != 0 {
1240	t.Fatalf("a build that already rolled back here was applied again: %v", f.calls)
1241	}
1242	if r := f.result(); r == nil || r.Outcome != systemOutcomeRolledBack {
1243	t.Fatalf("result = %+v, the standing outcome must be left alone", r)
1244	}
1245	// A refusal is not terminal: the same tuple is tried again.
1246	writeJSONDurable(results, systemResult{Channel: "stable", Version: req.Version, PromotedAt: req.PromotedAt,
1247	Outcome: systemOutcomeRefused, Reason: "seen.json unreadable", At: f.now}, 0o644)
1248	if err := f.a.Run(context.Background(), false); err != nil {
1249	t.Fatal(err)
1250	}
1251	if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied {
1252	t.Fatalf("result = %+v, a refused tuple must stay retryable", r)
1253	}
1254	}
1255
1256	// --recover runs at boot, before the network is up and on a box whose
1257	// environment may be no more than /etc/carlos/host.env: it must never need
1258	// a deploy store or CARLOS_RELEASE_PUBKEY. The seam says so out loud —
1259	// opening the store at all fails the test.
1260	func TestRecoverNeedsNeitherStoreNorPubkey(t *testing.T) {
1261	f, _ := applyHappy(t)
1262	f.a.openStore = func(context.Context) (release.Store, ed25519.PublicKey, error) {
1263	t.Error("--recover opened the deploy store")
1264	return nil, nil, errors.New("no store on this box")
1265	}
1266	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v1\n"), 0o755)
1267	os.WriteFile(systemBinaryPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v2\n"), 0o755)
1268	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
1269	systemJournal{Phase: systemPhaseUnits, Target: "v2", PrevBuild: "v1", StartedAt: f.now}, 0o600)
1270	if err := f.a.Run(context.Background(), true); err != nil {
1271	t.Fatal(err)
1272	}
1273	bin, _ := os.ReadFile(systemBinaryPath(f.a.optDir))
1274	if !strings.Contains(string(bin), "carlos v1") {
1275	t.Fatalf("binary = %q, want the .prev restored", bin)
1276	}
1277	// And a --recover with nothing to recover — the every-boot case — is
1278	// just as storeless.
1279	f2, _ := applyHappy(t)
1280	f2.a.openStore = func(context.Context) (release.Store, ed25519.PublicKey, error) {
1281	t.Error("a no-op --recover opened the deploy store")
1282	return nil, nil, errors.New("no store on this box")
1283	}
1284	if err := f2.a.Run(context.Background(), true); err != nil {
1285	t.Fatal(err)
1286	}
1287	if f2.result() != nil {
1288	t.Fatalf("a --recover with no journal wrote a result: %+v", f2.result())
1289	}
1290	}
1291
1292	// --recover waits for the apply lock, but not forever. carlos-system-recover
1293	// is ordered Before=carlos-edge.service, so a recover start job queued while
1294	// an apply is mid-`systemctl restart carlos-edge` has systemd holding the
1295	// restart for the recovery — and a recovery blocking on the lock would hold
1296	// it for the apply. Both would sit there until their unit timeouts. It waits
1297	// half a minute and stands down; the apply recovers at its own start.
1298	func TestRecoverGivesUpOnALockAnApplyHolds(t *testing.T) {
1299	f, _ := applyHappy(t)
1300	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
1301	systemJournal{Phase: systemPhaseUnits, Target: "v2", PrevBuild: "v1", StartedAt: f.now}, 0o600)
1302	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v1\n"), 0o755)
1303	held, err := os.OpenFile(f.a.lockPath(), os.O_CREATE|os.O_RDWR, 0o600)
1304	if err != nil {
1305	t.Fatal(err)
1306	}
1307	defer held.Close()
1308	if err := flockExclusiveNonblock(held); err != nil {
1309	t.Skipf("flock unsupported on this platform: %v", err)
1310	}
1311	var logged []string
1312	f.a.ops.logf = func(format string, args ...any) { logged = append(logged, format) }
1313	start := f.now
1314	if err := f.a.Run(context.Background(), true); err != nil {
1315	t.Fatalf("a locked-out --recover must exit 0: %v", err)
1316	}
1317	if waited := f.now.Sub(start); waited != systemRecoverLockWait {
1318	t.Errorf("waited %s, want %s", waited, systemRecoverLockWait)
1319	}
1320	if len(logged) != 1 || !strings.Contains(logged[0], "it recovers at its own start") {
1321	t.Errorf("log = %v", logged)
1322	}
1323	if j, _ := readSystemJournal(f.a.optDir); j == nil {
1324	t.Error("a locked-out --recover closed the journal it never read")
1325	}
1326	if len(f.calls) != 0 {
1327	t.Errorf("a locked-out --recover touched the box: %v", f.calls)
1328	}
1329	// The lock is free again: the next run gets it and does the work.
1330	held.Close()
1331	if err := f.a.Run(context.Background(), true); err != nil {
1332	t.Fatal(err)
1333	}
1334	if j, _ := readSystemJournal(f.a.optDir); j != nil {
1335	t.Error("the recovery that got the lock left the journal open")
1336	}
1337	}
1338
1339	// The carlos.prev backup is NOT journaled, though everything else that
1340	// touches the box is. copyFileDurable is whole-or-nothing, so there is
1341	// nothing to recover from — and a journal here is actively harmful: the
1342	// exec shim in carlos-system-recover.service reads it at the next boot and
1343	// would run a .prev one release behind the box for a whole apply.
1344	//
1345	// A sentinel journal makes the absence visible: decidePrev neither writes
1346	// over it nor removes it. (Nothing puts a journal there in real life —
1347	// recoverIfNeeded has closed any open one long before install runs.)
1348	func TestPrevBackupIsNotJournaled(t *testing.T) {
1349	f, _ := applyHappy(t)
1350	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
1351	systemJournal{Phase: "sentinel", Target: "v9", PrevBuild: "v1", StartedAt: f.now}, 0o600)
1352	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "applied.json"),
1353	systemApplied{Version: "v1", Channel: "stable", At: f.now}, 0o644)
1354	ok, err := f.a.decidePrev(context.Background())
1355	if err != nil || !ok {
1356	t.Fatalf("decidePrev = %v, %v", ok, err)
1357	}
1358	prev, _ := os.ReadFile(systemPrevPath(f.a.optDir))
1359	if !strings.Contains(string(prev), "carlos v1") {
1360	t.Fatalf(".prev = %q, want the known-good v1 backed up", prev)
1361	}
1362	j, err := readSystemJournal(f.a.optDir)
1363	if err != nil {
1364	t.Fatal(err)
1365	}
1366	if j == nil || j.Phase != "sentinel" {
1367	t.Fatalf("journal = %+v, want the sentinel untouched: the backup writes no journal", j)
1368	}
1369	}
1370
1371	// A journal written by the version that DID journal the backup outlives the
1372	// upgrade to the one that does not: recovery still knows the phase, cleans
1373	// the half-written copy, and carries on to the request it was started for.
1374	func TestRecoveryStillHandlesALegacyBackingUpJournal(t *testing.T) {
1375	f, req := applyHappy(t)
1376	tmp := systemPrevPath(f.a.optDir) + ".tmp"
1377	os.WriteFile(tmp, []byte("half a binary"), 0o755)
1378	writeJSONDurable(filepath.Join(systemRootDir(f.a.optDir), "journal.json"),
1379	systemJournal{Phase: systemPhaseBackingUp, PrevBuild: "v1", StartedAt: f.now}, 0o600)
1380	if err := f.a.Run(context.Background(), false); err != nil {
1381	t.Fatal(err)
1382	}
1383	if _, err := os.Stat(tmp); err == nil {
1384	t.Error("the half-written .prev copy was left behind")
1385	}
1386	if j, _ := readSystemJournal(f.a.optDir); j != nil {
1387	t.Error("the legacy journal was left open")
1388	}
1389	if r := f.result(); r == nil || !r.matches("stable", req.Version, req.PromotedAt) || r.Outcome != systemOutcomeApplied {
1390	t.Fatalf("result = %+v, want the run to carry on to its request", r)
1391	}
1392	}
1393
1394	// TestApplyArmsTheTargetBuildsList is the forward-compatibility half of
1395	// discussion #118. The applier is the OLD binary: when the target build
1396	// introduces a unit that must be started at apply time rather than at the
1397	// next boot, only the target build's own plan knows that. So arm() starts
1398	// what plan.Armed names, even a unit this binary's own rule would skip.
1399	//
1400	// Without this, a unit new to the box is enabled and left inactive — which
1401	// is exactly how carlos-relay.service reached the Ireland box on 2026-09-07
1402	// enabled, gated, armed with a masters file, and not running.
1403	func TestApplyArmsTheTargetBuildsList(t *testing.T) {
1404	_, started := armCalls(t, bootstrapPlan{
1405	Enabled: []string{"carlos-edge.service", "carlos-future.service"},
1406	// A service this binary's armedUnits would never pick.
1407	Armed: []string{"carlos-future.service"},
1408	})
1409	if !slices.Contains(started, "carlos-future.service") {
1410	t.Errorf("apply did not start the unit the target build asked for: started %v", started)
1411	}
1412	if slices.Contains(started, "carlos-edge.service") {
1413	t.Errorf("apply started the edge, which runs this very code: started %v", started)
1414	}
1415	}
1416
1417	// The plan's armed list comes from another build, so arm() does not take it
1418	// on trust: the edge is never started whatever the plan says, and a unit the
1419	// plan arms but does not enable is dropped rather than passed to systemctl,
1420	// because one bad name in the single batched `start` would lose every unit
1421	// in it.
1422	func TestApplyRefusesToArmTheEdgeOrAnUnenabledUnit(t *testing.T) {
1423	_, started := armCalls(t, bootstrapPlan{
1424	Enabled: []string{"carlos-adopt.timer", "carlos-edge.service"},
1425	Armed: []string{"carlos-adopt.timer", "carlos-edge.service", "carlos-never-installed.service"},
1426	})
1427	if slices.Contains(started, "carlos-edge.service") {
1428	t.Errorf("the edge was started from the plan's armed list: started %v", started)
1429	}
1430	if slices.Contains(started, "carlos-never-installed.service") {
1431	t.Errorf("a unit the plan never enabled was handed to systemctl: started %v", started)
1432	}
1433	if !slices.Contains(started, "carlos-adopt.timer") {
1434	t.Errorf("the good unit was lost along with the bad ones: started %v", started)
1435	}
1436	}
1437
1438	// A plan.json from a build older than the Armed field has no armed key. That
1439	// is what a rollback stages (carlos.prev's plan), so the fallback is not a
1440	// theoretical path — arm() must decide locally exactly as it did before.
1441	func TestApplyArmsLocallyWhenPlanPredatesArmed(t *testing.T) {
1442	var plan bootstrapPlan
1443	if err := json.Unmarshal([]byte(`{"enabled":["carlos-adopt.timer","carlos-edge.service","carlos-relay.service"]}`), &plan); err != nil {
1444	t.Fatal(err)
1445	}
1446	if plan.Armed != nil {
1447	t.Fatalf("armed should be absent, got %v", plan.Armed)
1448	}
1449	_, started := armCalls(t, plan)
1450	// Membership, not substring: the relay is batched onto the same start
1451	// line as the timer, and its bare name also appears on the enable line.
1452	// A substring check is how the first version of this test passed
1453	// against a fallback changed to drop the relay — the exact #118 failure
1454	// mode it exists to catch.
1455	for _, want := range []string{"carlos-adopt.timer", "carlos-relay.service"} {
1456	if !slices.Contains(started, want) {
1457	t.Errorf("fallback did not arm %s: started %v", want, started)
1458	}
1459	}
1460	if slices.Contains(started, "carlos-edge.service") {
1461	t.Errorf("fallback started the edge: started %v", started)
1462	}
1463	}
1464
1465	// armCalls runs arm(plan) against a recording systemd and returns the units
1466	// it enabled and the units it started. Both are read as SETS out of the two
1467	// batched systemctl lines: arm puts every unit on one `enable` and every
1468	// started unit on one `start`, so a substring check over the joined log
1469	// cannot tell "start carlos-relay.service" from the same name appearing in
1470	// the enable line, and cannot see a name that is merely batched beside
1471	// another. The first version of these tests got both wrong.
1472	func armCalls(t *testing.T, plan bootstrapPlan) (enabled, started []string) {
1473	t.Helper()
1474	a := &systemApplier{ops: sysOps{
1475	systemctl: func(args ...string) (string, error) {
1476	switch args[0] {
1477	case "enable":
1478	enabled = append(enabled, args[1:]...)
1479	case "start":
1480	started = append(started, args[1:]...)
1481	}
1482	return "", nil
1483	},
1484	logf: func(string, ...any) {},
1485	}}
1486	if err := a.arm(plan); err != nil {
1487	t.Fatal(err)
1488	}
1489	return enabled, started
1490	}
1491
1492	// seedRoutes writes rows into the fixture's registry and closes it, so the
1493	// guard's own open is the only one held when Run reads it.
1494	func (f *applyFixture) seedRoutes(routes ...registry.Route) {
1495	f.t.Helper()
1496	reg, err := registry.Open(filepath.Join(f.a.dataDir, "registry.db"))
1497	if err != nil {
1498	f.t.Fatal(err)
1499	}
1500	defer reg.Close()
1501	for _, rt := range routes {
1502	if err := reg.Upsert(rt); err != nil {
1503	f.t.Fatal(err)
1504	}
1505	}
1506	}
1507
1508	func flatRow(dataDir, host string) registry.Route {
1509	return registry.Route{Host: host, Kind: "instance", App: "tito", Channel: "stable", Hibernate: true, Status: "ready",
1510	DBPath: provision.DefaultDBPath(dataDir, host), Socket: provision.DefaultSocket(host)}
1511	}
1512
1513	func nestedRow(dataDir, host string) registry.Route {
1514	rt := flatRow(dataDir, host)
1515	rt.DBPath, rt.Socket = provision.NestedDBPath(dataDir, host), provision.NestedSocket(host)
1516	return rt
1517	}
1518
1519	// postLayout gives the fixture's candidate the mount unit the layout ships
1520	// with, which is what makes it a release that understands nested rows.
1521	func (f *applyFixture) postLayout() {
1522	staged := filepath.Join(f.a.stagingDir(), "carlos")
1523	f.answer["units:"+staged] += " data-instances.mount"
1524	}
1525
1526	// A pre-layout candidate would remove the mount units and then serve a
1527	// nested row from an empty mountpoint directory, so the apply refuses it
1528	// before anything on the box changes, and names the row in the way.
1529	func TestApplyRefusesAPreLayoutCandidateOverANestedRow(t *testing.T) {
1530	f, req := applyHappy(t)
1531	f.seedRoutes(flatRow(f.a.dataDir, "flat.example.com"), nestedRow(f.a.dataDir, "acme.example.com"))
1532	if err := f.a.Run(context.Background(), false); err != nil {
1533	t.Fatal(err)
1534	}
1535	r := f.result()
1536	if r == nil || r.Outcome != systemOutcomeRefused || !r.matches("stable", "v2", req.PromotedAt) {
1537	t.Fatalf("result = %+v", r)
1538	}
1539	want := "refused: v2 predates the instance layout, and this box still has: acme.example.com. " +
1540	"revert the layout and wait for every row to report flat before installing it"
1541	if r.Reason != want {
1542	t.Fatalf("reason = %q\nwant %q", r.Reason, want)
1543	}
1544	if j, _ := readSystemJournal(f.a.optDir); j != nil {
1545	t.Fatalf("a refused apply wrote a journal: %+v", j)
1546	}
1547	if bin, _ := os.ReadFile(systemBinaryPath(f.a.optDir)); !strings.Contains(string(bin), "carlos v1") {
1548	t.Fatal("a refused apply swapped the binary")
1549	}
1550	if f.called("systemctl stop") || f.called("systemctl restart") {
1551	t.Fatalf("a refused apply touched systemd: %v", f.calls)
1552	}
1553	}
1554
1555	func TestApplyInstallsAPreLayoutCandidateOverFlatRows(t *testing.T) {
1556	f, req := applyHappy(t)
1557	f.seedRoutes(flatRow(f.a.dataDir, "flat.example.com"), flatRow(f.a.dataDir, "acme.example.com"))
1558	if err := f.a.Run(context.Background(), false); err != nil {
1559	t.Fatal(err)
1560	}
1561	if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied || !r.matches("stable", "v2", req.PromotedAt) {
1562	t.Fatalf("result = %+v", r)
1563	}
1564	}
1565
1566	// A candidate that ships the mount units understands nested rows, so the
1567	// guard never even opens the registry for it.
1568	func TestApplyInstallsAPostLayoutCandidateOverNestedRows(t *testing.T) {
1569	f, req := applyHappy(t)
1570	f.postLayout()
1571	f.seedRoutes(nestedRow(f.a.dataDir, "acme.example.com"))
1572	opened := false
1573	f.a.openRegistry = func() (*registry.Registry, error) {
1574	opened = true
1575	return nil, errors.New("not reached")
1576	}
1577	if err := f.a.Run(context.Background(), false); err != nil {
1578	t.Fatal(err)
1579	}
1580	if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied || !r.matches("stable", "v2", req.PromotedAt) {
1581	t.Fatalf("result = %+v", r)
1582	}
1583	if opened {
1584	t.Error("the guard opened the registry for a candidate that ships the layout")
1585	}
1586	}
1587
1588	// A candidate that ships the mount units but reports a version below
1589	// release.LayoutFloor (v0.20.0 to v0.24.2: mounts, but an edge unit on
1590	// which nested rows cannot wake) is refused over a nested row all the same.
1591	// This is the box's own line of defence against `carlos system update
1592	// --file`, which the console's guard does not judge. Flat rows let it in.
1593	func TestApplyRefusesAMountedButBelowFloorCandidateOverANestedRow(t *testing.T) {
1594	t.Parallel()
1595	for _, tc := range []struct {
1596	name string
1597	rows []registry.Route
1598	want string
1599	reason string
1600	}{
1601	{"nested row", nil, systemOutcomeRefused, "v0.24.2 predates the instance layout, and this box still has: acme.example.com"},
1602	{"flat rows", nil, systemOutcomeApplied, ""},
1603	} {
1604	t.Run(tc.name, func(t *testing.T) {
1605	f, req := applyHappyVersion(t, "v0.24.2")
1606	f.postLayout()
1607	dd := f.a.dataDir
1608	if tc.name == "nested row" {
1609	f.seedRoutes(flatRow(dd, "flat.example.com"), nestedRow(dd, "acme.example.com"))
1610	} else {
1611	f.seedRoutes(flatRow(dd, "flat.example.com"), flatRow(dd, "acme.example.com"))
1612	}
1613	if err := f.a.Run(context.Background(), false); err != nil {
1614	t.Fatal(err)
1615	}
1616	r := f.result()
1617	if r == nil || r.Outcome != tc.want || !r.matches("stable", "v0.24.2", req.PromotedAt) {
1618	t.Fatalf("result = %+v, want %s", r, tc.want)
1619	}
1620	if tc.reason != "" && !strings.Contains(r.Reason, tc.reason) {
1621	t.Fatalf("reason = %q, want it to contain %q", r.Reason, tc.reason)
1622	}
1623	})
1624	}
1625	// At the floor the unit set is what counts, and it ships the mounts.
1626	f, req := applyHappyVersion(t, "v0.25.0")
1627	f.postLayout()
1628	f.seedRoutes(nestedRow(f.a.dataDir, "acme.example.com"))
1629	if err := f.a.Run(context.Background(), false); err != nil {
1630	t.Fatal(err)
1631	}
1632	if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied || !r.matches("stable", "v0.25.0", req.PromotedAt) {
1633	t.Fatalf("v0.25.0 over a nested row = %+v, want applied", r)
1634	}
1635	}
1636
1637	// A registry the guard cannot read cannot vouch that nothing is nested.
1638	func TestApplyRefusesAPreLayoutCandidateWhenTheRegistryWillNotOpen(t *testing.T) {
1639	f, _ := applyHappy(t)
1640	f.a.openRegistry = func() (*registry.Registry, error) { return nil, errors.New("registry.db: disk I/O error") }
1641	if err := f.a.Run(context.Background(), false); err != nil {
1642	t.Fatal(err)
1643	}
1644	r := f.result()
1645	if r == nil || r.Outcome != systemOutcomeRefused || !strings.Contains(r.Reason, "registry.db: disk I/O error") ||
1646	!strings.Contains(r.Reason, "predates the instance layout") {
1647	t.Fatalf("result = %+v", r)
1648	}
1649	if j, _ := readSystemJournal(f.a.optDir); j != nil {
1650	t.Fatalf("a refused apply wrote a journal: %+v", j)
1651	}
1652	}
1653
1654	// Every row a pre-layout release would mishandle is named once: nested by
1655	// path shape whatever its flags, fenced by a journal, and a journal whose
1656	// row is gone. Flat rows are not named.
1657	func TestLayoutReversalIncompleteNamesEveryRowInTheWay(t *testing.T) {
1658	t.Parallel()
1659	dd := t.TempDir()
1660	reg, err := registry.Open(filepath.Join(dd, "registry.db"))
1661	if err != nil {
1662	t.Fatal(err)
1663	}
1664	defer reg.Close()
1665	unflagged := nestedRow(dd, "unflagged.example.com")
1666	unflagged.Hibernate = false
1667	for _, rt := range []registry.Route{
1668	flatRow(dd, "flat.example.com"),
1669	nestedRow(dd, "nested.example.com"),
1670	unflagged,
1671	flatRow(dd, "moving.example.com"),
1672	} {
1673	if err := reg.Upsert(rt); err != nil {
1674	t.Fatal(err)
1675	}
1676	}
1677	os.MkdirAll(filepath.Join(dd, "layout"), 0o700)
1678	for _, h := range []string{"moving.example.com", "gone.example.com"} {
1679	os.WriteFile(provision.LayoutJournalPath(dd, h), []byte("{}"), 0o600)
1680	}
1681	// A journal's temp file is not a journal.
1682	os.WriteFile(filepath.Join(dd, "layout", ".other.example.com.tmp-1"), []byte("{}"), 0o600)
1683
1684	got := layoutReversalIncomplete(reg, dd)
1685	want := []string{"gone.example.com", "moving.example.com", "nested.example.com", "unflagged.example.com"}
1686	if !slices.Equal(got, want) {
1687	t.Fatalf("reasons = %q, want %q", got, want)
1688	}
1689
1690	flatOnly := t.TempDir()
1691	reg2, err := registry.Open(filepath.Join(flatOnly, "registry.db"))
1692	if err != nil {
1693	t.Fatal(err)
1694	}
1695	defer reg2.Close()
1696	if err := reg2.Upsert(flatRow(flatOnly, "flat.example.com")); err != nil {
1697	t.Fatal(err)
1698	}
1699	if got := layoutReversalIncomplete(reg2, flatOnly); len(got) != 0 {
1700	t.Fatalf("a flat box reported %q", got)
1701	}
1702	}
1703
1704	// ---- the moves protocol floor (moves spec §11, Task 13) ----
1705
1706	// floorFixture is applyHappy with release numbers the floor can compare: a
1707	// box running `running` (applied.json names it, so the install refreshes
1708	// carlos.prev from it) and a signed candidate at target. The candidate
1709	// ships the layout's mount unit, so the layout guard never opens the
1710	// registry and every registry open in these tests is the floor's own.
1711	func floorFixture(t *testing.T, running, target string) (*applyFixture, systemRequest) {
1712	t.Helper()
1713	f := newApplyFixture(t, "stable", running)
1714	req := f.promote(target, []byte("#!/bin/sh\necho carlos "+target+"\n"), false)
1715	bin, staged, prev := systemBinaryPath(f.a.optDir), filepath.Join(f.a.stagingDir(), "carlos"), systemPrevPath(f.a.optDir)
1716	f.answer["version:"+staged] = "carlos " + target + "\n"
1717	f.answer["plan:"+staged] = f.answer["plan:"+bin]
1718	f.answer["units:"+staged] = f.answer["units:"+bin] + " " + layoutMountUnit
1719	f.answer["plan:"+prev] = f.answer["plan:"+bin]
1720	f.answer["units:"+prev] = f.answer["units:"+bin]
1721	// Every binary answers `version` from its own bytes from here on.
1722	delete(f.answer, "version:"+bin)
1723	if err := writeJSONDurable(f.a.file("applied.json"), systemApplied{Version: running, Channel: "stable", At: f.now}, 0o644); err != nil {
1724	t.Fatal(err)
1725	}
1726	return f, req
1727	}
1728
1729	// holdRow seeds a row and holds it under a move, as the agent's SetHold does.
1730	func (f *applyFixture) holdRow(host string) {
1731	f.t.Helper()
1732	f.seedRoutes(flatRow(f.a.dataDir, host))
1733	reg, err := registry.Open(filepath.Join(f.a.dataDir, "registry.db"))
1734	if err != nil {
1735	f.t.Fatal(err)
1736	}
1737	defer reg.Close()
1738	if err := reg.SetHold(host, "moving", "op-1"); err != nil {
1739	f.t.Fatal(err)
1740	}
1741	}
1742
1743	// onDisk is what the box's carlos binary says it is.
1744	func (f *applyFixture) onDisk() string { return f.a.onDiskBuild(context.Background()) }
1745
1746	// brokenBox puts a failed install on disk (running), a carlos.prev at prev
1747	// with its sidecar (none when sidecar is ""), and the journal an install
1748	// leaves open after its restart.
1749	func (f *applyFixture) brokenBox(running, prev, sidecar string) systemJournal {
1750	f.t.Helper()
1751	os.WriteFile(systemBinaryPath(f.a.optDir), []byte("#!/bin/sh\necho carlos "+running+"\n"), 0o755)
1752	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos "+prev+"\n"), 0o755)
1753	if sidecar != "" {
1754	if err := writePrevVersion(f.a.optDir, sidecar); err != nil {
1755	f.t.Fatal(err)
1756	}
1757	}
1758	j := systemJournal{Phase: systemPhaseRestarted, Target: running, PrevBuild: prev, StartedAt: f.now}
1759	if err := f.a.journal(j); err != nil {
1760	f.t.Fatal(err)
1761	}
1762	return j
1763	}
1764
1765	func TestSystemApplyArmsTheFloorBeforeStaging(t *testing.T) {
1766	f, _ := floorFixture(t, "v0.28.0", "v0.29.0")
1767	putFloor(t, f.store, "v0.30.0")
1768	staged := filepath.Join(f.a.stagingDir(), "carlos")
1769	run := f.a.ops.run
1770	f.a.ops.run = func(ctx context.Context, d time.Duration, name string, args ...string) (string, error) {
1771	if name == staged {
1772	t.Errorf("a below-floor candidate was staged and run: %s %v", name, args)
1773	}
1774	return run(ctx, d, name, args...)
1775	}
1776	before, _ := os.ReadFile(systemBinaryPath(f.a.optDir))
1777	if err := f.a.Run(context.Background(), false); err != nil {
1778	t.Fatal(err)
1779	}
1780	if lf, err := readLocalFloor(f.a.dataDir); err != nil || lf == nil || lf.MinRelease != "v0.30.0" {
1781	t.Fatalf("local floor = %+v, %v", lf, err)
1782	}
1783	if b, _ := os.ReadFile(filepath.Join(f.a.dataDir, moves.FloorMinReleaseFile)); string(b) != "v0.30.0\n" {
1784	t.Fatalf("floor.min-release = %q", b)
1785	}
1786	r := f.result()
1787	if r == nil || r.Outcome != systemOutcomeRefused || !strings.Contains(r.Reason, "moves protocol floor: v0.29.0 is below v0.30.0") {
1788	t.Fatalf("result = %+v", r)
1789	}
1790	if after, _ := os.ReadFile(systemBinaryPath(f.a.optDir)); !bytes.Equal(before, after) {
1791	t.Fatal("the binary changed")
1792	}
1793	if _, err := os.Stat(systemPrevPath(f.a.optDir)); err == nil {
1794	t.Fatal("a refused install refreshed carlos.prev")
1795	}
1796	if j, _ := readSystemJournal(f.a.optDir); j != nil {
1797	t.Fatal("a refused install opened a journal")
1798	}
1799	}
1800
1801	// Boot recovery runs before the network: the floor it enforces is the one
1802	// on disk, and the version it judges is carlos.prev's sidecar.
1803	func TestBootRecoveryUsesTheArmedFloorOffline(t *testing.T) {
1804	for _, tc := range []struct {
1805	name, sidecar string
1806	restored bool
1807	}{
1808	{"below the floor", "v0.29.0", false},
1809	{"at the floor", "v0.30.0", true},
1810	{"no sidecar", "", false},
1811	} {
1812	t.Run(tc.name, func(t *testing.T) {
1813	f, _ := floorFixture(t, "v0.31.0", "v0.32.0")
1814	f.a.openStore = func(context.Context) (release.Store, ed25519.PublicKey, error) {
1815	return unreadableStore{f.store}, f.pub, nil
1816	}
1817	armLocalFloor(t, f.a.dataDir, "v0.30.0")
1818	prev := tc.sidecar
1819	if prev == "" {
1820	prev = "v0.30.0"
1821	}
1822	f.brokenBox("v0.31.0", prev, tc.sidecar)
1823	f.edgeStartedThisBoot = false
1824	err := f.a.Run(context.Background(), true)
1825	j, _ := readSystemJournal(f.a.optDir)
1826	if tc.restored {
1827	if err != nil || f.onDisk() != "v0.30.0" || j != nil {
1828	t.Fatalf("err %v, on disk %q, journal %+v", err, f.onDisk(), j)
1829	}
1830	return
1831	}
1832	if err == nil || !strings.Contains(err.Error(), "moves protocol floor") {
1833	t.Fatalf("recovery err = %v", err)
1834	}
1835	if f.onDisk() != "v0.31.0" || j == nil {
1836	t.Fatalf("a refused recovery changed the box: on disk %q, journal %+v", f.onDisk(), j)
1837	}
1838	})
1839	}
1840	}
1841
1842	// decidePrev learns carlos.prev's version and judges it before the journal
1843	// is written. The proof of "before": region HA defers this install at the
1844	// step just ahead of the journal, and the two files are already there.
1845	func TestDecidePrevWritesTheSidecarAndVerdictBeforeTheJournal(t *testing.T) {
1846	t.Parallel()
1847	for _, tc := range []struct {
1848	name, running string
1849	held, verdict bool
1850	}{
1851	{"at or above the floor", "v0.31.0", false, true},
1852	{"below the floor", "v0.29.0", false, false},
1853	{"a held row", "v0.31.0", true, false},
1854	} {
1855	t.Run(tc.name, func(t *testing.T) {
1856	f, _ := floorFixture(t, tc.running, "v0.32.0")
1857	putFloor(t, f.store, "v0.30.0")
1858	if tc.held {
1859	f.holdRow("app.example.com")
1860	}
1861	// A verdict from an earlier judgement must not outlive this one.
1862	writeFileDurable(filepath.Join(f.a.optDir, moves.PrevAllowedFile), []byte("v0.30.0\n"), 0o644)
1863	f.a.regionHA = true
1864	os.MkdirAll(filepath.Join(f.a.dataDir, hostexecDir, "op-9"), 0o755)
1865	if err := f.a.Run(context.Background(), false); err != nil {
1866	t.Fatal(err)
1867	}
1868	if r := f.result(); r == nil || r.Outcome != systemOutcomeDeferred {
1869	t.Fatalf("result = %+v", r)
1870	}
1871	if j, _ := readSystemJournal(f.a.optDir); j != nil {
1872	t.Fatal("journal written")
1873	}
1874	if b, _ := os.ReadFile(filepath.Join(f.a.optDir, moves.PrevVersionFile)); string(b) != tc.running+"\n" {
1875	t.Fatalf("sidecar = %q, want %s", b, tc.running)
1876	}
1877	b, err := os.ReadFile(filepath.Join(f.a.optDir, moves.PrevAllowedFile))
1878	if tc.verdict != (err == nil) || (tc.verdict && string(b) != "v0.30.0\n") {
1879	t.Fatalf("verdict = %q, %v; want present %v", b, err, tc.verdict)
1880	}
1881	})
1882	}
1883	}
1884
1885	// Replacing carlos.prev withdraws its predecessor's sidecar and verdict
1886	// first, so a copy that dies part way leaves a carlos.prev nothing vouches
1887	// for, never the new bytes under the old verdict.
1888	func TestDecidePrevForgetsTheOldVerdictBeforeTheCopy(t *testing.T) {
1889	f, _ := floorFixture(t, "v0.31.0", "v0.32.0")
1890	putFloor(t, f.store, "v0.30.0")
1891	os.WriteFile(systemPrevPath(f.a.optDir), []byte("#!/bin/sh\necho carlos v0.30.0\n"), 0o755)
1892	writePrevVersion(f.a.optDir, "v0.30.0")
1893	writeFileDurable(filepath.Join(f.a.optDir, moves.PrevAllowedFile), []byte("v0.30.0\n"), 0o644)
1894	// Prevent publication without depending on the writer's temporary name.
1895	if err := os.Remove(systemPrevPath(f.a.optDir)); err != nil {
1896	t.Fatal(err)
1897	}
1898	if err := os.Mkdir(systemPrevPath(f.a.optDir), 0o755); err != nil {
1899	t.Fatal(err)
1900	}
1901	if err := f.a.Run(context.Background(), false); err != nil {
1902	t.Fatal(err)
1903	}
1904	if r := f.result(); r == nil || r.Outcome != systemOutcomeRefused {
1905	t.Fatalf("result = %+v", r)
1906	}
1907	for _, name := range []string{moves.PrevAllowedFile, moves.PrevVersionFile} {
1908	if _, err := os.Stat(filepath.Join(f.a.optDir, name)); err == nil {
1909	t.Errorf("%s survived a failed replacement of carlos.prev", name)
1910	}
1911	}
1912	}
1913
1914	// Moves spec §11 (round-6 finding 5): while a row is held, no activating
1915	// path takes the box below the release it runs, whatever the floor allows.
1916	func TestHeldParticipantRefusesEveryDowngrade(t *testing.T) {
1917	const downgrade = "a participant in a held operation does not downgrade"
1918	for _, held := range []bool{true, false} {
1919	t.Run(fmt.Sprintf("held=%v", held), func(t *testing.T) {
1920	// install
1921	f, _ := floorFixture(t, "v0.32.0", "v0.31.0")
1922	putFloor(t, f.store, "v0.30.0")
1923	if held {
1924	f.holdRow("app.example.com")
1925	}
1926	f.a.Run(context.Background(), false)
1927	r := f.result()
1928	if held && (r == nil || r.Outcome != systemOutcomeRefused || !strings.Contains(r.Reason, downgrade) || f.onDisk() != "v0.32.0") {
1929	t.Fatalf("held install: result %+v, on disk %q", r, f.onDisk())
1930	}
1931	if !held && (r == nil || r.Outcome != systemOutcomeApplied || f.onDisk() != "v0.31.0") {
1932	t.Fatalf("unheld install: result %+v, on disk %q", r, f.onDisk())
1933	}
1934
1935	// rollback
1936	f, req := floorFixture(t, "v0.32.0", "v0.33.0")
1937	armLocalFloor(t, f.a.dataDir, "v0.30.0")
1938	if held {
1939	f.holdRow("app.example.com")
1940	}
1941	j := f.brokenBox("v0.32.0", "v0.31.0", "v0.31.0")
1942	f.a.rollback(context.Background(), req, "gate: test", &j)
1943	r = f.result()
1944	if held && (r == nil || r.Outcome != systemOutcomeRollbackFailed || !strings.Contains(r.Reason, downgrade) || f.onDisk() != "v0.32.0") {
1945	t.Fatalf("held rollback: result %+v, on disk %q", r, f.onDisk())
1946	}
1947	if !held && (r == nil || r.Outcome != systemOutcomeRolledBack || f.onDisk() != "v0.31.0") {
1948	t.Fatalf("unheld rollback: result %+v, on disk %q", r, f.onDisk())
1949	}
1950
1951	// restoreFiles, which recovery calls without rollback
1952	f, _ = floorFixture(t, "v0.32.0", "v0.33.0")
1953	armLocalFloor(t, f.a.dataDir, "v0.30.0")
1954	if held {
1955	f.holdRow("app.example.com")
1956	}
1957	j = f.brokenBox("v0.32.0", "v0.31.0", "v0.31.0")
1958	_, err := f.a.restoreFiles(context.Background(), &j)
1959	if held && (err == nil || !strings.Contains(err.Error(), downgrade) || f.onDisk() != "v0.32.0") {
1960	t.Fatalf("held restore: err %v, on disk %q", err, f.onDisk())
1961	}
1962	if !held && (err != nil || f.onDisk() != "v0.31.0") {
1963	t.Fatalf("unheld restore: err %v, on disk %q", err, f.onDisk())
1964	}
1965	})
1966	}
1967	}
1968
1969	// D13: the floor is armed by publishing its object, not by the switch. A
1970	// deployment that never published it behaves as main, held rows and all,
1971	// and never opens the registry for the floor. Once armed, turning the
1972	// switch off does not disarm it.
1973	func TestFloorIsDormantWithoutTheObject(t *testing.T) {
1974	t.Setenv(moves.SwitchEnv, "")
1975	counting := func(f *applyFixture) *int {
1976	n := 0
1977	open := f.a.openRegistry
1978	f.a.openRegistry = func() (*registry.Registry, error) { n++; return open() }
1979	return &n
1980	}
1981
1982	f, _ := floorFixture(t, "v0.32.0", "v0.31.0")
1983	f.holdRow("app.example.com")
1984	opened := counting(f)
1985	if err := f.a.Run(context.Background(), false); err != nil {
1986	t.Fatal(err)
1987	}
1988	if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied || f.onDisk() != "v0.31.0" {
1989	t.Fatalf("install: result %+v, on disk %q", r, f.onDisk())
1990	}
1991	if *opened != 0 {
1992	t.Fatalf("an unarmed install opened the registry %d times", *opened)
1993	}
1994	// The activation guard creates .moves for its lock on every box (fix
1995	// round 2); an unarmed install writes no floor into it.
1996	for _, name := range []string{moves.FloorFile, moves.FloorMinReleaseFile} {
1997	if _, err := os.Stat(filepath.Join(f.a.dataDir, name)); err == nil {
1998	t.Fatalf("an unarmed install wrote %s", name)
1999	}
2000	}
2001	if _, err := os.Stat(filepath.Join(f.a.optDir, moves.PrevAllowedFile)); err == nil {
2002	t.Fatal("an unarmed install wrote a verdict")
2003	}
2004
2005	f, req := floorFixture(t, "v0.32.0", "v0.33.0")
2006	f.holdRow("app.example.com")
2007	opened = counting(f)
2008	j := f.brokenBox("v0.32.0", "v0.31.0", "")
2009	f.a.rollback(context.Background(), req, "gate: test", &j)
2010	if r := f.result(); r == nil || r.Outcome != systemOutcomeRolledBack || f.onDisk() != "v0.31.0" {
2011	t.Fatalf("rollback: result %+v, on disk %q", r, f.onDisk())
2012	}
2013	if *opened != 0 {
2014	t.Fatalf("an unarmed rollback opened the registry %d times", *opened)
2015	}
2016
2017	f, _ = floorFixture(t, "v0.32.0", "v0.33.0")
2018	f.holdRow("app.example.com")
2019	opened = counting(f)
2020	j = f.brokenBox("v0.32.0", "v0.31.0", "")
2021	if _, err := f.a.restoreFiles(context.Background(), &j); err != nil || f.onDisk() != "v0.31.0" {
2022	t.Fatalf("restore: err %v, on disk %q", err, f.onDisk())
2023	}
2024	if *opened != 0 {
2025	t.Fatalf("an unarmed restore opened the registry %d times", *opened)
2026	}
2027
2028	// Armed on disk, the switch still off: a below-floor carlos.prev stays put.
2029	f, _ = floorFixture(t, "v0.32.0", "v0.33.0")
2030	armLocalFloor(t, f.a.dataDir, "v0.30.0")
2031	j = f.brokenBox("v0.32.0", "v0.29.0", "v0.29.0")
2032	if _, err := f.a.restoreFiles(context.Background(), &j); err == nil || !strings.Contains(err.Error(), "moves protocol floor: v0.29.0 is below v0.30.0") {
2033	t.Fatalf("armed restore with the switch off: err %v", err)
2034	}
2035	if f.onDisk() != "v0.32.0" {
2036	t.Fatalf("on disk %q", f.onDisk())
2037	}
2038	}
2039
2040	// duringStaging runs change when the apply runs the staged candidate's
2041	// `version`, which is the middle of staging: after the early activation
2042	// check, before the install puts anything in place.
2043	func (f *applyFixture) duringStaging(change func()) {
2044	staged := filepath.Join(f.a.stagingDir(), "carlos")
2045	run := f.a.ops.run
2046	once := false
2047	f.a.ops.run = func(ctx context.Context, d time.Duration, name string, args ...string) (string, error) {
2048	if name == staged && !once {
2049	once = true
2050	change()
2051	}
2052	return run(ctx, d, name, args...)
2053	}
2054	}
2055
2056	// Fix round 1, finding 1: the early check can go stale while the candidate
2057	// is staged. The install checks again at the rename, under the floor lock
2058	// and with the held rows frozen, and a refusal there leaves the box as it
2059	// was: the binary untouched, no journal, and the quiesced passes re-armed.
2060	func TestActivationIsCheckedAgainAtTheRename(t *testing.T) {
2061	for _, tc := range []struct {
2062	name, running, target, want string
2063	change func(f *applyFixture)
2064	}{
2065	{"a floor raised during staging", "v0.31.0", "v0.32.0", "moves protocol floor: v0.32.0 is below v0.33.0",
2066	func(f *applyFixture) { armLocalFloor(t, f.a.dataDir, "v0.33.0") }},
2067	{"a hold set during staging", "v0.32.0", "v0.31.0", "a participant in a held operation does not downgrade",
2068	func(f *applyFixture) { f.holdRow("app.example.com") }},
2069	} {
2070	t.Run(tc.name, func(t *testing.T) {
2071	f, _ := floorFixture(t, tc.running, tc.target)
2072	putFloor(t, f.store, "v0.30.0")
2073	f.duringStaging(func() { tc.change(f) })
2074	if err := f.a.Run(context.Background(), false); err != nil {
2075	t.Fatal(err)
2076	}
2077	r := f.result()
2078	if r == nil || r.Outcome != systemOutcomeRefused || !strings.Contains(r.Reason, tc.want) {
2079	t.Fatalf("result = %+v", r)
2080	}
2081	if f.onDisk() != tc.running {
2082	t.Fatalf("on disk %q, want %s", f.onDisk(), tc.running)
2083	}
2084	if j, _ := readSystemJournal(f.a.optDir); j != nil {
2085	t.Fatalf("journal left open: %+v", j)
2086	}
2087	joined := strings.Join(f.calls, "\n")
2088	if !strings.Contains(joined, "systemctl start carlos-adopt.timer") {
2089	t.Fatalf("the quiesced passes were not re-armed:\n%s", joined)
2090	}
2091	if strings.Contains(joined, "systemctl restart carlos-edge") {
2092	t.Fatalf("a refused install restarted the edge:\n%s", joined)
2093	}
2094	})
2095	}
2096	}
2097
2098	// Fix round 1, finding 2: a gated journal is finished forward only when its
2099	// target is at or above the floor armed now. Below it, recovery goes back
2100	// to carlos.prev, which restoreFiles judges in turn.
2101	func TestForwardRecoveryChecksTheFloor(t *testing.T) {
2102	for _, tc := range []struct {
2103	name, prev, want string
2104	ok bool
2105	}{
2106	{"carlos.prev below the floor too", "v0.30.0", "v0.31.0", false},
2107	{"carlos.prev at the floor", "v0.32.0", "v0.32.0", true},
2108	} {
2109	t.Run(tc.name, func(t *testing.T) {
2110	f, _ := floorFixture(t, "v0.30.0", "v0.31.0")
2111	f.brokenBox("v0.31.0", tc.prev, tc.prev)
2112	f.a.journal(systemJournal{Phase: systemPhaseGated, Target: "v0.31.0", PrevBuild: tc.prev, StartedAt: f.now})
2113	armLocalFloor(t, f.a.dataDir, "v0.32.0")
2114	f.edgeStartedThisBoot = false
2115	err := f.a.Run(context.Background(), true)
2116	if ap, _ := readSystemApplied(f.a.optDir); ap != nil && ap.Version == "v0.31.0" {
2117	t.Fatalf("a below-floor target was finished forward: %+v", ap)
2118	}
2119	j, _ := readSystemJournal(f.a.optDir)
2120	if f.onDisk() != tc.want {
2121	t.Fatalf("on disk %q, want %s", f.onDisk(), tc.want)
2122	}
2123	if tc.ok && (err != nil || j != nil) {
2124	t.Fatalf("err %v, journal %+v", err, j)
2125	}
2126	if !tc.ok && (err == nil || !strings.Contains(err.Error(), "moves protocol floor") || j == nil) {
2127	t.Fatalf("err %v, journal %+v", err, j)
2128	}
2129	})
2130	}
2131	}
2132
2133	// restoreFiles puts carlos.prev in place under the floor lock: a floor
2134	// writer holding it holds the restore off.
2135	func TestRestoreWaitsForTheFloorLock(t *testing.T) {
2136	f, _ := floorFixture(t, "v0.32.0", "v0.33.0")
2137	armLocalFloor(t, f.a.dataDir, "v0.30.0")
2138	j := f.brokenBox("v0.32.0", "v0.31.0", "v0.31.0")
2139	held, free := make(chan struct{}), make(chan struct{})
2140	go withFloorLock(f.a.dataDir, false, func(*movesDir) error {
2141	close(held)
2142	<-free
2143	return nil
2144	})
2145	<-held
2146	done := make(chan error, 1)
2147	go func() { _, err := f.a.restoreFiles(context.Background(), &j); done <- err }()
2148	time.Sleep(300 * time.Millisecond)
2149	// The bytes, not f.onDisk: the fake ops are the restore's while it runs.
2150	if b, _ := os.ReadFile(systemBinaryPath(f.a.optDir)); !strings.Contains(string(b), "v0.32.0") {
2151	close(free)
2152	t.Fatalf("restored while the floor lock was held: %q", b)
2153	}
2154	close(free)
2155	if err := <-done; err != nil || f.onDisk() != "v0.31.0" {
2156	t.Fatalf("after the lock: err %v, on disk %q", err, f.onDisk())
2157	}
2158	}
2159
2160	// The install's rename waits for the floor lock: a floor writer that takes
2161	// it after the journal (at the quiesce) holds the binary where it is.
2162	func TestInstallRenameWaitsForTheFloorLock(t *testing.T) {
2163	f, _ := floorFixture(t, "v0.31.0", "v0.32.0")
2164	putFloor(t, f.store, "v0.30.0")
2165	var duringLock []byte
2166	held := make(chan struct{})
2167	freed := make(chan struct{})
2168	systemctl := f.a.ops.systemctl
2169	once := false
2170	f.a.ops.systemctl = func(args ...string) (string, error) {
2171	if args[0] == "stop" && !once {
2172	once = true
2173	go func() {
2174	defer close(freed)
2175	withFloorLock(f.a.dataDir, false, func(*movesDir) error {
2176	close(held)
2177	time.Sleep(time.Second)
2178	duringLock, _ = os.ReadFile(systemBinaryPath(f.a.optDir))
2179	return nil
2180	})
2181	}()
2182	<-held
2183	}
2184	return systemctl(args...)
2185	}
2186	if err := f.a.Run(context.Background(), false); err != nil {
2187	t.Fatal(err)
2188	}
2189	<-freed
2190	if !strings.Contains(string(duringLock), "carlos v0.31.0") {
2191	t.Fatalf("the binary was replaced while the floor lock was held: %q", duringLock)
2192	}
2193	if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied || f.onDisk() != "v0.32.0" {
2194	t.Fatalf("result %+v, on disk %q", r, f.onDisk())
2195	}
2196	}
2197
2198	// ---- fix round 2: one invariant. No executable is renamed into place, and
2199	// no journal is finished, unless the floor and hold check ran under
2200	// floor.lock and WithHoldsFrozen immediately before that step. ----
2201
2202	// unitActivity fakes systemctl's view of the root passes' triggers, so a
2203	// test can see which were active before a step and after it.
2204	func (f *applyFixture) unitActivity(active map[string]bool) {
2205	systemctl := f.a.ops.systemctl
2206	f.a.ops.systemctl = func(args ...string) (string, error) {
2207	switch args[0] {
2208	case "is-active":
2209	if len(args) > 1 && args[1] != systemEdgeUnit {
2210	var out []string
2211	for _, u := range args[1:] {
2212	if active[u] {
2213	out = append(out, "active")
2214	} else {
2215	out = append(out, "inactive")
2216	}
2217	}
2218	f.calls = append(f.calls, "systemctl "+strings.Join(args, " "))
2219	return strings.Join(out, "\n") + "\n", nil
2220	}
2221	case "stop":
2222	for _, u := range args[1:] {
2223	active[u] = false
2224	}
2225	case "start":
2226	for _, u := range args[1:] {
2227	active[u] = true
2228	}
2229	}
2230	return systemctl(args...)
2231	}
2232	}
2233
2234	func copyActivity(m map[string]bool) map[string]bool {
2235	c := map[string]bool{}
2236	for k, v := range m {
2237	c[k] = v
2238	}
2239	return c
2240	}
2241
2242	// First arming, during staging: the agent arms a floor on a box that had
2243	// none when the apply began. The rename refuses.
2244	func TestFirstFloorArmedDuringStagingRefusesTheRename(t *testing.T) {
2245	f, _ := floorFixture(t, "v0.31.0", "v0.32.0")
2246	f.duringStaging(func() {
2247	if err := writeLocalFloor(f.a.dataDir, moves.Floor{Min: 1, MinRelease: "v0.33.0"}); err != nil {
2248	t.Error(err)
2249	}
2250	})
2251	if err := f.a.Run(context.Background(), false); err != nil {
2252	t.Fatal(err)
2253	}
2254	if r := f.result(); r == nil || r.Outcome != systemOutcomeRefused || !strings.Contains(r.Reason, "moves protocol floor: v0.32.0 is below v0.33.0") {
2255	t.Fatalf("result = %+v", r)
2256	}
2257	if f.onDisk() != "v0.31.0" {
2258	t.Fatalf("on disk %q", f.onDisk())
2259	}
2260	}
2261
2262	// First arming, between the rule and the rename: the guard holds floor.lock
2263	// even on a box with no floor yet (it creates .moves for it), so the agent's
2264	// first arming waits until the rename is done and cannot slip in between.
2265	func TestFirstArmingWaitsForTheActivationLock(t *testing.T) {
2266	t.Parallel()
2267	f, _ := floorFixture(t, "v0.31.0", "v0.32.0")
2268	armed := make(chan error, 1)
2269	f.a.beforeSwap = func() {
2270	go func() { armed <- writeLocalFloor(f.a.dataDir, moves.Floor{Min: 1, MinRelease: "v0.40.0"}) }()
2271	time.Sleep(300 * time.Millisecond)
2272	if _, err := os.Stat(filepath.Join(f.a.dataDir, moves.FloorFile)); err == nil {
2273	t.Error("a first floor was armed between the activation check and the rename")
2274	}
2275	}
2276	if err := f.a.Run(context.Background(), false); err != nil {
2277	t.Fatal(err)
2278	}
2279	if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied {
2280	t.Fatalf("result = %+v", r)
2281	}
2282	if err := <-armed; err != nil {
2283	t.Fatal(err)
2284	}
2285	if lf, _ := readLocalFloor(f.a.dataDir); lf == nil || lf.MinRelease != "v0.40.0" {
2286	t.Fatalf("the agent's arming after the rename = %+v", lf)
2287	}
2288	}
2289
2290	// Forward recovery re-checks at the step it guards: a floor raised while
2291	// recovery stages the gated build's plan stops it being finished forward.
2292	func TestForwardRecoveryRechecksAfterStagePlan(t *testing.T) {
2293	f, _ := floorFixture(t, "v0.30.0", "v0.31.0")
2294	f.brokenBox("v0.31.0", "v0.30.0", "v0.30.0")
2295	f.a.journal(systemJournal{Phase: systemPhaseGated, Target: "v0.31.0", PrevBuild: "v0.30.0", StartedAt: f.now})
2296	armLocalFloor(t, f.a.dataDir, "v0.30.0")
2297	bin := systemBinaryPath(f.a.optDir)
2298	run := f.a.ops.run
2299	f.a.ops.run = func(ctx context.Context, d time.Duration, name string, args ...string) (string, error) {
2300	if name == bin && len(args) > 0 && args[0] == "bootstrap" {
2301	if err := writeLocalFloor(f.a.dataDir, moves.Floor{Min: 1, MinRelease: "v0.32.0"}); err != nil {
2302	t.Error(err)
2303	}
2304	}
2305	return run(ctx, d, name, args...)
2306	}
2307	f.edgeStartedThisBoot = false
2308	err := f.a.Run(context.Background(), true)
2309	if ap, _ := readSystemApplied(f.a.optDir); ap != nil && ap.Version == "v0.31.0" {
2310	t.Fatalf("finished forward below a floor raised during stagePlan: %+v", ap)
2311	}
2312	if err == nil || !strings.Contains(err.Error(), "moves protocol floor") {
2313	t.Fatalf("err = %v", err)
2314	}
2315	if j, _ := readSystemJournal(f.a.optDir); j == nil {
2316	t.Fatal("journal removed")
2317	}
2318	}
2319
2320	// A restore refused after the quiesce (the floor rose during the copy) puts
2321	// the root passes' triggers back as they were, and keeps the journal.
2322	func TestRefusedRestoreLeavesTheRootPassesAsTheyWere(t *testing.T) {
2323	f, _ := floorFixture(t, "v0.32.0", "v0.33.0")
2324	armLocalFloor(t, f.a.dataDir, "v0.30.0")
2325	j := f.brokenBox("v0.32.0", "v0.31.0", "v0.31.0")
2326	active := map[string]bool{"carlos-adopt.timer": true, "carlos-adopt.path": true, "carlos-stage.path": false, "carlos-restart.path": true}
2327	before := copyActivity(active)
2328	f.unitActivity(active)
2329	systemctl := f.a.ops.systemctl
2330	f.a.ops.systemctl = func(args ...string) (string, error) {
2331	out, err := systemctl(args...)
2332	if args[0] == "stop" {
2333	armLocalFloor(t, f.a.dataDir, "v0.33.0")
2334	}
2335	return out, err
2336	}
2337	_, err := f.a.restoreFiles(context.Background(), &j)
2338	if err == nil || !strings.Contains(err.Error(), "moves protocol floor: v0.31.0 is below v0.33.0") {
2339	t.Fatalf("err = %v", err)
2340	}
2341	for u, was := range before {
2342	if active[u] != was {
2343	t.Errorf("%s active = %v after the refusal, was %v", u, active[u], was)
2344	}
2345	}
2346	if f.onDisk() != "v0.32.0" {
2347	t.Fatalf("on disk %q", f.onDisk())
2348	}
2349	if _, err := os.Stat(systemBinaryPath(f.a.optDir) + ".restore"); err == nil {
2350	t.Fatal("the restore copy was left behind")
2351	}
2352	if j, _ := readSystemJournal(f.a.optDir); j == nil {
2353	t.Fatal("the recovery journal was dropped")
2354	}
2355	}
2356
2357	// An install refused at the rename starts the paused triggers again first,
2358	// and drops the journal only once they are started.
2359	func TestRefusedInstallRearmsBeforeDroppingTheJournal(t *testing.T) {
2360	f, _ := floorFixture(t, "v0.31.0", "v0.32.0")
2361	putFloor(t, f.store, "v0.30.0")
2362	active := map[string]bool{"carlos-adopt.timer": true, "carlos-adopt.path": true, "carlos-stage.path": true, "carlos-restart.path": false}
2363	before := copyActivity(active)
2364	f.unitActivity(active)
2365	systemctl := f.a.ops.systemctl
2366	f.a.ops.systemctl = func(args ...string) (string, error) {
2367	if args[0] == "start" {
2368	if j, _ := readSystemJournal(f.a.optDir); j == nil {
2369	t.Error("the journal was dropped before the paused passes were started again")
2370	}
2371	}
2372	return systemctl(args...)
2373	}
2374	f.duringStaging(func() { armLocalFloor(t, f.a.dataDir, "v0.33.0") })
2375	if err := f.a.Run(context.Background(), false); err != nil {
2376	t.Fatal(err)
2377	}
2378	if r := f.result(); r == nil || r.Outcome != systemOutcomeRefused {
2379	t.Fatalf("result = %+v", r)
2380	}
2381	for u, was := range before {
2382	if active[u] != was {
2383	t.Errorf("%s active = %v, was %v", u, active[u], was)
2384	}
2385	}
2386	if j, _ := readSystemJournal(f.a.optDir); j != nil {
2387	t.Fatalf("journal left: %+v", j)
2388	}
2389	}
2390
2391	// When starting them again fails, the run says so and fails, the retry
2392	// state stays on disk, and the next run (here the OnFailure recovery)
2393	// starts them. The journal does not stay: its units would pick carlos.prev,
2394	// and an older carlos.prev would restore itself over a binary that never
2395	// changed.
2396	func TestRefusedInstallWhoseRearmFailsSaysSoAndRetries(t *testing.T) {
2397	f, _ := floorFixture(t, "v0.31.0", "v0.32.0")
2398	putFloor(t, f.store, "v0.30.0")
2399	active := map[string]bool{"carlos-adopt.timer": true, "carlos-adopt.path": true, "carlos-stage.path": true, "carlos-restart.path": true}
2400	before := copyActivity(active)
2401	f.unitActivity(active)
2402	failStart := true
2403	systemctl := f.a.ops.systemctl
2404	f.a.ops.systemctl = func(args ...string) (string, error) {
2405	if args[0] == "start" && failStart {
2406	return "", errors.New("unit masked")
2407	}
2408	return systemctl(args...)
2409	}
2410	f.duringStaging(func() { armLocalFloor(t, f.a.dataDir, "v0.33.0") })
2411	err := f.a.Run(context.Background(), false)
2412	if err == nil || !strings.Contains(err.Error(), "could not be started again") {
2413	t.Fatalf("Run err = %v", err)
2414	}
2415	r := f.result()
2416	if r == nil || r.Outcome != systemOutcomeRefused || !strings.Contains(r.Reason, "could not be started again: unit masked") {
2417	t.Fatalf("result = %+v", r)
2418	}
2419	if _, err := os.Stat(f.a.file(rearmPendingFile)); err != nil {
2420	t.Fatalf("no retry state: %v", err)
2421	}
2422	if j, _ := readSystemJournal(f.a.optDir); j != nil {
2423	t.Fatalf("journal kept: %+v", j)
2424	}
2425	if f.onDisk() != "v0.31.0" {
2426	t.Fatalf("on disk %q", f.onDisk())
2427	}
2428
2429	failStart = false
2430	if err := f.a.Run(context.Background(), true); err != nil {
2431	t.Fatal(err)
2432	}
2433	if _, err := os.Stat(f.a.file(rearmPendingFile)); err == nil {
2434	t.Fatal("retry state left after the passes started")
2435	}
2436	for u, was := range before {
2437	if active[u] != was {
2438	t.Errorf("%s active = %v, was %v", u, active[u], was)
2439	}
2440	}
2441	if f.onDisk() != "v0.31.0" {
2442	t.Fatalf("the retry changed the binary: %q", f.onDisk())
2443	}
2444	}
2445
2446	// F15's dormancy rule for the lock: on a box with no floor, an activation
2447	// that cannot create .moves for its lock (a read-only data dir here) logs
2448	// it and goes ahead as main's would.
2449	func TestUnarmedBoxWithoutALockDirUpdatesAsMain(t *testing.T) {
2450	if os.Geteuid() == 0 {
2451	t.Skip("root ignores the read-only mode")
2452	}
2453	f, _ := floorFixture(t, "v0.31.0", "v0.32.0")
2454	if err := os.Chmod(f.a.dataDir, 0o555); err != nil {
2455	t.Fatal(err)
2456	}
2457	t.Cleanup(func() { os.Chmod(f.a.dataDir, 0o755) })
2458	if err := f.a.Run(context.Background(), false); err != nil {
2459	t.Fatal(err)
2460	}
2461	if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied || f.onDisk() != "v0.32.0" {
2462	t.Fatalf("result %+v, on disk %q", r, f.onDisk())
2463	}
2464	}
2465
2466	// ---- fix round 3: one guard, both locks, no unlocked path on contention ----
2467
2468	// shortFloorLockWait makes a contended floor.lock time out quickly.
2469	func shortFloorLockWait(t *testing.T) {
2470	old := floorLockWait
2471	floorLockWait = 200 * time.Millisecond
2472	t.Cleanup(func() { floorLockWait = old })
2473	}
2474
2475	// holdFloorLock takes floor.lock as a writer would, without publishing
2476	// anything, until the returned func is called.
2477	func holdFloorLock(t *testing.T, dataDir string) func() {
2478	t.Helper()
2479	held, free, done := make(chan struct{}), make(chan struct{}), make(chan struct{})
2480	go func() {
2481	defer close(done)
2482	if err := withFloorLock(dataDir, true, func(*movesDir) error {
2483	close(held)
2484	<-free
2485	return nil
2486	}); err != nil {
2487	t.Error(err)
2488	}
2489	}()
2490	<-held
2491	return func() { close(free); <-done }
2492	}
2493
2494	// A writer holding floor.lock has not necessarily published yet: a reader
2495	// that times out on it must not go ahead unlocked. The apply is deferred,
2496	// and nothing is activated.
2497	func TestContendedFloorLockDefersTheInstall(t *testing.T) {
2498	shortFloorLockWait(t)
2499	f, _ := floorFixture(t, "v0.31.0", "v0.32.0")
2500	unlock := holdFloorLock(t, f.a.dataDir)
2501	defer unlock()
2502	if err := f.a.Run(context.Background(), false); err != nil {
2503	t.Fatal(err)
2504	}
2505	r := f.result()
2506	if r == nil || r.Outcome != systemOutcomeDeferred || !strings.Contains(r.Reason, "another writer holds the floor lock") {
2507	t.Fatalf("result = %+v", r)
2508	}
2509	if f.onDisk() != "v0.31.0" {
2510	t.Fatalf("activated under a contended lock: on disk %q", f.onDisk())
2511	}
2512	if j, _ := readSystemJournal(f.a.optDir); j != nil {
2513	t.Fatalf("journal left: %+v", j)
2514	}
2515	}
2516
2517	// With a floor armed, the guard freezes the holds even where the version
2518	// being made current is the one already on disk (a journal finish).
2519	func TestForwardFinishFreezesTheHolds(t *testing.T) {
2520	f, _ := floorFixture(t, "v0.30.0", "v0.31.0")
2521	f.brokenBox("v0.31.0", "v0.30.0", "v0.30.0")
2522	f.a.journal(systemJournal{Phase: systemPhaseGated, Target: "v0.31.0", PrevBuild: "v0.30.0", StartedAt: f.now})
2523	armLocalFloor(t, f.a.dataDir, "v0.30.0")
2524	frozen := 0
2525	open := f.a.openRegistry
2526	f.a.openRegistry = func() (*registry.Registry, error) { frozen++; return open() }
2527	f.edgeStartedThisBoot = false
2528	if err := f.a.Run(context.Background(), true); err != nil {
2529	t.Fatal(err)
2530	}
2531	if ap, _ := readSystemApplied(f.a.optDir); ap == nil || ap.Version != "v0.31.0" {
2532	t.Fatalf("not finished forward: %+v", ap)
2533	}
2534	if frozen == 0 {
2535	t.Fatal("the forward finish did not freeze the holds")
2536	}
2537	}
2538
2539	// The backward recovery's own finish (result and journal removal) is
2540	// guarded too: a floor raised after the restore's rename, before the
2541	// finish, keeps the journal.
2542	func TestBackwardRecoveryFinishRechecksTheFloor(t *testing.T) {
2543	f, _ := floorFixture(t, "v0.31.0", "v0.32.0")
2544	armLocalFloor(t, f.a.dataDir, "v0.30.0")
2545	f.brokenBox("v0.31.0", "v0.30.0", "v0.30.0")
2546	systemctl := f.a.ops.systemctl
2547	f.a.ops.systemctl = func(args ...string) (string, error) {
2548	if args[0] == "restart" {
2549	armLocalFloor(t, f.a.dataDir, "v0.31.0")
2550	}
2551	return systemctl(args...)
2552	}
2553	f.edgeStartedThisBoot = true
2554	err := f.a.Run(context.Background(), true)
2555	if err == nil || !strings.Contains(err.Error(), "moves protocol floor: v0.30.0 is below v0.31.0") {
2556	t.Fatalf("err = %v", err)
2557	}
2558	if j, _ := readSystemJournal(f.a.optDir); j == nil {
2559	t.Fatal("the journal was finished below a floor raised before the finish")
2560	}
2561	}
2562
2563	// A restore refused before its rename whose rearm then fails records the
2564	// units it paused (as they were before the quiesce) in rearm-pending.json.
2565	// The next recovery starts them first, and only then refuses again.
2566	func TestRefusedRestoreWhoseRearmFailsIsRetriedFirst(t *testing.T) {
2567	f, _ := floorFixture(t, "v0.32.0", "v0.33.0")
2568	armLocalFloor(t, f.a.dataDir, "v0.30.0")
2569	j := f.brokenBox("v0.32.0", "v0.31.0", "v0.31.0")
2570	active := map[string]bool{"carlos-adopt.timer": true, "carlos-adopt.path": true, "carlos-stage.path": false, "carlos-restart.path": true}
2571	before := copyActivity(active)
2572	f.unitActivity(active)
2573	failStart := true
2574	systemctl := f.a.ops.systemctl
2575	var order []string
2576	f.a.ops.systemctl = func(args ...string) (string, error) {
2577	if args[0] == "start" {
2578	order = append(order, "start")
2579	if failStart {
2580	return "", errors.New("unit masked")
2581	}
2582	}
2583	out, err := systemctl(args...)
2584	if args[0] == "stop" && failStart {
2585	armLocalFloor(t, f.a.dataDir, "v0.33.0")
2586	}
2587	return out, err
2588	}
2589	if _, err := f.a.restoreFiles(context.Background(), &j); err == nil || !strings.Contains(err.Error(), "could not be started again") {
2590	t.Fatalf("err = %v", err)
2591	}
2592	var p rearmPending
2593	if err := readJSONFile(f.a.file(rearmPendingFile), &p); err != nil {
2594	t.Fatalf("no retry state: %v", err)
2595	}
2596	if want := []string{"carlos-adopt.timer", "carlos-adopt.path", "carlos-restart.path"}; !slices.Equal(p.Units, want) {
2597	t.Fatalf("pending units = %v, want %v (as they were before the quiesce)", p.Units, want)
2598	}
2599
2600	failStart = false
2601	f.edgeStartedThisBoot = false
2602	err := f.a.Run(context.Background(), true)
2603	if err == nil || !strings.Contains(err.Error(), "moves protocol floor") {
2604	t.Fatalf("recovery err = %v", err)
2605	}
2606	if len(order) < 2 || order[len(order)-1] != "start" {
2607	t.Fatalf("the recovery did not start the pending passes: %v", order)
2608	}
2609	if _, err := os.Stat(f.a.file(rearmPendingFile)); err == nil {
2610	t.Fatal("retry state left after the passes started")
2611	}
2612	for u, was := range before {
2613	if active[u] != was {
2614	t.Errorf("%s active = %v, was %v", u, active[u], was)
2615	}
2616	}
2617	if j, _ := readSystemJournal(f.a.optDir); j == nil {
2618	t.Fatal("journal dropped")
2619	}
2620	}
2621