amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1//go:build linux && rootrunner
2
3package host
4
5import (
6 "bytes"
7 "context"
8 "crypto/ed25519"
9 "encoding/base64"
10 "encoding/json"
11 "errors"
12 "fmt"
13 "os"
14 "os/exec"
15 "path/filepath"
16 "slices"
17 "strconv"
18 "strings"
19 "syscall"
20 "testing"
21 "time"
22
23 "github.com/carlosframework/platform/internal/claimactivation"
24 "github.com/carlosframework/platform/internal/domains"
25 "github.com/carlosframework/platform/internal/genselect"
26 "github.com/carlosframework/platform/internal/hostactors"
27 "github.com/carlosframework/platform/internal/hostbox"
28 "github.com/carlosframework/platform/internal/hostclaim"
29 "github.com/carlosframework/platform/internal/hostfloor"
30 "github.com/carlosframework/platform/internal/hostops"
31 "github.com/carlosframework/platform/internal/hostwrite"
32 "github.com/carlosframework/platform/internal/moves"
33 "github.com/carlosframework/platform/internal/park"
34 "github.com/carlosframework/platform/internal/registry"
35 "github.com/carlosframework/platform/internal/release"
36 "github.com/carlosframework/platform/internal/rootparticipation"
37 "github.com/carlosframework/platform/internal/rootstate"
38)
39
40// Step 8 readiness witness (R4). Leo's real tree, rebuilt from its metadata
41// snapshot, goes through the window in order:
42//
43// 1. 8.1's masks, applied with the runbook's own commands to leo's unit set
44// installed in this runner's /etc/systemd/system, and checked by
45// systemd and by install-control's verifyOfflineRootMigration;
46// 2. backfill-collect (claims run, production runner);
47// 3. journal-migrate without --window: the census refuses the masks (S2 as
48// found), and leaves the migration resumable;
49// 4. journal-migrate with --window, under umask 077 (R3): census and
50// migration complete, the floor directories come out 0755, and leo's
51// signed census report names claimtrial as its route installation;
52// 5. backfill-adopt: on the dev roster it refuses, because sophie is a
53// roster box with no census key (STOP S3). With sophie's report standing
54// in, it publishes, claimtrial goes to leo, and the others stay unowned;
55// 6. the console's adoption pass (production ReconcileClaims) gives
56// claimtrial a serving claim on leo; the files seed selects it;
57// 7. root's first step for an adopted host with no installation, the
58// per-host empty census, refuses claimtrial's legacy row and files
59// (STOP S4): nothing on main installs an adopted legacy host.
60//
61// Scaffold writes (state from before the window, never claim state on the
62// window's path): the roster at revision 9 and the floor, signed by a
63// fixture release key (steps 5 to 7 made them); the three bucket records
64// and their committed domain claims (the bucket as it is); leo's identity
65// and census key (fixture owner bindings, as every accepted root migration
66// witness uses); the root floor history and guard binary (guard-arm,
67// step 6); claimtrial's registry row (leo-routes.tsv); and, in the second
68// test only, sophie's census report, which nothing on dev can produce.
69//
70// On the dev roster, after step 4, backfill-done is also run against leo's
71// real report: it names claimtrial, so done refuses (fail-closed on
72// installations).
73func TestRootStep8WindowWitnessDevRoster(t *testing.T) { rootStep8Witness(t, step8Shape{}) }
74
75func TestRootStep8WindowWitnessSophieReported(t *testing.T) {
76 rootStep8Witness(t, step8Shape{sophieReports: true})
77}
78
79// The empty-leo flow (claim trial Option B, Paul's D1 of 2026-10-20): the
80// dev records are deleted before the window (B1; all of them, as the
81// controller ruled after the live precheck, or all but rollcheck-0261 and
82// wakecheck, the two-host shape), and claimtrial's and wakecheck's leftovers
83// on leo are moved aside out of /data and /opt/carlos (B2). Leo's real tree
84// then runs without claimtrial's registry row and without those leftovers,
85// under 8.1's masks:
86//
87// 1. 8.1's masks, as above;
88// 2. backfill-collect freezes the records left (zero, or two);
89// 3. journal-migrate without --window still refuses the masks;
90// 4. journal-migrate --window under umask 077 completes, and leo's signed
91// census report lists zero installations;
92// 5. backfill-done, dry run then apply: done under If-Match over the
93// collecting plan, sophie (no census key) unreported, and the
94// production CheckCreateFreeze accepts the plan; the console's adoption
95// pass is a no-op on it and gives no host a claim.
96//
97// Scaffold writes are those above less claimtrial's registry row, less the
98// deleted records and their domain claims, and with no stand-in report for
99// sophie. Nothing on the window's path (the plan in both phases, leo's
100// census report, the migration, the masks) is hand-made.
101func TestRootStep8WindowWitnessEmptyLeoZeroHosts(t *testing.T) {
102 rootStep8Witness(t, step8Shape{empty: true})
103}
104
105func TestRootStep8WindowWitnessEmptyLeoTwoHosts(t *testing.T) {
106 rootStep8Witness(t, step8Shape{empty: true, keep: []string{leoRollcheck, leoWakecheck}})
107}
108
109// The rerun witnesses (fix round 1, F2): the empty-leo zero-host flow, with
110// a public floor directory left behind by an earlier journal-migrate
111// attempt in the window's umask 077 shell. The rerun, also under umask
112// 077, repairs it under the migration's lock before the first floor read,
113// completes, and leaves guard/moves and guard/seeds 0755 root:root:
114//
115// - Moves0700: guard/moves (guard-arm's floor history) left 0700;
116// - Seeds0700: guard/seeds left 0700;
117// - Interrupted: guard/seeds made by an earlier build's mkdirAllExact
118// mkdir under umask 077, its chmod never reached (0700, empty);
119// - Foreign: guard/seeds left 0700 but owned by the carlos user. The
120// migration refuses it and leaves it as it was; once it is root's again
121// the rerun repairs it and completes.
122//
123// The earlier attempt's leftover is the only scaffold write these add, made
124// after step 3's refused attempt and checked before the rerun.
125func TestRootStep8WindowWitnessRerunMoves0700(t *testing.T) {
126 rootStep8Witness(t, step8Shape{empty: true, floorDirs: "moves"})
127}
128
129func TestRootStep8WindowWitnessRerunSeeds0700(t *testing.T) {
130 rootStep8Witness(t, step8Shape{empty: true, floorDirs: "seeds"})
131}
132
133func TestRootStep8WindowWitnessRerunInterrupted(t *testing.T) {
134 rootStep8Witness(t, step8Shape{empty: true, floorDirs: "interrupted"})
135}
136
137func TestRootStep8WindowWitnessRerunForeign(t *testing.T) {
138 rootStep8Witness(t, step8Shape{empty: true, floorDirs: "foreign"})
139}
140
141type step8Shape struct {
142 // sophieReports gives sophie a census key and a stand-in report.
143 sophieReports bool
144 // empty is Option B: B1 and B2 done, and keep the records B1 left.
145 empty bool
146 keep []string
147 // floorDirs is the floor directory an earlier attempt left behind
148 // ("" for none; see the rerun witnesses).
149 floorDirs string
150}
151
152const (
153 step8Window = "/etc/carlos/claim-window-cohort1-20261021T0900Z"
154 step8Bucket = "deployment"
155 step8Leo = "leo-climbing"
156 step8Sophie = "sophie-knitting"
157 step8ID = "cohort1-20261021"
158)
159
160func rootStep8Witness(t *testing.T, shape step8Shape) {
161 sophieReports := shape.sophieReports
162 if os.Geteuid() != 0 {
163 t.Fatal("designated runner must execute as root")
164 }
165 ctx := t.Context()
166 uid, gid := generationRunnerRegistryUser(t)
167 for _, path := range []string{"/var/lib/carlos-claims", "/var/lib/carlos-release", "/var/lib/carlos-swap-status", "/etc/carlos"} {
168 if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) {
169 t.Fatal("runner already holds", path, err)
170 }
171 t.Cleanup(func() {
172 if err := os.RemoveAll(path); err != nil {
173 t.Error(err)
174 }
175 })
176 }
177 for _, path := range []string{"/run/carlos", "/run/carlos/hostguard"} {
178 if err := os.Mkdir(path, 0755); err == nil {
179 t.Cleanup(func() { os.RemoveAll(path) })
180 } else if !os.IsExist(err) {
181 t.Fatal(err)
182 }
183 }
184
185 // Leo's /data and /opt/carlos from the snapshot, owned as on leo.
186 scratch := generationRootRunnerDir(t)
187 data, opt := filepath.Join(scratch, "data"), filepath.Join(scratch, "opt")
188 for _, dir := range []string{data, opt} {
189 if err := os.Mkdir(dir, 0755); err != nil {
190 t.Fatal(err)
191 }
192 }
193 owner := func(path, who string) error {
194 switch who {
195 case "root:root":
196 return os.Lchown(path, 0, 0)
197 case "carlos:carlos":
198 return os.Lchown(path, uid, gid)
199 case "root:carlos":
200 return os.Lchown(path, 0, gid)
201 }
202 return fmt.Errorf("unexpected owner %q at %s", who, path)
203 }
204 buildLeoTree(t, leoTreeRows(t, false), leoTreeOptions{
205 Roots: map[string]string{"/data": data, "/opt/carlos": opt},
206 Owner: owner,
207 Content: func(root, path string) []byte {
208 if root == "/data" && path == "swap-failures.json" {
209 return []byte(`{"schema":1,"records":null}`)
210 }
211 return nil
212 },
213 Skip: func(r leoRow) bool {
214 // Rebuilt by the scaffold: the registry, root's guard and
215 // update state, the platform binary and its fallback.
216 if r.Root == "/data" && strings.HasPrefix(r.Path, "registry.db") {
217 return true
218 }
219 if shape.empty && step8MovedAside(r) {
220 return true
221 }
222 return r.Root == "/opt/carlos" && (r.Path == "system" || strings.HasPrefix(r.Path, "system/") || r.Path == "carlos" || strings.HasPrefix(r.Path, "carlos.prev"))
223 },
224 })
225
226 // Leo's unit set in this runner's /etc/systemd/system, then 8.1's masks.
227 installLeoUnits(t)
228 runStep8Shell(t, `
229WINDOW='`+step8Window+`'
230TRIGGERS=(carlos-adopt.timer carlos-adopt.path carlos-stage.path carlos-restart.path carlos-system-update.path)
231SERVICES=(carlos-adopt.service carlos-stage.service carlos-config-sync.service carlos-restart.service carlos-system-update.service carlos-system-recover.service)
232UNITS=("${TRIGGERS[@]}" "${SERVICES[@]}" carlos-edge.service)
233mkdir -p -m 0755 /etc/carlos
234mkdir -m 0700 -- "$WINDOW"
235mkdir -m 0700 -- "$WINDOW/held"
236systemctl daemon-reload
237systemctl stop "${TRIGGERS[@]}"
238systemctl stop "${SERVICES[@]}"
239systemctl stop carlos-edge.service
240systemctl disable carlos-edge.service
241for unit in "${UNITS[@]}"; do
242 mv -T -- "/etc/systemd/system/$unit" "$WINDOW/held/$unit"
243 ln -s -- /dev/null "/etc/systemd/system/$unit"
244done
245systemctl daemon-reload
246systemctl show "${UNITS[@]}" -p Id -p LoadState -p ActiveState
247`)
248 for _, unit := range claimWindowUnits {
249 out, err := exec.Command("systemctl", "show", unit, "-p", "LoadState", "--value").CombinedOutput()
250 if err != nil || strings.TrimSpace(string(out)) != "masked" {
251 t.Fatalf("8.1: %s is %q (%v)", unit, out, err)
252 }
253 }
254 if _, err := os.Lstat("/etc/systemd/system/multi-user.target.wants/carlos-edge.service"); !errors.Is(err, os.ErrNotExist) {
255 t.Fatal("8.1: the edge is still wanted", err)
256 }
257 if err := verifyOfflineRootMigration(ctx); err != nil {
258 t.Fatal("8.1: install-control's offline check", err)
259 }
260
261 // The bucket and roster as steps 5 to 7 left them (scaffold).
262 store := release.NewDirStore(filepath.Join(scratch, "bucket"))
263 key := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{1}, ed25519.SeedSize))
264 signer := release.RootSigner(key)
265 directory := hostactors.Directory{Store: store, PublicKey: key.Public().(ed25519.PublicKey), Bucket: step8Bucket}
266 leoKey := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{2}, ed25519.SeedSize))
267 sophieKey := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{3}, ed25519.SeedSize))
268 enrollment := strings.Repeat("a", 32)
269 identity := handoverRecoveryIdentity(t, hostbox.Actor{Box: step8Leo, Enrollment: enrollment, Revision: 9, Binding: "9", Region: "ie", ControlBucket: step8Bucket})
270 actor, err := identity.Verify(ctx)
271 if err != nil {
272 t.Fatal(err)
273 }
274 if err := hostfloor.Publish(ctx, store, signer, step8Bucket, hostfloor.Floor{Min: 1, MinRelease: "v2.0.0"}); err != nil {
275 t.Fatal(err)
276 }
277 floor, err := hostfloor.Read(ctx, store, directory.PublicKey, step8Bucket)
278 if err != nil {
279 t.Fatal(err)
280 }
281 roster := hostactors.Roster{V: 1, DeploymentBucket: step8Bucket, Revision: 9, RequestLimitNS: int64(hostbox.RequestLimit), HomeRegion: "ie", Publication: "published", FloorSHA256: floor.Digest,
282 Regions: map[string]hostactors.Region{"ie": {Account: actor.Cloud.Account, AWSRegion: "eu-west-1", Namespaces: []string{"platform"}, Pools: []string{"cohort1", "console"}}},
283 Boxes: map[string]hostactors.Box{},
284 Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}}
285 roster.Boxes[step8Leo] = hostactors.Box{BoxID: step8Leo, Enrollment: enrollment, Namespace: "platform", Label: step8Leo, Region: "ie", Pool: "cohort1", State: "active", LossIDs: []string{},
286 Cloud: hostactors.Cloud{Provider: "aws", Account: actor.Cloud.Account, Region: actor.Cloud.Region, InstanceID: actor.Cloud.InstanceID}, EvidenceSHA256: strings.Repeat("c", 64), CensusPublicKey: leoKey.Public().(ed25519.PublicKey)}
287 sophie := hostactors.Box{BoxID: step8Sophie, Enrollment: strings.Repeat("b", 32), Namespace: "platform", Label: step8Sophie, Region: "ie", Pool: "console", State: "active", LossIDs: []string{},
288 Cloud: hostactors.Cloud{Provider: "aws", Account: actor.Cloud.Account, Region: actor.Cloud.Region, InstanceID: "i-0def"}, EvidenceSHA256: strings.Repeat("d", 64)}
289 if sophieReports {
290 sophie.CensusPublicKey = sophieKey.Public().(ed25519.PublicKey)
291 }
292 roster.Boxes[step8Sophie] = sophie
293 body, err := release.SignRootEnvelope(signer, roster)
294 if err != nil {
295 t.Fatal(err)
296 }
297 if _, err := store.Put(ctx, hostactors.Key, body, ""); err != nil {
298 t.Fatal(err)
299 }
300 records := map[string]hostops.Record{}
301 for _, r := range leoRecords(t) {
302 if shape.empty && !slices.Contains(shape.keep, r.Holder.Host) {
303 continue // deleted at B1
304 }
305 if _, err := store.Put(ctx, r.Holder.Key, r.Body, ""); err != nil {
306 t.Fatal(err)
307 }
308 if r, err = hostops.ReadRecord(ctx, store, r.Holder.Key); err != nil {
309 t.Fatal(err)
310 }
311 records[r.Holder.Host] = r
312 inc, _ := r.Incarnation()
313 raw, _ := json.Marshal(domains.Claim{Schema: 1, Host: r.Holder.Host, Account: r.Holder.Account, App: r.Holder.App, Acquisition: r.Holder.Acquisition, RecordIncarnation: inc, OwnershipState: "committed"})
314 if _, err := store.Put(ctx, domains.ClaimKey(r.Holder.Host), raw, ""); err != nil {
315 t.Fatal(err)
316 }
317 }
318 // Leo's registry: its one route (leo-routes.tsv), owned as on leo; none
319 // once B1 has retired claimtrial.
320 reg, err := registry.Open(filepath.Join(data, "registry.db"))
321 if err != nil {
322 t.Fatal(err)
323 }
324 t.Cleanup(func() { reg.Close() })
325 if !shape.empty {
326 if err := reg.Upsert(leoRoute(data)); err != nil {
327 t.Fatal(err)
328 }
329 }
330 for _, suffix := range []string{"", "-wal", "-shm"} {
331 if err := os.Lchown(filepath.Join(data, "registry.db"+suffix), uid, gid); err != nil && !os.IsNotExist(err) {
332 t.Fatal(err)
333 }
334 }
335 // Root's floor history and guard binary, as guard-arm left them.
336 t.Setenv("CARLOS_OPT_DIR", opt)
337 t.Setenv("CARLOS_DATA_DIR", data)
338 t.Setenv("CARLOS_DEPLOYMENT_BUCKET", step8Bucket)
339 t.Setenv("CARLOS_RELEASE_PUBKEY", base64.StdEncoding.EncodeToString(directory.PublicKey))
340 t.Setenv("CARLOS_HOST_CLAIM", "1")
341 t.Setenv("CARLOS_HOST_CLAIM_ORIGINS", "1")
342 t.Setenv("CARLOS_POOLS", "cohort1")
343 t.Setenv("TMPDIR", scratch)
344 // guard-arm (step 6) leaves guard/ and guard/moves 0755 root:root. Make
345 // them so explicitly, under umask 022, rather than at whatever umask the
346 // runner's shell has, and check it: the migration below must find either
347 // that or the leftover a rerun shape makes after step 3, never a state
348 // the scaffold produced by accident.
349 floorID := rootFloorIdentity{Deployment: step8Bucket, Box: actor.Box, Enrollment: actor.Enrollment}
350 scaffoldUmask := syscall.Umask(0o022)
351 floors, err := openRootMovesFloors(opt, floorID, true)
352 if err != nil {
353 syscall.Umask(scaffoldUmask)
354 t.Fatal(err)
355 }
356 err = floors.reconstruct(ctx, rootFloorReconstruction{Identity: floorID, Floor: &moves.Floor{Min: 1, MinRelease: "v2.0.0"}, Source: "enrolled root history", Digest: floorTestDigest, Authorization: floorTestAuthorization(t, floors)})
357 floors.close()
358 syscall.Umask(scaffoldUmask)
359 if err != nil {
360 t.Fatal(err)
361 }
362 for _, dir := range []string{systemGuardDir(opt), rootFloorDir(opt)} {
363 st, err := os.Lstat(dir)
364 if err != nil || !st.IsDir() || st.Mode().Perm() != 0o755 || st.Sys().(*syscall.Stat_t).Uid != 0 || st.Sys().(*syscall.Stat_t).Gid != 0 {
365 t.Fatalf("scaffold: %s is not guard-arm's 0755 root:root directory: %v %v", dir, st.Mode(), err)
366 }
367 }
368 rootFloorRunnerBinary(t, opt, systemBinaryPath(opt), "v2.0.0", key)
369 rootFloorRunnerBinary(t, opt, filepath.Join(systemGuardDir(opt), "carlos"), "v2.0.0", key)
370 if _, err := os.Lstat(rootSeedsFloorDir(opt)); !errors.Is(err, os.ErrNotExist) {
371 t.Fatal("the seeds floor directory exists before journal-migrate", err)
372 }
373 unused := fix4UnusedDependencies{}
374 owners := fix4Owners{ClaimOwnerBindings{
375 Identity: identity, RetirementAuthority: &hostops.ProductionAuthority{Store: store, Directory: directory, Now: time.Now}, EvidenceKey: leoKey,
376 Local: unused, Controllers: unused, Data: unused, Inputs: unused, Evidence: unused,
377 InputPolicy: func(context.Context, *hostclaim.Claim, hostbox.Selection, hostwrite.Identity) error { return nil },
378 MovesPolicy: func(context.Context, *hostclaim.Claim, hostbox.Selection, hostwrite.Identity) error { return nil },
379 LeaseSet: func(context.Context, string, string) ([]string, error) { return nil, nil },
380 View: func(context.Context, hostclaim.Observed) (park.View, error) { return park.View{}, nil },
381 Authorize: func(context.Context, hostclaim.Observed, hostclaim.Obligation) error { return nil },
382 AdmissionEvidence: func(context.Context, hostclaim.Observed, hostclaim.Admission, *genselect.Plan) error { return nil },
383 PublicationEvidence: func(context.Context, hostclaim.Observed) error { return nil },
384 Close: func(context.Context) error { return nil },
385 }}
386
387 // 2. backfill-collect, dry run then apply.
388 runner := claimactivation.Runner{Directory: directory, Signer: signer}
389 collect := claimactivation.Plan{Phase: "backfill-collect", Bucket: step8Bucket, ExpectedRevision: 9, Backfill: &claimactivation.BackfillInput{ID: step8ID}}
390 if res, err := runner.Run(ctx, collect, true); err != nil || len(res.Backfill.Hosts) != len(records) {
391 t.Fatalf("backfill-collect dry run: %+v %v", res.Backfill, err)
392 }
393 res, err := runner.Run(ctx, collect, false)
394 if err != nil {
395 t.Fatal("backfill-collect", err)
396 }
397 t.Logf("backfill-collect: %d hosts frozen, phase %s, revision %d", len(res.Backfill.Hosts), res.Backfill.Phase, res.Backfill.DirectoryRevision)
398
399 r := ClaimRuntimeResources{Mode: "instance-control", Registry: reg, DataDir: data, OptDir: opt, UnitsDir: "/etc/systemd/system"}
400 // 3. journal-migrate as S2 found it: no --window, the masks refuse.
401 err = migrateClaimJournalsWith(ctx, store, r, owners, true)
402 if !errors.Is(err, hostbox.ErrTypedInventory) || !strings.Contains(err.Error(), "units entry") {
403 t.Fatalf("journal-migrate without --window = %v, want the census to refuse a mask", err)
404 }
405 t.Logf("journal-migrate without --window refused: %v", err)
406 // The floor directories are prepared under the migration's lock before
407 // its first floor read, so even this refused attempt made guard/seeds,
408 // 0755 root:root, and left guard/moves so.
409 for _, dir := range []string{rootFloorDir(opt), rootSeedsFloorDir(opt)} {
410 st, err := os.Lstat(dir)
411 if err != nil || !st.IsDir() || st.Mode().Perm() != 0o755 || st.Sys().(*syscall.Stat_t).Uid != 0 || st.Sys().(*syscall.Stat_t).Gid != 0 {
412 t.Fatalf("after the refused attempt: %s %v %v", dir, st.Mode(), err)
413 }
414 }
415 // 4. journal-migrate --window, as the window's root shell runs it, but
416 // under umask 077 (R3).
417 r.MigrationWindow = step8Window
418 masked, err := verifyJournalMigrateWindow(r)
419 if err != nil || len(masked) != len(claimWindowUnits) {
420 t.Fatalf("--window: %v %v", masked, err)
421 }
422 step8EarlierAttempt(t, opt, shape.floorDirs, uid, gid)
423 // Step 3's refused attempt got as far as the census in ready, so its
424 // migration.json is there; a refused rerun must leave it as it is.
425 migrationRecord := func() []byte {
426 state, err := rootstate.Open(rootMigrationDir, false, false)
427 if err != nil {
428 t.Fatal(err)
429 }
430 defer state.Close()
431 raw, err := state.Read("migration.json")
432 if err != nil {
433 t.Fatal(err)
434 }
435 return raw
436 }
437 var before []byte
438 if shape.floorDirs == "foreign" {
439 before = migrationRecord()
440 }
441 old := syscall.Umask(0o077)
442 err = migrateClaimJournalsWith(ctx, store, r, owners, true)
443 syscall.Umask(old)
444 if shape.floorDirs == "foreign" {
445 // Not root's: refused before any floor read, and left as it was.
446 if !errors.Is(err, os.ErrPermission) {
447 t.Fatalf("journal-migrate --window over a carlos-owned guard/seeds = %v, want a refusal", err)
448 }
449 st, serr := os.Lstat(rootSeedsFloorDir(opt))
450 if serr != nil || st.Mode().Perm() != 0o700 || st.Sys().(*syscall.Stat_t).Uid != uint32(uid) {
451 t.Fatalf("the refused guard/seeds changed: %v %v", st.Mode(), serr)
452 }
453 if !bytes.Equal(migrationRecord(), before) {
454 t.Fatal("the refused rerun changed migration.json")
455 }
456 t.Logf("journal-migrate --window refused a carlos-owned guard/seeds and left it 0700: %v", err)
457 if err := os.Lchown(rootSeedsFloorDir(opt), 0, 0); err != nil {
458 t.Fatal(err)
459 }
460 old = syscall.Umask(0o077)
461 err = migrateClaimJournalsWith(ctx, store, r, owners, true)
462 syscall.Umask(old)
463 }
464 if err != nil {
465 t.Fatal("journal-migrate --window", err)
466 }
467 state, err := rootstate.Open(rootMigrationDir, false, false)
468 if err != nil {
469 t.Fatal(err)
470 }
471 raw, err := state.Read("migration.json")
472 state.Close()
473 var saved rootMigrationRecord
474 if err != nil || rootparticipation.Decode(raw, &saved) != nil || saved.Phase != "complete" {
475 t.Fatalf("migration.json: %s %v", raw, err)
476 }
477 for path, want := range map[string]os.FileMode{"/var/lib/carlos-claims": 0o700, rootFloorDir(opt): 0o755, rootSeedsFloorDir(opt): 0o755, systemGuardDir(opt): 0o755} {
478 st, err := os.Stat(path)
479 if err != nil || !st.IsDir() || st.Mode().Perm() != want || st.Sys().(*syscall.Stat_t).Uid != 0 || st.Sys().(*syscall.Stat_t).Gid != 0 {
480 t.Fatalf("mode check after journal-migrate: %s %v %v", path, st.Mode(), err)
481 }
482 }
483 if shape.floorDirs != "" {
484 t.Logf("rerun over %s: complete; guard/moves and guard/seeds are 0755 root:root", shape.floorDirs)
485 }
486 reportRaw, _, err := store.Get(ctx, "control/host-census/"+step8ID+"/"+step8Leo+".json")
487 if err != nil {
488 t.Fatal(err)
489 }
490 var envelope hostops.CensusEnvelope
491 if err := json.Unmarshal(reportRaw, &envelope); err != nil {
492 t.Fatal(err)
493 }
494 if shape.empty {
495 if envelope.Report.Installations == nil || len(envelope.Report.Installations) != 0 {
496 t.Fatalf("leo's census report: %+v", envelope.Report.Installations)
497 }
498 t.Logf("journal-migrate --window: complete; leo's report lists zero installations (%d hosts frozen)", len(records))
499 rootStep8Done(t, store, directory, runner, shape)
500 return
501 }
502 inc, _ := records[leoClaimtrial].Incarnation()
503 if len(envelope.Report.Installations) != 1 || envelope.Report.Installations[0] != (hostops.CensusInstallation{Host: leoClaimtrial, Account: "bab", App: "claimtrial", Family: "instance", Key: records[leoClaimtrial].Holder.Key, Incarnation: inc, Route: true}) {
504 t.Fatalf("leo's census report: %+v", envelope.Report.Installations)
505 }
506 t.Logf("journal-migrate --window: complete; leo's report names %s as a route installation", leoClaimtrial)
507
508 // 5. backfill-adopt.
509 adopt := claimactivation.Plan{Phase: "backfill-adopt", Bucket: step8Bucket, ExpectedRevision: 9, Backfill: &claimactivation.BackfillInput{ID: step8ID}}
510 if !sophieReports {
511 // backfill-done over leo's real report, which names claimtrial
512 // (B1 skipped): fail-closed, the plan stays collecting.
513 done := claimactivation.Plan{Phase: "backfill-done", Bucket: step8Bucket, ExpectedRevision: 9, Backfill: &claimactivation.BackfillInput{ID: step8ID}}
514 if _, doneErr := runner.Run(ctx, done, false); !errors.Is(doneErr, claimactivation.ErrBackfillInstallations) {
515 t.Fatalf("backfill-done over leo's report naming claimtrial = %v, want the installations refusal", doneErr)
516 } else {
517 t.Logf("backfill-done refused over leo's report naming claimtrial: %v", doneErr)
518 }
519 _, err := runner.Run(ctx, adopt, false)
520 if !errors.Is(err, os.ErrNotExist) {
521 t.Fatalf("backfill-adopt on the dev roster = %v, want sophie's missing census report refused", err)
522 }
523 var plan hostops.BackfillPlan
524 if raw, _, err := store.Get(ctx, hostops.BackfillKey); err != nil || release.VerifyRootEnvelope(directory.PublicKey, raw, &plan) != nil || plan.Phase != "collecting" {
525 t.Fatalf("the plan after a refused adopt: %+v %v", plan, err)
526 }
527 t.Logf("STOP S3: backfill-adopt refused on the dev roster: %v", err)
528 return
529 }
530 sophieReport := hostops.CensusReport{Box: step8Sophie, Enrollment: sophie.Enrollment, BootID: "sophie-boot", Reservation: step8ID, DirectoryRevision: 9, CapturedAt: time.Now().UTC(), Complete: true,
531 Installations: []hostops.CensusInstallation{}, Seeds: &hostops.CensusSeeds{Journaled: []string{}, Gated: []string{}}}
532 if err := hostops.PublishCensusReport(ctx, store, directory, sophieReport, sophieKey); err != nil {
533 t.Fatal(err)
534 }
535 if res, err = runner.Run(ctx, adopt, true); err != nil {
536 t.Fatal("backfill-adopt dry run", err)
537 }
538 if res, err = runner.Run(ctx, adopt, false); err != nil {
539 t.Fatal("backfill-adopt", err)
540 }
541 for _, h := range res.Backfill.Hosts {
542 want := ""
543 if h.Host == leoClaimtrial {
544 want = step8Leo
545 }
546 if h.Box != want {
547 t.Fatalf("adopting plan: %s on %q", h.Host, h.Box)
548 }
549 }
550 if !slices.Equal(res.Backfill.Unowned, []string{leoRollcheck, leoWakecheck}) {
551 t.Fatalf("unowned: %v", res.Backfill.Unowned)
552 }
553 t.Logf("backfill-adopt: adopting; %s on %s; unowned %v", leoClaimtrial, step8Leo, res.Backfill.Unowned)
554
555 // 6. The console's adoption pass, as production builds it.
556 serial := 0
557 controller := hostops.NewBackfillController(store, directory, time.Now, func() string { serial++; return "witness-op-" + strconv.Itoa(serial) })
558 err = controller.ReconcileClaims(ctx)
559 if !errors.Is(err, hostops.ErrIdentity) {
560 t.Fatalf("adoption pass = %v, want the unowned hosts refused", err)
561 }
562 observed, err := hostclaim.Read(ctx, store, leoClaimtrial)
563 if err != nil || observed.Absent || observed.Claim.Op.Kind != hostclaim.KindServing || observed.Claim.Op.Exec != step8Leo || observed.Claim.Holder.WriterBox != step8Leo {
564 t.Fatalf("claimtrial's claim: %+v %v", observed.Claim, err)
565 }
566 candidates, err := filesSeedCandidates(ctx, store, step8Leo, "")
567 if err != nil || len(candidates) != 1 || candidates[0].Host != leoClaimtrial {
568 t.Fatalf("files seed candidates: %+v %v", candidates, err)
569 }
570 t.Logf("adoption: %s serving on %s (stamp %d, obligations %d); the files seed selects it", leoClaimtrial, step8Leo, observed.Claim.Holder.Stamp, len(observed.Claim.Obligations))
571
572 // 7. Root's first step for a claim with no installation on this box
573 // (rootexecution_authority's ErrNoRecord path): the per-host empty
574 // census, built as root's startup builds it, over the migrated box.
575 launches, err := openClaimRootJournal()
576 if err != nil {
577 t.Fatal(err)
578 }
579 defer launches.Close()
580 journals, err := hostbox.NewJournalService(launches)
581 if err != nil {
582 t.Fatal(err)
583 }
584 roots := map[string]string{"data": r.DataDir, "opt": r.OptDir, "units": r.UnitsDir}
585 census := &hostbox.EmptyInstallationCensus{Inventory: journals.Inventory, Registry: r.Registry, Launches: launches, Box: actor.Box, Enrollment: actor.Enrollment, DirectoryRevision: "9", Roots: roots, SharedPaths: claimGlobalSharedPaths(r)}
586 h := ClaimRootHTTP{Journals: journals, Launches: launches, Roots: roots, Reconcile: hostbox.ReconcileAdapter{Registry: r.Registry, Inventory: journals.Inventory, Census: census, Box: actor.Box, Enrollment: actor.Enrollment}}
587 err = h.rootEmptyCensus().CheckLocal(ctx, observed, hostbox.RecordSelection{Host: leoClaimtrial, Key: records[leoClaimtrial].Holder.Key, ETag: records[leoClaimtrial].ETag, Incarnation: inc, Family: "instance"})
588 if !errors.Is(err, hostbox.ErrTypedInventory) {
589 t.Fatalf("root's per-host census for the adopted legacy host = %v, want a refusal", err)
590 }
591 t.Logf("STOP S4: root refuses the adopted legacy host before any seed admission: %v", err)
592}
593
594// step8EarlierAttempt leaves the public floor directory an earlier
595// journal-migrate attempt in the window's umask 077 shell would have, and
596// checks it is there: the rerun witnesses' only extra scaffold write.
597func step8EarlierAttempt(t *testing.T, opt, shape string, uid, gid int) {
598 t.Helper()
599 dir := rootSeedsFloorDir(opt)
600 owner := uint32(0)
601 if shape == "" {
602 return
603 }
604 if shape != "moves" {
605 // Step 3's refused attempt made guard/seeds, empty; the earlier
606 // attempt this shape stands for left it differently.
607 if err := os.Remove(dir); err != nil {
608 t.Fatal(err)
609 }
610 }
611 switch shape {
612 case "moves":
613 dir = rootFloorDir(opt)
614 if err := os.Chmod(dir, 0o700); err != nil {
615 t.Fatal(err)
616 }
617 case "seeds":
618 if err := os.Mkdir(dir, 0o700); err != nil {
619 t.Fatal(err)
620 }
621 case "interrupted":
622 // An earlier build's mkdirAllExact, first half: os.MkdirAll(dir,
623 // 0755) under the window shell's umask; the process died before
624 // its chmod.
625 old := syscall.Umask(0o077)
626 err := os.MkdirAll(dir, 0o755)
627 syscall.Umask(old)
628 if err != nil {
629 t.Fatal(err)
630 }
631 case "foreign":
632 if err := os.Mkdir(dir, 0o700); err != nil {
633 t.Fatal(err)
634 }
635 if err := os.Lchown(dir, uid, gid); err != nil {
636 t.Fatal(err)
637 }
638 owner = uint32(uid)
639 default:
640 t.Fatalf("unknown floor-directory shape %q", shape)
641 }
642 st, err := os.Lstat(dir)
643 if err != nil || !st.IsDir() || st.Mode().Perm() != 0o700 || st.Sys().(*syscall.Stat_t).Uid != owner {
644 t.Fatalf("earlier attempt: %s is %v (%v), want 0700 owned by %d", dir, st.Mode(), err, owner)
645 }
646 t.Logf("earlier attempt left %s 0700 (owner %d)", dir, owner)
647}
648
649// rootStep8Done is the empty-leo flow's step 5: backfill-done, dry run then
650// apply, through the production runner, then the console's own readers.
651func rootStep8Done(t *testing.T, store release.Store, directory hostactors.Directory, runner claimactivation.Runner, shape step8Shape) {
652 t.Helper()
653 ctx := t.Context()
654 _, collectingTag, err := store.Get(ctx, hostops.BackfillKey)
655 if err != nil {
656 t.Fatal(err)
657 }
658 done := claimactivation.Plan{Phase: "backfill-done", Bucket: step8Bucket, ExpectedRevision: 9, Backfill: &claimactivation.BackfillInput{ID: step8ID}}
659 dry, err := runner.Run(ctx, done, true)
660 if err != nil || len(dry.Writes) != 1 || dry.Writes[0].BeforeETag != collectingTag || dry.Backfill.Phase != "done" {
661 t.Fatalf("backfill-done dry run: %+v %v", dry, err)
662 }
663 if _, tag, err := store.Get(ctx, hostops.BackfillKey); err != nil || tag != collectingTag {
664 t.Fatalf("backfill-done dry run wrote the plan: %v", err)
665 }
666 if err := hostops.CheckCreateFreeze(ctx, store, directory.PublicKey); !errors.Is(err, hostops.ErrPending) {
667 t.Fatalf("freeze while collecting = %v", err)
668 }
669 res, err := runner.Run(ctx, done, false)
670 if err != nil {
671 t.Fatal("backfill-done", err)
672 }
673 if res.Backfill.Phase != "done" || len(res.Backfill.Hosts) != len(shape.keep) || !slices.Equal(res.Backfill.Unreported, []string{step8Sophie}) || len(res.Backfill.Reports) != 1 || res.Backfill.Reports[step8Leo] == "" {
674 t.Fatalf("backfill-done: %+v", res.Backfill)
675 }
676 if len(res.Writes) != 1 || res.Writes[0].Key != hostops.BackfillKey || res.Writes[0].BeforeETag != collectingTag {
677 t.Fatalf("backfill-done writes: %+v", res.Writes)
678 }
679 var plan hostops.BackfillPlan
680 raw, _, err := store.Get(ctx, hostops.BackfillKey)
681 if err != nil || release.VerifyRootEnvelope(directory.PublicKey, raw, &plan) != nil || plan.Phase != "done" || plan.Hosts == nil || len(plan.Hosts) != len(shape.keep) {
682 t.Fatalf("the done plan: %+v %v", plan, err)
683 }
684 if err := hostops.CheckCreateFreeze(ctx, store, directory.PublicKey); err != nil {
685 t.Fatalf("CheckCreateFreeze after done = %v", err)
686 }
687 t.Logf("backfill-done: done under If-Match; %d hosts; unreported %v; CheckCreateFreeze nil", len(plan.Hosts), res.Backfill.Unreported)
688 // The console's adoption pass, as production builds it, does nothing.
689 serial := 0
690 controller := hostops.NewBackfillController(store, directory, time.Now, func() string { serial++; return "witness-op-" + strconv.Itoa(serial) })
691 if err := controller.ReconcileClaims(ctx); err != nil {
692 t.Fatalf("adoption pass over the done plan = %v", err)
693 }
694 if keys, err := store.List(ctx, "control/hosts/"); err != nil || len(keys) != 0 {
695 t.Fatalf("host claims after the adoption pass: %v %v", keys, err)
696 }
697 t.Logf("adoption pass over the done plan: no-op, no host claim")
698}
699
700// installLeoUnits writes leo's carlos units, drop-ins and enablement links
701// into this runner's /etc/systemd/system: unit files with the embedded
702// bytes leo's were installed from, drop-ins with a minimal body, and the
703// links exactly as the snapshot names them. The runner's own vendor links
704// stand in for leo's. Everything written here is removed afterwards.
705func installLeoUnits(t *testing.T) {
706 t.Helper()
707 const units = "/etc/systemd/system"
708 var made []string
709 t.Cleanup(func() {
710 for i := len(made) - 1; i >= 0; i-- {
711 if err := os.Remove(made[i]); err != nil && !os.IsNotExist(err) {
712 t.Error(err)
713 }
714 }
715 for _, unit := range claimWindowUnits {
716 // The window's masks and held files.
717 if target, err := os.Readlink(filepath.Join(units, unit)); err == nil && target == "/dev/null" {
718 os.Remove(filepath.Join(units, unit))
719 }
720 }
721 exec.Command("systemctl", "daemon-reload").Run()
722 })
723 carlos := func(name string) bool {
724 base := filepath.Base(name)
725 return strings.HasPrefix(base, "carlos") || base == "data-instances.mount" || strings.HasPrefix(base, "run-carlos-")
726 }
727 for _, r := range leoTreeRows(t, false) {
728 if r.Root != units || r.Path == "." {
729 continue
730 }
731 path := filepath.Join(units, r.Path)
732 switch r.Type {
733 case "d":
734 if !carlos(r.Path) {
735 if err := os.Mkdir(path, 0755); err == nil {
736 made = append(made, path)
737 } else if !os.IsExist(err) {
738 t.Fatal(err)
739 }
740 continue
741 }
742 if err := os.Mkdir(path, r.Mode); err != nil {
743 t.Fatal("runner already holds", path, err)
744 }
745 made = append(made, path)
746 case "f":
747 if !carlos(r.Path) {
748 continue
749 }
750 body, err := unitsFS.ReadFile("units/" + filepath.Base(r.Path))
751 if err != nil {
752 body = []byte("[Service]\n")
753 }
754 if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) {
755 t.Fatal("runner already holds", path, err)
756 }
757 if err := os.WriteFile(path, body, r.Mode); err != nil {
758 t.Fatal(err)
759 }
760 made = append(made, path)
761 case "l":
762 if !carlos(r.Path) || !strings.HasPrefix(r.Target, units+"/") {
763 continue
764 }
765 if err := os.Symlink(r.Target, path); err != nil {
766 t.Fatal("runner already holds", path, err)
767 }
768 made = append(made, path)
769 }
770 }
771 // Directories are removed after what is in them.
772 slices.SortStableFunc(made, func(a, b string) int { return strings.Count(a, "/") - strings.Count(b, "/") })
773}
774
775func runStep8Shell(t *testing.T, script string) {
776 t.Helper()
777 cmd := exec.Command("bash", "-c", "set -eu\numask 077\n"+script)
778 out, err := cmd.CombinedOutput()
779 if err != nil {
780 t.Fatalf("runbook block: %v\n%s", err, out)
781 }
782 t.Logf("runbook block:\n%s", out)
783}
784