amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1//go:build linux && rootrunner
2
3package host
4
5import (
6 "bytes"
7 "context"
8 "crypto/ed25519"
9 "crypto/rand"
10 "crypto/sha256"
11 "database/sql"
12 "encoding/base64"
13 "encoding/hex"
14 "encoding/json"
15 "errors"
16 "io"
17 "net"
18 "net/http"
19 "net/http/httptest"
20 "os"
21 "os/exec"
22 "path/filepath"
23 "runtime"
24 "strings"
25 "sync"
26 "testing"
27 "time"
28
29 "github.com/carlosframework/platform/internal/activator"
30 "github.com/carlosframework/platform/internal/cgroup"
31 "github.com/carlosframework/platform/internal/controlsock"
32 "github.com/carlosframework/platform/internal/hostactors"
33 "github.com/carlosframework/platform/internal/hostbox"
34 "github.com/carlosframework/platform/internal/hostclaim"
35 "github.com/carlosframework/platform/internal/hostfloor"
36 "github.com/carlosframework/platform/internal/hostguard"
37 "github.com/carlosframework/platform/internal/hostops"
38 "github.com/carlosframework/platform/internal/hostwrite"
39 "github.com/carlosframework/platform/internal/instancectl"
40 "github.com/carlosframework/platform/internal/lease"
41 "github.com/carlosframework/platform/internal/lease/leasetest"
42 "github.com/carlosframework/platform/internal/park"
43 "github.com/carlosframework/platform/internal/provision"
44 "github.com/carlosframework/platform/internal/registry"
45 "github.com/carlosframework/platform/internal/release"
46 "github.com/carlosframework/platform/internal/rollout"
47 "github.com/carlosframework/platform/internal/rootparticipation"
48 "github.com/carlosframework/platform/internal/rootstate"
49)
50
51const lifecycleHost = "lifecycle.bab.example"
52const lifecycleSibling = "sibling.bab.example"
53const lifecycleHome = "/var/lib/plan14-lifecycle"
54
55func init() {
56 if lifecyclePayloadArgs(os.Args) {
57 db, err := sql.Open("sqlite", os.Args[4])
58 if err != nil {
59 panic(err)
60 }
61 if _, err = db.Exec("PRAGMA journal_mode=WAL; CREATE TABLE IF NOT EXISTS lifecycle (value TEXT)"); err != nil {
62 panic(err)
63 }
64 listener, err := net.Listen("unix", os.Args[2])
65 if err != nil {
66 panic(err)
67 }
68 if err = http.Serve(listener, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { io.WriteString(w, "lifecycle tenant") })); err != nil {
69 panic(err)
70 }
71 os.Exit(0)
72 }
73}
74
75func lifecyclePayloadArgs(args []string) bool {
76 if len(args) != 5 {
77 return false
78 }
79 host := lifecycleHost
80 if args[2] == filepath.Join(provision.InstanceRunDir(lifecycleSibling), "app.sock") {
81 host = lifecycleSibling
82 }
83 return args[1] == "--socket" && args[2] == filepath.Join(provision.InstanceRunDir(host), "app.sock") && args[3] == "--db" && args[4] == provision.NestedDBPath("/data", host)
84}
85
86func TestPlan14LifecyclePayloadDispatch(t *testing.T) {
87 ours := []string{"tenant", "--socket", filepath.Join(provision.InstanceRunDir(lifecycleHost), "app.sock"), "--db", provision.NestedDBPath("/data", lifecycleHost)}
88 if !lifecyclePayloadArgs(ours) {
89 t.Fatal("lifecycle payload refused")
90 }
91 for _, index := range []int{1, 2, 3, 4} {
92 other := append([]string(nil), ours...)
93 other[index] = "another-test"
94 if lifecyclePayloadArgs(other) {
95 t.Fatalf("another payload intercepted at argument %d", index)
96 }
97 }
98 if lifecyclePayloadArgs(ours[:4]) {
99 t.Fatal("incomplete payload intercepted")
100 }
101}
102
103func lifecycleCheck(t *testing.T, step string, err error) {
104 t.Helper()
105 if err != nil {
106 t.Fatalf("%s: %v", step, err)
107 }
108}
109
110// Only the external box enrollment and object service are supplied by this
111// harness. Local authority, execution, settings, files and launch wiring use
112// the same constructors as instance-control and the edge.
113type lifecycleOwners struct{ t *testing.T }
114
115func (p lifecycleOwners) OpenClaimOwners(ctx context.Context, store release.Store, r ClaimRuntimeResources) (ClaimOwnerBindings, error) {
116 key := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{1}, ed25519.SeedSize))
117 identity := handoverRecoveryIdentity(p.t, hostbox.Actor{Box: "box", Enrollment: strings.Repeat("a", 32), Revision: 1, Binding: "1", Region: "ie", FleetHomeRegion: "ie", ControlBucket: "deployment"})
118 identity.HomeRegion = "ie"
119 actor, err := identity.Verify(ctx)
120 if err != nil {
121 return ClaimOwnerBindings{}, err
122 }
123 directory := hostactors.Directory{Store: store, PublicKey: key.Public().(ed25519.PublicKey), Bucket: actor.ControlBucket}
124 data, err := lease.NewVersionedS3(ctx, map[string]string{actor.Bucket: actor.Cloud.Region})
125 if err != nil {
126 return ClaimOwnerBindings{}, err
127 }
128 markers, err := newProductionClaimMarkers(ctx, actor.Bucket, actor.Cloud.Region)
129 if err != nil {
130 return ClaimOwnerBindings{}, err
131 }
132 local := &productionClaimLocal{Resources: r, Identity: identity, Directory: directory, Store: store, slots: map[string]*claimParticipation{}}
133 backend := claimExecBackend(r, productionInstanceKeyReaders())
134 tokens := claimAdminTokens{Guard: hostguard.Guard{DataDir: r.DataDir, Claims: hostguard.StoreReader{Store: store}}, Claims: store, Verify: identity.Verify, Local: identity.Local(local), Policy: hostbox.Policy{Held: hostbox.RegistryHoldPolicy(r.Registry)}.Check, Now: time.Now}
135 input := claimUnitInputReader(rootClaimUnitInputs)
136 if r.Mode == "agent" {
137 input = NewClaimRoleRootClient(controlsock.Path).UnitInputs
138 }
139 regional := productionClaimRegional{Identity: identity, Local: local}
140 out := ClaimOwnerBindings{Identity: identity, RetirementAuthority: &hostops.ProductionAuthority{Store: store, Directory: directory, Now: time.Now}, EvidenceKey: ed25519.NewKeyFromSeed(bytes.Repeat([]byte{2}, ed25519.SeedSize)), Controllers: productionStopControllers{}, Data: data, Local: local,
141 Serving: ClaimServingSources{Launch: claimServingLaunchSource(backend, tokens, input), Markers: markers},
142 InputPolicy: claimInputPolicy(r.Registry.Get, store, key.Public().(ed25519.PublicKey), backend, input), MovesPolicy: MovesClaimPolicy(MovesRecordsFrom(store)),
143 Evidence: regional, LeaseSet: regional.LeaseSet, View: regional.View, Authorize: regional.Authorize, Close: local.Close}
144 source := &movesFilesSource{DataDir: r.DataDir, Routes: r.Registry, newFiles: newProductionFilesStore}
145 _, err = openClaimFilesManifest(ctx, os.Getenv, source, store, data, actor, &out, claimBoundIdentity(identity.Verify, actor))
146 if r.Mode != "instance-control" {
147 out.EvidenceKey = nil
148 }
149 bindClaimLegacyOwners(&out)
150 return out, err
151}
152
153func lifecycleEnvironment(t *testing.T, endpoint string) {
154 t.Helper()
155 for k, v := range map[string]string{"CARLOS_HOST_CLAIM": "1", "CARLOS_HOST_CLAIM_ORIGINS": "1", "CARLOS_MOVES_PHASE1": "", "CARLOS_REGION_HA": "", "CARLOS_DATA_DIR": "/data", "CARLOS_OPT_DIR": "/opt/carlos", "CARLOS_DEPLOYMENT_BUCKET": "", "CARLOS_DEPLOYMENT_DIR": lifecycleHome + "/deployment", "CARLOS_REGION": "", "AWS_ACCESS_KEY_ID": "test-key", "AWS_SECRET_ACCESS_KEY": "test-secret", "AWS_SESSION_TOKEN": "", "AWS_EC2_METADATA_DISABLED": "true", "AWS_CONFIG_FILE": "/nonexistent", "AWS_SHARED_CREDENTIALS_FILE": "/nonexistent", "CARLOS_S3_ENDPOINT": endpoint, "AWS_ENDPOINT_URL_S3": endpoint, "CARLOS_RELEASE_PUBKEY": base64.StdEncoding.EncodeToString(ed25519.NewKeyFromSeed(bytes.Repeat([]byte{1}, ed25519.SeedSize)).Public().(ed25519.PublicKey))} {
156 t.Setenv(k, v)
157 }
158}
159
160func TestPlan14LifecycleColdWakeRootRunner(t *testing.T) {
161 t.Setenv("PLAN14_LIFECYCLE_PHASE", "probe")
162 lifecycleRun(t)
163}
164
165func TestRootRebootPlan14LifecycleEndToEndPhase1(t *testing.T) {
166 t.Setenv("PLAN14_LIFECYCLE_PHASE", "1")
167 TestPlan14LifecycleEndToEndRootRunnerPayload(t)
168}
169
170func TestRootRebootPlan14LifecycleEndToEndPhase2(t *testing.T) {
171 t.Setenv("PLAN14_LIFECYCLE_PHASE", "2")
172 TestPlan14LifecycleEndToEndRootRunnerPayload(t)
173}
174
175func TestPlan14LifecycleEndToEndRootRunnerPayload(t *testing.T) {
176 if os.Getenv("PLAN14_LIFECYCLE_PHASE") == "" {
177 t.Fatal("lifecycle E2E requires root-runner --reboot Plan14LifecycleEndToEnd")
178 }
179 t.Run("ordinary", lifecycleRun)
180 if os.Getenv("PLAN14_LIFECYCLE_PHASE") == "2" && os.Getenv("PLAN14_LIFECYCLE_AGENT") == "" {
181 for _, point := range []string{"reservation", "handover", "retirement"} {
182 t.Run("crash-"+point, func(t *testing.T) {
183 t.Setenv("PLAN14_LIFECYCLE_PHASE", "crash")
184 t.Setenv("PLAN14_LIFECYCLE_CRASH", point)
185 lifecycleRun(t)
186 })
187 }
188 }
189}
190
191func lifecycleRun(t *testing.T) {
192 if os.Getenv("PLAN14_LIFECYCLE_AGENT") == "1" {
193 lifecycleAgent(t)
194 return
195 }
196 TestPlan14RootJournalNamespaceRootRunner(t)
197 phase := os.Getenv("PLAN14_LIFECYCLE_PHASE")
198 resume := phase == "2"
199 uid, gid := generationRunnerRegistryUser(t)
200 for _, path := range []string{lifecycleHome, "/data", "/opt/carlos", "/run/carlos", "/run/carlos-control", "/etc/carlos", "/var/lib/carlos-claims", "/var/lib/carlos-release", rootRetryProjection} {
201 if !resume || strings.HasPrefix(path, "/run/") {
202 lifecycleCheck(t, "exclusive fixture "+path, os.Mkdir(path, 0755))
203 }
204 t.Cleanup(func() {
205 if phase == "1" && !t.Failed() {
206 return
207 }
208 lifecycleCheck(t, "cleanup "+path, os.RemoveAll(path))
209 })
210 }
211 lifecycleCheck(t, "private authority", os.Chmod("/var/lib/carlos-claims", 0700))
212 lifecycleCheck(t, "input service owner", os.Chown(lifecycleHome, uid, gid))
213 lifecycleCheck(t, "runtime service owner", os.Chown("/run/carlos", uid, gid))
214 lifecycleCheck(t, "data owner", os.Chown("/data", uid, gid))
215 for _, path := range []string{lifecycleHome + "/deployment", "/data/hostguard", "/data/.parked", "/data/layout", "/data/instances", "/data/.instances", "/run/carlos/.instances", "/run/carlos/.logs", "/data/.hostexec", "/run/carlos/instances", "/run/carlos/logs", "/run/carlos/hostguard", "/run/carlos/hostexec"} {
216 lifecycleCheck(t, "directory", os.MkdirAll(path, 0755))
217 lifecycleCheck(t, "directory owner", os.Chown(path, uid, gid))
218 }
219 for _, path := range []string{"/data/.instances", "/data/instances", "/data/layout", "/run/carlos/.logs", "/run/carlos/logs"} {
220 lifecycleCheck(t, "private tree", os.Chmod(path, 0700))
221 }
222 model := leasetest.NewVersionedBucket()
223 model.AddBucket("data")
224 if resume {
225 lifecycleRestoreBucket(t, model)
226 }
227 handler, _ := leasetest.NewS3Server(model)
228 server := httptest.NewServer(lifecycleS3(handler, model))
229 t.Cleanup(server.Close)
230 lifecycleEnvironment(t, server.URL)
231 store := release.NewDirStore(lifecycleHome + "/deployment")
232 key := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{1}, ed25519.SeedSize))
233 identity := handoverRecoveryIdentity(t, hostbox.Actor{Box: "box", Enrollment: strings.Repeat("a", 32), Revision: 1, Binding: "1", Region: "ie", FleetHomeRegion: "ie", ControlBucket: "deployment"})
234 identity.HomeRegion = "ie"
235 actor, err := identity.Verify(t.Context())
236 lifecycleCheck(t, "identity", err)
237 if !resume {
238 roster := hostactors.Roster{V: 1, HomeRegion: "ie", DeploymentBucket: actor.ControlBucket, Revision: 1, RequestLimitNS: int64(hostbox.RequestLimit), Boxes: map[string]hostactors.Box{actor.Box: {BoxID: actor.Box, Enrollment: actor.Enrollment, Namespace: actor.Namespace, Label: actor.Box, Region: actor.Region, Pool: "default", Cloud: hostactors.Cloud{Provider: actor.Cloud.Provider, Account: actor.Cloud.Account, Region: actor.Cloud.Region, InstanceID: actor.Cloud.InstanceID}, EvidenceSHA256: strings.Repeat("c", 64), CensusPublicKey: ed25519.NewKeyFromSeed(bytes.Repeat([]byte{2}, ed25519.SeedSize)).Public().(ed25519.PublicKey), State: "active"}}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}}
239 raw, err := release.SignRootEnvelope(release.RootSigner(key), roster)
240 lifecycleCheck(t, "sign roster", err)
241 _, err = store.Put(t.Context(), hostactors.Key, raw, "")
242 lifecycleCheck(t, "roster", err)
243 filesSeedConstructionMigration(t, "/opt/carlos", actor.ControlBucket, actor.Box, actor.Enrollment)
244
245 }
246 binary, err := os.Executable()
247 lifecycleCheck(t, "test binary", err)
248 image, err := os.ReadFile(binary)
249 lifecycleCheck(t, "test image", err)
250 if !resume {
251 lifecycleCheck(t, "initial version namespace", os.MkdirAll("/opt/carlos/versions", 0755))
252 lifecycleCheck(t, "executor", os.WriteFile("/opt/carlos/carlos", image, 0755))
253 lifecycleCheck(t, "previous executor", os.WriteFile("/opt/carlos/carlos.prev", image, 0755))
254 }
255 sum := sha256.Sum256(image)
256 platform := runtime.GOOS + "-" + runtime.GOARCH
257 pointer := release.SystemPointer{Schema: 1, Component: "router", Channel: "stable", Spread: "0s", PromotedAt: time.Now(), Current: release.SystemRelease{Version: "v2.0.0", ReleaseTag: "v2.0.0", MovesProtocol: 1, Artifacts: map[string]release.SystemArtifact{platform: {Key: release.SystemArtifactKey("v2.0.0", platform), SHA256: hex.EncodeToString(sum[:])}}}}
258 if !resume {
259 signed, err := release.SignRootEnvelope(release.RootSigner(key), pointer)
260 lifecycleCheck(t, "sign executor", err)
261 identities, err := rootstate.Open("/opt/carlos/system/guard/identities", false, true)
262 lifecycleCheck(t, "executor identities", err)
263 lifecycleCheck(t, "executor identity", identities.Write("v2.0.0.json", signed))
264 identities.Close()
265 }
266
267 lifecycleArmGuard(t, store, key, actor, pointer, resume)
268 if !resume {
269 state, err := rootstate.Open(rootparticipation.Namespace, true, false)
270 lifecycleCheck(t, "census state", err)
271 census := rootparticipation.Census{Schema: 1, Box: actor.Box, Enrollment: actor.Enrollment, Migration: 1, Revision: 1, LaunchRevision: 1, Complete: true}
272 raw, err := json.Marshal(census)
273 lifecycleCheck(t, "census encode", err)
274 lifecycleCheck(t, "census", state.Write("census.json", raw))
275 state.Close()
276 failures, err := rootstate.Open(rollout.RootFailuresDir, true, false)
277 lifecycleCheck(t, "failure state", err)
278 _, err = rollout.InitializeRootFailures(failures, rootUpdatesMigration(census), nil)
279 lifecycleCheck(t, "initialize migration failures", err)
280 failures.Close()
281 pins, err := rootstate.Open(release.RootGenerationsDir, true, false)
282 lifecycleCheck(t, "generation store", err)
283 pins.Close()
284 }
285
286 now := time.Now().UTC()
287 rec := provision.Record{Schema: 1, Host: lifecycleHost, Account: "bab", App: "lifecycle", Channel: "stable", Region: "", CreatedAt: now}
288 completeCreate := func() error { return nil }
289 if !resume {
290 completeCreate = lifecycleCreate(t, store, key, actor, pointer, rec)
291 manifest := release.Manifest{App: rec.App, Version: "v1", Kind: "binary", Artifacts: map[string]release.Artifact{release.AdoptPlatform(): {}}}
292 lifecycleCheck(t, "ship", release.Ship(t.Context(), store, rec.Account, manifest, map[string][]byte{release.AdoptPlatform(): image}))
293 lifecycleCheck(t, "promote", release.Promote(t.Context(), store, key, rec.Account, rec.App, "stable", "v1"))
294 } else {
295 found, _, e := provision.FindRecordByHost(t.Context(), store, lifecycleHost)
296 lifecycleCheck(t, "retained record", e)
297 rec = *found
298 }
299 withSibling := phase == "1" || phase == "2" || phase == "crash"
300 var completeSibling func() error
301 createSibling := func() {
302 if !withSibling || resume {
303 return
304 }
305 if completeSibling == nil {
306 sibling := provision.Record{Schema: 1, Host: lifecycleSibling, Account: "bab", App: "sibling", Channel: "stable", CreatedAt: time.Now().UTC()}
307 completeSibling = lifecycleCreate(t, store, key, actor, pointer, sibling)
308 manifest := release.Manifest{App: "sibling", Version: "v1", Kind: "binary", Artifacts: map[string]release.Artifact{release.AdoptPlatform(): {}}}
309 lifecycleCheck(t, "ship active sibling", release.Ship(t.Context(), store, "bab", manifest, map[string][]byte{release.AdoptPlatform(): image}))
310 lifecycleCheck(t, "promote active sibling", release.Promote(t.Context(), store, key, "bab", "sibling", "stable", "v1"))
311 _, _, e := release.StageSignedHead(t.Context(), store, key.Public().(ed25519.PublicKey), nil, "/opt/carlos", registry.Route{Host: lifecycleSibling, Account: "bab", App: "sibling", Channel: "stable"})
312 lifecycleCheck(t, "signed sibling stage", e)
313 lifecycleCheck(t, "new deployment owner", filepath.Walk(lifecycleHome+"/deployment", func(path string, info os.FileInfo, e error) error {
314 if e != nil {
315 return e
316 }
317 return os.Chown(path, uid, gid)
318 }))
319 }
320 if err := completeSibling(); errors.Is(err, hostops.ErrPending) {
321 return
322 } else {
323 lifecycleCheck(t, "complete populated-box create", err)
324 }
325 lifecycleCheck(t, "sibling create ack", os.WriteFile("/run/carlos/lifecycle-sibling-created", []byte("ready"), 0644))
326 }
327 // Only deployment inputs are service-owned. Root authority remains private.
328 lifecycleCheck(t, "deployment owner", filepath.Walk(lifecycleHome+"/deployment", func(path string, info os.FileInfo, err error) error {
329 if err != nil {
330 return err
331 }
332 return os.Chown(path, uid, gid)
333 }))
334 ctx := t.Context()
335 lifecycleInstallUnits(t)
336 out, mountErr := exec.Command("systemctl", "daemon-reload").CombinedOutput()
337 lifecycleCheck(t, string(out), mountErr)
338 out, mountErr = exec.Command("systemctl", "start", "data-instances.mount", "run-carlos-instances.mount", "run-carlos-logs.mount").CombinedOutput()
339 lifecycleCheck(t, string(out), mountErr)
340 recoverControl := lifecycleStartControl(t, server.URL)
341 lifecycleStartAgent(t, server.URL, completeCreate, createSibling, func() {
342 _, _, err = release.StageSignedHead(ctx, store, key.Public().(ed25519.PublicKey), nil, "/opt/carlos", registry.Route{Host: rec.Host, Account: rec.Account, App: rec.App, Channel: "stable"})
343 lifecycleCheck(t, "signed stage", err)
344 lifecycleCheck(t, "point initial version", release.PointLive("/opt/carlos", rec.Host, rec.Account, "v1", rec.App))
345 if withSibling && resume {
346 _, _, err := release.StageSignedHead(ctx, store, key.Public().(ed25519.PublicKey), nil, "/opt/carlos", registry.Route{Host: lifecycleSibling, Account: "bab", App: "sibling", Channel: "stable"})
347 lifecycleCheck(t, "signed sibling stage", err)
348 if resume {
349 lifecycleCheck(t, "sibling live", release.PointLive("/opt/carlos", lifecycleSibling, "bab", "v1", "sibling"))
350 }
351 }
352 }, lifecycleDeleteDriver(t, phase, store, key, actor, pointer), recoverControl)
353 if phase == "1" {
354 lifecycleSaveBucket(t, model)
355 }
356 if phase == "2" || phase == "crash" {
357 lifecycleVerifyRetiredPins(t, store, key)
358 lifecycleReinstall(t, store, key, actor, pointer)
359 lifecycleCheck(t, "release agent", os.WriteFile("/run/carlos/lifecycle-finish", []byte("ready"), 0644))
360 deadline := time.Now().Add(time.Minute)
361 for {
362 state, _ := exec.Command("systemctl", "show", "carlos-edge.service", "-p", "ActiveState", "--value").Output()
363 if strings.TrimSpace(string(state)) == "inactive" {
364 break
365 }
366 if strings.TrimSpace(string(state)) == "failed" || time.Now().After(deadline) {
367 log, _ := exec.Command("journalctl", "-u", "carlos-edge.service", "--no-pager", "-n", "80").CombinedOutput()
368 t.Fatalf("agent completion: %s", log)
369 }
370 time.Sleep(20 * time.Millisecond)
371 }
372 if phase == "crash" {
373 if _, err := os.Stat(lifecycleCrashPath); err != nil {
374 t.Fatal("crash point was not reached", err)
375 }
376 }
377 }
378
379}
380
381func lifecycleStartAgent(t *testing.T, endpoint string, completeCreate func() error, createSibling func(), stage func(), deletion func(), recoverControl func()) {
382 t.Helper()
383 unit, err := unitsFS.ReadFile("units/carlos-edge.service")
384 lifecycleCheck(t, "edge template", err)
385 var lines []string
386 for _, line := range strings.Split(string(unit), "\n") {
387 key, _, _ := strings.Cut(line, "=")
388 switch key {
389 case "ExecReload", "Restart":
390 continue
391 }
392 lines = append(lines, line)
393 }
394 lines = append(lines, "[Service]", "Restart=no", "Environment=PLAN14_LIFECYCLE_AGENT=1", "Environment=PLAN14_LIFECYCLE_PHASE="+os.Getenv("PLAN14_LIFECYCLE_PHASE"), "Environment=PLAN14_LIFECYCLE_DATA_ENDPOINT="+endpoint, "Environment=TMPDIR=/run/carlos", "ReadWritePaths="+lifecycleHome)
395 path := "/run/systemd/system/carlos-edge.service"
396 f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
397 lifecycleCheck(t, "edge exclusive", err)
398 _, err = f.WriteString(strings.Join(lines, "\n"))
399 f.Close()
400 lifecycleCheck(t, "edge write", err)
401 t.Cleanup(func() {
402 exec.Command("systemctl", "stop", "carlos-edge.service").Run()
403 os.Remove(path)
404 exec.Command("systemctl", "daemon-reload").Run()
405 })
406 out, err := exec.Command("systemctl", "daemon-reload").CombinedOutput()
407 lifecycleCheck(t, string(out), err)
408 out, err = exec.Command("systemctl", "start", "carlos-edge.service").CombinedOutput()
409 lifecycleCheck(t, string(out), err)
410 deadline := time.Now().Add(3 * time.Minute)
411 staged, created, recovered := false, false, false
412 for time.Now().Before(deadline) {
413 if _, err := os.Stat(lifecycleCrashPath); err == nil && !recovered {
414 recoverControl()
415 until := time.Now().Add(110 * time.Second)
416 for {
417 state, _ := exec.Command("systemctl", "show", "carlos-edge.service", "-p", "ActiveState", "--value").Output()
418 if strings.TrimSpace(string(state)) == "failed" || strings.TrimSpace(string(state)) == "inactive" {
419 break
420 }
421 if time.Now().After(until) {
422 lifecycleCrashDiagnostic(t)
423 t.Fatal("edge did not finish cleanup after controller crash")
424 }
425 time.Sleep(100 * time.Millisecond)
426 }
427 os.Remove("/run/carlos/lifecycle-result")
428 if os.Getenv("PLAN14_LIFECYCLE_CRASH") == "retirement" {
429 lifecycleCheck(t, "resume retirement checkpoint", os.WriteFile("/run/carlos/lifecycle-resume-retirement", []byte("ready"), 0644))
430 }
431 recoverControl()
432 out, err := exec.Command("systemctl", "start", "carlos-edge.service").CombinedOutput()
433 lifecycleCheck(t, string(out), err)
434 recovered = true
435 t.Log("controller and edge restarted after", os.Getenv("PLAN14_LIFECYCLE_CRASH"))
436 }
437 if _, err := os.Stat("/run/carlos/lifecycle-create-sibling"); err == nil {
438 createSibling()
439 }
440 if _, err := os.Stat("/run/carlos/lifecycle-sibling-installed"); err == nil {
441 lifecycleCheck(t, "sibling live", release.PointLive("/opt/carlos", lifecycleSibling, "bab", "v1", "sibling"))
442 }
443 if !created {
444 err := completeCreate()
445 if err == nil {
446 created = true
447 lifecycleCheck(t, "create ack", os.WriteFile("/run/carlos/lifecycle-created", []byte("ready"), 0644))
448 } else if !errors.Is(err, hostops.ErrPending) {
449 lifecycleCheck(t, "complete create", err)
450 }
451 }
452 if !staged {
453 if _, err := os.Stat("/run/carlos/lifecycle-stage"); err == nil {
454 stage()
455 lifecycleCheck(t, "stage ack", os.WriteFile("/run/carlos/lifecycle-staged", []byte("ready"), 0644))
456 staged = true
457 }
458 }
459 if _, err := os.Stat("/run/carlos/lifecycle-delete"); err == nil {
460 deletion()
461 }
462 raw, err := os.ReadFile("/run/carlos/lifecycle-result")
463 if err == nil {
464 if string(raw) != "pass" {
465 log, _ := exec.Command("journalctl", "-u", "carlos-edge.service", "--no-pager").CombinedOutput()
466 t.Fatalf("%s: %s", raw, log)
467 }
468 log, _ := exec.Command("journalctl", "-u", "carlos-edge.service", "--no-pager", "-n", "100").CombinedOutput()
469 t.Logf("completed agent: %s", log)
470 return
471 }
472 status, _ := exec.Command("systemctl", "show", "carlos-edge.service", "-p", "ActiveState", "--value").Output()
473 if strings.TrimSpace(string(status)) == "failed" {
474 break
475 }
476 time.Sleep(100 * time.Millisecond)
477 }
478 log, _ := exec.Command("journalctl", "-u", "carlos-edge.service", "--no-pager", "-n", "100").CombinedOutput()
479 t.Fatalf("agent did not complete: %s", log)
480}
481
482func lifecycleAgent(t *testing.T) {
483 lifecycleStackTrace(t)
484 deadlineReady := time.Now().Add(3 * time.Minute)
485 for {
486 if _, err := os.Stat("/run/carlos-control/lifecycle-ready"); err == nil {
487 break
488 }
489 if time.Now().After(deadlineReady) {
490 t.Fatal("controller activation timeout")
491 }
492 time.Sleep(20 * time.Millisecond)
493 }
494 lifecycleEnvironment(t, os.Getenv("CARLOS_S3_ENDPOINT"))
495 completed := false
496 defer func() {
497 result := "agent failed; inspect journal"
498 if completed && !t.Failed() {
499 result = "pass"
500 }
501 os.WriteFile("/run/carlos/lifecycle-result", []byte(result), 0600)
502 }()
503 store := release.NewDirStore(lifecycleHome + "/deployment")
504 reg, err := registry.Open("/data/registry.db")
505 lifecycleCheck(t, "agent registry", err)
506 defer reg.Close()
507 manager := cgroup.SetupRetaining(t.Logf)
508 resources := ClaimRuntimeResources{Mode: "agent", Registry: reg, Manager: manager, DataDir: "/data", OptDir: "/opt/carlos", UnitsDir: "/etc/systemd/system"}
509 ctx := context.WithValue(t.Context(), claimEntryKey{}, claimEntry{Enabled: true, FlagRead: true, Store: store, Resources: resources})
510 assembly, err := OpenProductionClaimAssembly(ctx, store, resources, lifecycleOwners{t})
511 lifecycleCheck(t, "agent assembly", err)
512 defer func() {
513 cleanup, cancel := context.WithTimeout(context.Background(), 30*time.Second)
514 defer cancel()
515 for {
516 t.Log("agent assembly close enter")
517 err := assembly.Close(cleanup)
518 t.Log("agent assembly close returned", err)
519 if err == nil {
520 return
521 }
522 if cleanup.Err() != nil {
523 t.Error("agent cleanup", err)
524 return
525 }
526 time.Sleep(100 * time.Millisecond)
527 }
528 }()
529 providers, err := assembly.Build(ctx)
530 lifecycleCheck(t, "agent build", err)
531 lifecycleTraceRegional(t, &providers)
532 oldPrerequisites := providers.Wake.Prerequisites
533 providers.Wake.Prerequisites = func(ctx context.Context, rt registry.Route, o hostclaim.Observed, policy activator.StartPolicy) error {
534 err := oldPrerequisites(ctx, rt, o, policy)
535 if err != nil && strings.Contains(err.Error(), "no restorable data") {
536 h := o.Claim.Holder
537 if assembly.Guard.FenceView != nil {
538 view, e := assembly.Guard.FenceView(ctx, o.Host, hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp})
539 t.Logf("empty recovery root view %+v error=%v", view, e)
540 for _, r := range view.Receipts {
541 t.Logf("empty recovery receipt target=%+v body=%s", r.Target, r.Bytes)
542 }
543 }
544 launches, e := assembly.Launches.HostLaunches(ctx, o.Host)
545 t.Logf("empty recovery launches %+v error=%v", launches, e)
546 }
547 return err
548 }
549 lifecycleTraceWake(t, assembly.Wake)
550 deadline := time.Now().Add(time.Minute)
551 for {
552 if _, err := os.Stat("/run/carlos/lifecycle-created"); err == nil {
553 break
554 }
555 if time.Now().After(deadline) {
556 t.Fatal("create acknowledgement timeout")
557 }
558 time.Sleep(20 * time.Millisecond)
559 }
560 cfg := provision.ReconcileConfig{Region: "", DataDir: "/data", Layout: func(context.Context, release.Store) (string, error) { return release.LayoutModeNested, nil }}
561 providers.Provision(&cfg)
562 resumingRetirement := false
563 if _, e := os.Stat("/run/carlos/lifecycle-resume-retirement"); e == nil {
564 obs, e := hostclaim.Read(ctx, store, lifecycleHost)
565 lifecycleCheck(t, "retirement recovery claim", e)
566 if obs.Claim == nil || (!obs.Claim.Free() && obs.Claim.Op.Kind != hostclaim.KindParking) {
567 t.Fatal("retirement recovery lost parking claim")
568 }
569 _, _, e = store.Get(ctx, provision.RecordKey("bab", "lifecycle", lifecycleHost))
570 if !errors.Is(e, os.ErrNotExist) {
571 t.Fatal("retirement recovery record still present", e)
572 }
573 resumingRetirement = true
574 }
575 lifecycleCheck(t, "provision", provision.Reconcile(ctx, store, reg, cfg))
576 if !resumingRetirement {
577 route, err := reg.Get(lifecycleHost)
578 lifecycleCheck(t, "route", err)
579 if route == nil {
580 t.Fatal("create did not install route")
581 }
582 lifecycleCheck(t, "request staging", os.WriteFile("/run/carlos/lifecycle-stage", []byte("ready"), 0600))
583 stageDeadline := time.Now().Add(time.Minute)
584 for {
585 if _, err := os.Stat("/run/carlos/lifecycle-staged"); err == nil {
586 break
587 }
588 if time.Now().After(stageDeadline) {
589 t.Fatal("stage timeout")
590 }
591 time.Sleep(20 * time.Millisecond)
592 }
593 wakes := 2
594 if os.Getenv("PLAN14_LIFECYCLE_PHASE") == "2" {
595 wakes = 1
596 }
597 for wake := 0; wake < wakes; wake++ {
598 route, err = reg.Get(lifecycleHost)
599 lifecycleCheck(t, "fresh route for wake", err)
600 if route == nil {
601 t.Fatal("route disappeared before wake")
602 }
603 finish := lifecycleLaunch(t, ctx, providers.Wake, assembly.Wake, *route)
604 if wake == 0 && (os.Getenv("PLAN14_LIFECYCLE_PHASE") == "1" || os.Getenv("PLAN14_LIFECYCLE_PHASE") == "crash") {
605 lifecycleCheck(t, "request populated-box create", os.WriteFile("/run/carlos/lifecycle-create-sibling", []byte("ready"), 0644))
606 until := time.Now().Add(time.Minute)
607 for {
608 if _, e := os.Stat("/run/carlos/lifecycle-sibling-created"); e == nil {
609 break
610 }
611 if time.Now().After(until) {
612 t.Fatal("populated-box creation timeout")
613 }
614 lifecycleCheck(t, "populated-box reconcile", provision.Reconcile(ctx, store, reg, cfg))
615 time.Sleep(20 * time.Millisecond)
616 }
617 lifecycleCheck(t, "provision sibling while main active", provision.Reconcile(ctx, store, reg, cfg))
618 sibling, e := reg.Get(lifecycleSibling)
619 lifecycleCheck(t, "new sibling row", e)
620 if sibling == nil {
621 t.Fatal("populated-box creation has no route")
622 }
623 lifecycleCheck(t, "sibling installed", os.WriteFile("/run/carlos/lifecycle-sibling-installed", []byte("ready"), 0644))
624 }
625 finish()
626 t.Logf("completed cold wake %d", wake+1)
627 }
628 }
629 if os.Getenv("PLAN14_LIFECYCLE_PHASE") == "2" || os.Getenv("PLAN14_LIFECYCLE_PHASE") == "crash" {
630 if !resumingRetirement {
631 parkDeadline := time.Now().Add(time.Minute)
632 for {
633 obs, err := hostclaim.Read(ctx, store, lifecycleHost)
634 lifecycleCheck(t, "pre-delete observation", err)
635 if len(obs.Claim.Obligations) == 0 {
636 break
637 }
638 if time.Now().After(parkDeadline) {
639 lifecycleTraceParkState(t, providers, obs)
640 t.Fatal("pre-delete obligations did not drain", obs.Claim.Obligations)
641 }
642 if err := providers.Regional.(hostbox.RegionalWorker).Run(ctx); err != nil {
643 t.Log("pre-delete parks", err)
644 }
645 time.Sleep(100 * time.Millisecond)
646 }
647 }
648 var finishSibling func()
649 {
650 sibling, err := reg.Get(lifecycleSibling)
651 lifecycleCheck(t, "sibling route", err)
652 if sibling == nil {
653 t.Fatal("missing sibling installation")
654 }
655 finishSibling = lifecycleLaunch(t, ctx, providers.Wake, assembly.Wake, *sibling)
656 lifecycleCheck(t, "active sibling ready", os.WriteFile("/run/carlos/lifecycle-sibling", []byte("active"), 0644))
657 }
658 lifecycleCheck(t, "request deletion", os.WriteFile("/run/carlos/lifecycle-delete", []byte("ready"), 0600))
659 deadline := time.Now().Add(time.Minute)
660 for time.Now().Before(deadline) {
661 if e := provision.Reconcile(ctx, store, reg, cfg); e != nil {
662 t.Log("retirement reconcile", e)
663 }
664 if e := providers.Regional.(hostbox.RegionalWorker).Run(ctx); e != nil {
665 t.Log("regional retirement", e)
666 }
667 obs, e := hostclaim.Read(ctx, store, lifecycleHost)
668 lifecycleCheck(t, "retirement observation", e)
669 if obs.Claim.Free() {
670 lifecycleCheck(t, "agent checkpoint", os.WriteFile("/run/carlos/lifecycle-result", []byte("pass"), 0600))
671 lifecycleAgentReinstall(t, ctx, store)
672 if finishSibling != nil {
673 finishSibling()
674 }
675 completed = true
676 return
677 }
678 time.Sleep(100 * time.Millisecond)
679 }
680 t.Fatal("retirement did not complete")
681 }
682 completed = true
683}
684
685func lifecycleWake(t *testing.T, ctx context.Context, runtime *activator.ClaimRuntime, wake *ClaimWakeAssembly, route registry.Route) {
686 lifecycleLaunch(t, ctx, runtime, wake, route)()
687}
688
689func lifecycleLaunch(t *testing.T, ctx context.Context, runtime *activator.ClaimRuntime, wake *ClaimWakeAssembly, route registry.Route) func() {
690 t.Helper()
691 observed, err := runtime.Read(ctx, route.Host)
692 lifecycleCheck(t, "read for wake", err)
693 policy := activator.StartPolicy{MaxStarts: 5, Window: 30 * time.Second}
694 prerequisiteDeadline := time.Now().Add(2 * time.Minute)
695 for {
696 e := runtime.Prerequisites(ctx, route, observed, policy)
697 if e == nil {
698 break
699 }
700 fresh, readErr := runtime.Read(ctx, route.Host)
701 retry := errors.Is(e, hostbox.ErrChanged) && readErr == nil && fresh.ETag != observed.ETag
702 if errors.Is(e, hostclaim.ErrWrongState) && readErr == nil && fresh.Claim.Holder.WakeRetry != nil && fresh.Claim.Holder.WakeRetry.State == "open" {
703 retry = true
704 }
705 if !retry || time.Now().After(prerequisiteDeadline) {
706 t.Logf("prerequisite state: holder=%+v route=%+v", observed.Claim.Holder, route)
707 lifecycleCheck(t, "wake prerequisites", e)
708 }
709 observed = fresh
710 time.Sleep(100 * time.Millisecond)
711 }
712 admitted, err := runtime.Admit(ctx, route, observed, policy)
713 lifecycleCheck(t, "cold admission", err)
714 start, err := runtime.Prepare(ctx, route, admitted, policy)
715 lifecycleCheck(t, "wake prepare", err)
716 lifecycleCheck(t, "restore", start.Restore(ctx))
717 lifecycleCheck(t, "reserve", start.Reserve(ctx, 0))
718 lifecycleCheck(t, "materialize", start.Materialize(ctx))
719 launch, err := start.PrepareLaunch(ctx)
720 lifecycleCheck(t, "launch preparation", err)
721 cmd, _, settle, producer, err := launch.Start(ctx, "main", start.CurrentAttempt(), start.Build, func() error { return ctx.Err() })
722 if err != nil && settle != nil {
723 cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
724 cleanupErr := lifecycleRetry(cleanup, settle)
725 cancel()
726 t.Log("failed launch settlement", cleanupErr)
727 }
728 lifecycleCheck(t, "root launch", err)
729 var stopMu sync.Mutex
730 stopped := make(chan struct{})
731 stop := func(call context.Context) error {
732 stopMu.Lock()
733 defer stopMu.Unlock()
734 select {
735 case <-stopped:
736 return nil
737 default:
738 }
739 if err := settle(call); err != nil {
740 return err
741 }
742 if cmd.Process != nil {
743 _ = cmd.Process.Release()
744 }
745 close(stopped)
746 return nil
747 }
748 if producer.Context != nil {
749 go func() {
750 select {
751 case <-stopped:
752 return
753 case <-producer.Context.Done():
754 }
755 cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
756 defer cancel()
757 _ = lifecycleRetry(cleanup, stop)
758 }()
759 }
760 lifecycleCheck(t, "tenant ready", start.Ready(ctx, cmd))
761 prefix, err := hostbox.GenerationPrefix(*admitted.Claim.Holder)
762 lifecycleCheck(t, "generation prefix", err)
763 rep, err := start.Replicate(ctx, prefix)
764 lifecycleCheck(t, "replicate", err)
765 current, err := runtime.Read(ctx, route.Host)
766 lifecycleCheck(t, "read publication", err)
767 _, err = start.Publish(ctx, current)
768 lifecycleCheck(t, "publish", err)
769 lifecycleCheck(t, "healthy", start.Healthy(ctx, 0, cmd))
770 transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
771 return (&net.Dialer{}).DialContext(ctx, "unix", route.Socket)
772 }}
773 defer transport.CloseIdleConnections()
774 response, err := (&http.Client{Transport: transport, Timeout: time.Second}).Get("http://tenant/")
775 lifecycleCheck(t, "serve", err)
776 raw, err := io.ReadAll(response.Body)
777 response.Body.Close()
778 lifecycleCheck(t, "read tenant", err)
779 if response.StatusCode != 200 || string(raw) != "lifecycle tenant" {
780 t.Fatalf("tenant response %d %q", response.StatusCode, raw)
781 }
782 return func() {
783 lifecycleCheck(t, "stop tenant", lifecycleRetry(ctx, stop))
784 lifecycleCheck(t, "stop replication", rep.Close(ctx))
785 current, err = runtime.Read(ctx, route.Host)
786 lifecycleCheck(t, "read stopped session", err)
787 lifecycleCheck(t, "stop session", lifecycleRetry(ctx, func(ctx context.Context) error { return wake.Stop(ctx, current) }))
788 }
789}
790
791func lifecycleInstallUnits(t *testing.T) {
792 t.Helper()
793 for _, name := range []string{"carlos.slice", "carlos-instances.slice", "data-instances.mount", "run-carlos-instances.mount", "run-carlos-logs.mount", "carlos-instance@.service", "carlos-system-recover.service"} {
794 raw, err := unitsFS.ReadFile("units/" + name)
795 lifecycleCheck(t, "unit template", err)
796 path := filepath.Join("/etc/systemd/system", name)
797 f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
798 lifecycleCheck(t, "exclusive unit", err)
799 _, err = f.Write(raw)
800 f.Close()
801 lifecycleCheck(t, "install unit", err)
802 t.Cleanup(func() {
803 exec.Command("systemctl", "stop", name).Run()
804 os.Remove(path)
805 exec.Command("systemctl", "daemon-reload").Run()
806 })
807 }
808 t.Cleanup(func() {
809 exec.Command("systemctl", "stop", instancectl.UnitName(instancectl.RoleRun, lifecycleSibling)).Run()
810 exec.Command("systemctl", "stop", instancectl.SliceName(lifecycleSibling)).Run()
811 exec.Command("systemctl", "stop", instancectl.UnitName(instancectl.RoleRun, lifecycleHost)).Run()
812 exec.Command("systemctl", "stop", instancectl.SliceName(lifecycleHost)).Run()
813 for _, name := range []string{instancectl.UnitName(instancectl.RoleRun, lifecycleHost), instancectl.SliceName(lifecycleHost), instancectl.UnitName(instancectl.RoleRun, lifecycleSibling), instancectl.SliceName(lifecycleSibling)} {
814 os.RemoveAll(filepath.Join("/run/systemd/system", name+".d"))
815 os.RemoveAll(filepath.Join("/run/systemd/system.control", name+".d"))
816 }
817 })
818}
819
820type lifecycleActorEvidence struct {
821 box, console hostactors.Report
822 identity []byte
823}
824
825func (e lifecycleActorEvidence) BoxReport(context.Context, string, hostactors.Box) (hostactors.Report, error) {
826 return e.box, nil
827}
828func (e lifecycleActorEvidence) ConsoleReport(context.Context, string, hostactors.Console) (hostactors.Report, error) {
829 return e.console, nil
830}
831func (e lifecycleActorEvidence) LocalConsole(context.Context) (hostactors.Report, error) {
832 return e.console, nil
833}
834func (e lifecycleActorEvidence) Identity(context.Context, string) ([]byte, error) {
835 return e.identity, nil
836}
837func (e lifecycleActorEvidence) Fence(context.Context, hostactors.Observed, string, time.Time) (bool, error) {
838 return false, nil
839}
840
841func lifecycleCreate(t *testing.T, raw release.Store, key ed25519.PrivateKey, actor hostbox.Actor, pointer release.SystemPointer, rec provision.Record) func() error {
842 t.Helper()
843 ctx := t.Context()
844 signer := release.RootSigner(key)
845 directory := hostactors.Directory{Store: raw, PublicKey: key.Public().(ed25519.PublicKey), Bucket: actor.ControlBucket}
846 lifecycleCheck(t, "publish test deployment floor", hostfloor.Publish(ctx, raw, signer, actor.ControlBucket, hostfloor.Floor{Min: 1, MinRelease: "v2.0.0"}))
847 floor, err := hostfloor.Read(ctx, raw, directory.PublicKey, actor.ControlBucket)
848 lifecycleCheck(t, "read deployment floor", err)
849 roster, err := directory.Read(ctx)
850 lifecycleCheck(t, "read actor roster", err)
851 if roster.Roster.Publication != "published" {
852 roster.Roster.Publication, roster.Roster.FloorSHA256 = "published", floor.Digest
853 roster.Roster.Regions = map[string]hostactors.Region{actor.Region: {Account: actor.Cloud.Account, AWSRegion: actor.Cloud.Region, Namespaces: []string{actor.Namespace}, Pools: []string{"default"}}}
854 roster.Roster.Consoles["console"] = hostactors.Console{Enrollment: strings.Repeat("d", 32), InstanceID: "console", Namespaces: []string{actor.Namespace}, DeploymentBucket: actor.ControlBucket}
855 _, err = directory.Replace(ctx, signer, roster, roster.Roster)
856 lifecycleCheck(t, "publish actors", err)
857 }
858 controller, authority, control := lifecycleController(t, raw, key, actor, pointer)
859 data := controller.Store
860 appID, err := control.EnrollNewApp(ctx, rec.Account, rec.App)
861 lifecycleCheck(t, "enroll app", err)
862 before, err := hostclaim.Read(ctx, data, rec.Host)
863 lifecycleCheck(t, "read before registration", err)
864 id := "create-" + rec.App
865 start, err := control.RegisterStart(ctx, rec.Account, rec.App, appID, id, hostops.Registration{Kind: "create", Host: rec.Host, Owner: hostops.RegistrationOwner{Console: "console", Enrollment: strings.Repeat("d", 32), Controller: "console-controller"}, AppID: appID, Epoch: 1, Claim: hostops.SavedObservation(before)}, time.Now())
866 lifecycleCheck(t, "register create", err)
867 rec.Acquisition = id
868 requested := rec
869 requested.CreatedAt = time.Time{}
870 record, err := hostops.PrepareInstance(requested, rec.CreatedAt, park.PlacementRegion(rec.Region))
871 lifecycleCheck(t, "prepare create record", err)
872 incarnation, err := record.Incarnation()
873 lifecycleCheck(t, "create incarnation", err)
874 lifecycleCheck(t, "bind registration", start.Bind(ctx, rec.CreatedAt, incarnation, time.Now()))
875 createCtx := hostops.WithStartHandle(start.Lifecycle(ctx), start)
876 observed, err := authority.StartCreate(createCtx, before, record, id, "runner")
877 lifecycleCheck(t, "production begin create", err)
878 lifecycleCheck(t, "production record effects", authority.RecordEffects(createCtx, observed, record))
879 created := false
880 return func() error {
881 if created {
882 return nil
883 }
884 _, err := controller.Complete(createCtx, rec.Host, id)
885 if err != nil {
886 return err
887 }
888 err = control.SettleRegistration(ctx, rec.Account, rec.App, appID, id, time.Now())
889 created = err == nil
890 return err
891 }
892}
893
894func TestPlan14LifecycleCreateController(t *testing.T) {
895 store := release.NewDirStore(t.TempDir())
896 key := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{1}, ed25519.SeedSize))
897 actor := hostbox.Actor{Box: "box", Enrollment: strings.Repeat("a", 32), Namespace: "platform", Region: "ie", ControlBucket: "deployment", Cloud: hostbox.CloudIdentity{Provider: "aws", Account: "111122223333", Region: "eu-west-1", InstanceID: "i-0abc"}}
898 roster := hostactors.Roster{V: 1, HomeRegion: "ie", DeploymentBucket: actor.ControlBucket, Revision: 1, Boxes: map[string]hostactors.Box{actor.Box: {BoxID: actor.Box, Enrollment: actor.Enrollment, Namespace: actor.Namespace, Label: actor.Box, Region: actor.Region, Pool: "default", Cloud: hostactors.Cloud{Provider: actor.Cloud.Provider, Account: actor.Cloud.Account, Region: actor.Cloud.Region, InstanceID: actor.Cloud.InstanceID}, EvidenceSHA256: strings.Repeat("c", 64), State: "active"}}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}}
899 raw, err := release.SignRootEnvelope(release.RootSigner(key), roster)
900 lifecycleCheck(t, "sign roster", err)
901 _, err = store.Put(t.Context(), hostactors.Key, raw, "")
902 lifecycleCheck(t, "write roster", err)
903 platform := release.AdoptPlatform()
904 pointer := release.SystemPointer{Schema: 1, Component: "router", Channel: "stable", Spread: "0s", PromotedAt: time.Now(), Current: release.SystemRelease{Version: "v2.0.0", ReleaseTag: "v2.0.0", MovesProtocol: 1, Artifacts: map[string]release.SystemArtifact{platform: {Key: release.SystemArtifactKey("v2.0.0", platform), SHA256: strings.Repeat("a", 64)}}}}
905 lifecycleCreate(t, store, key, actor, pointer, provision.Record{Schema: 1, Host: lifecycleHost, Account: "bab", App: "lifecycle", Channel: "stable", Region: "", CreatedAt: time.Now().UTC()})
906 record, err := hostops.ReadRecord(t.Context(), store, provision.RecordKey("bab", "lifecycle", lifecycleHost))
907 lifecycleCheck(t, "read created record", err)
908 if record.Holder.Acquisition == "" {
909 t.Fatal("create did not retain acquisition")
910 }
911}
912
913func lifecycleController(t *testing.T, raw release.Store, key ed25519.PrivateKey, actor hostbox.Actor, pointer release.SystemPointer) (hostops.Controller, *hostops.ProductionAuthority, *hostops.ControlWriter) {
914 t.Helper()
915 ctx := t.Context()
916 signer := release.RootSigner(key)
917 directory := hostactors.Directory{Store: raw, PublicKey: key.Public().(ed25519.PublicKey), Bucket: actor.ControlBucket}
918 floor, err := hostfloor.Read(ctx, raw, directory.PublicKey, actor.ControlBucket)
919 lifecycleCheck(t, "controller floor", err)
920 platform := release.AdoptPlatform()
921 pointer.Current.ConsoleArtifacts = map[string]release.SystemArtifact{platform: {Key: "bootstrap/console/v2.0.0/carlos-" + platform, SHA256: pointer.Current.Artifacts[platform].SHA256}}
922 signed, err := release.SignRootEnvelope(signer, pointer)
923 lifecycleCheck(t, "sign actor release", err)
924 report := hostactors.Report{Component: "router", GeneratedAt: time.Now(), Actor: actor.Box, Enrollment: actor.Enrollment, InstanceID: actor.Cloud.InstanceID, Release: "v2.0.0", ReleaseTag: "v2.0.0", Platform: platform, BinarySHA256: pointer.Current.Artifacts[platform].SHA256, MovesProtocol: 1, BootID: "external-box-boot", FloorSHA256: floor.Digest, FloorGuardReady: true, HostClaimEnabled: true}
925 console := report
926 console.Component, console.Actor, console.InstanceID, console.Enrollment, console.ProcessID = "console", "console", "console", strings.Repeat("d", 32), "console-process"
927 control, err := hostops.NewControlWriter(raw, directory.PublicKey, time.Now)
928 lifecycleCheck(t, "control writer", err)
929 check, err := hostops.NewProductionMutationCheck(raw, control)
930 lifecycleCheck(t, "mutation check", err)
931 records, err := hostops.NewConditionalRecordStore(raw, directory.PublicKey)
932 lifecycleCheck(t, "record boundary", err)
933 data, err := hostops.NewObservedMutationBoundary(true, records, check, nil)
934 lifecycleCheck(t, "mutation boundary", err)
935 directory.Store = data
936 authority := &hostops.ProductionAuthority{HomeRegion: "ie", Store: data, Control: control, Directory: directory, ConsoleID: "console", Enrollment: console.Enrollment, Namespace: actor.Namespace, Signer: signer, Now: time.Now, Evidence: lifecycleActorEvidence{report, console, signed}}
937 controller := hostops.Controller{Store: data, Control: control, Authority: authority, Now: time.Now, NewID: func() string {
938 var id [24]byte
939 if _, err := rand.Read(id[:]); err != nil {
940 t.Fatal(err)
941 }
942 return hex.EncodeToString(id[:])
943 }}
944 return controller, authority, control
945}
946
947func TestPlan14LifecycleControlProcessPayload(t *testing.T) {
948 lifecycleStackTrace(t)
949 if os.Getenv("PLAN14_LIFECYCLE_CONTROL") != "1" {
950 t.Fatal("controller helper")
951 }
952 lifecycleEnvironment(t, os.Getenv("CARLOS_S3_ENDPOINT"))
953 _, gid := generationRunnerRegistryUser(t)
954 store := release.NewDirStore(lifecycleHome + "/deployment")
955 lifecycleControlFaults(t)
956 reg, err := openServiceRegistry("/data/registry.db")
957 lifecycleCheck(t, "registry", err)
958 t.Cleanup(func() { reg.Close() })
959 resources := ClaimRuntimeResources{Mode: "instance-control", Registry: reg, DataDir: "/data", OptDir: "/opt/carlos", UnitsDir: "/etc/systemd/system"}
960 ctx := context.WithValue(t.Context(), claimEntryKey{}, claimEntry{Enabled: true, FlagRead: true, Store: store, Resources: resources})
961 assembly, err := OpenProductionClaimAssembly(ctx, store, resources, lifecycleOwners{t})
962 lifecycleCheck(t, "root assembly", err)
963 t.Cleanup(func() { assembly.Close(context.Background()) })
964 ctx = WithClaimAssembly(ctx, assembly)
965 lifecycleCheck(t, "launch verifier", registerProductionLaunchVerifier())
966 root, err := NewProductionRootClaimStop(ctx)
967 lifecycleCheck(t, "root constructor", err)
968 wrapped, err := WithProductionClaimLaunch(ctx, root)
969 lifecycleCheck(t, "launch constructor", err)
970 rootHTTP := wrapped.(ClaimRootWithLaunch)
971 // lifecycleOwners opens no cleanup enrollment, so production wiring
972 // leaves the stop numeral bound unset (every stop refuses); the fixture
973 // bounds by its identity's roster revision instead.
974 rootHTTP.Root.Stop.Coordinator.VerifiedRosterRevision = func(ctx context.Context) (uint64, error) {
975 actor, err := assembly.Identity.Verify(ctx)
976 return actor.Revision, err
977 }
978 rootHTTP.Root.Stop.Coordinator.Kernel = &lifecycleStopTrace{claimUnitStopKernel: rootHTTP.Root.Stop.Coordinator.Kernel.(claimUnitStopKernel), t: t}
979 rootHTTP.Launch.Kernel = lifecycleKernelTrace{rootHTTP.Launch.Kernel, t}
980 rootHTTP.Launch.Units = lifecycleUnitsTrace{rootHTTP.Launch.Units, t}
981 begin := rootHTTP.Launch.generations.beginUpdate
982 rootHTTP.Launch.generations.beginUpdate = func(ctx context.Context, s *rootGenerationSource, b release.GenerationBinding) (func(string, error, func() error) error, func() error, error) {
983 lifecycleCrash(t, "reservation")
984 finish, close, err := begin(ctx, s, b)
985 t.Log("generation update opener", err)
986 return finish, close, err
987 }
988 t.Cleanup(func() { rootHTTP.Root.executionRuntime.Close(context.Background()) })
989 deadline := time.Now().Add(time.Minute)
990 for {
991 out, _ := exec.Command("systemctl", "show", "carlos-edge.service", "-p", "MainPID", "--value").Output()
992 if strings.TrimSpace(string(out)) != "" && strings.TrimSpace(string(out)) != "0" {
993 break
994 }
995 if time.Now().After(deadline) {
996 t.Fatal("edge startup timeout")
997 }
998 time.Sleep(20 * time.Millisecond)
999 }
1000 if rootHTTP.Root.servingParks != nil {
1001 if p, ok := rootHTTP.Root.servingParks.authority.(*servingParkRoot); ok {
1002 fs := p.ledger.fs.(*servingParkFS)
1003 for _, name := range []string{"namespace-parent.json", "namespace.json", "current-namespace.json"} {
1004 raw, err := p.ledger.state.Read(name)
1005 t.Log("namespace before activation", name, string(raw), err)
1006 }
1007 parent, id, e := fs.parent()
1008 t.Log("namespace current parent", id, e)
1009 if e == nil {
1010 ph, pe := fs.handle(parent)
1011 t.Log("namespace parent proof", ph, pe)
1012 d, did, de := fs.privateDir(parent, ".serving-parked")
1013 t.Log("namespace current root", did, de)
1014 if de == nil {
1015 dh, he := fs.handle(d)
1016 t.Log("namespace root proof", dh, he)
1017 d.Close()
1018 }
1019 parent.Close()
1020 }
1021 }
1022 }
1023 startup, finishStartup := context.WithTimeout(ctx, 2*time.Minute)
1024 defer finishStartup()
1025 if err := activateClaimRoot(ctx, startup, rootHTTP); err != nil {
1026 lifecycleActivationDiagnostic(t, ctx)
1027 lifecycleCheck(t, "production activation gate", err)
1028 }
1029 finishStartup()
1030 listener, err := net.Listen("unix", controlsock.Path)
1031 lifecycleCheck(t, "root socket", err)
1032 lifecycleCheck(t, "root socket mode", os.Chmod(controlsock.Path, 0660))
1033 lifecycleCheck(t, "root socket group", os.Chown(controlsock.Path, 0, gid))
1034 controlCtx, cancelControl := context.WithCancel(ctx)
1035 defer cancelControl()
1036 control := &instancectl.Server{ClaimStop: lifecycleControlTrace{rootHTTP, t}, PeerOK: func(c net.Conn) error { _, err := rootHTTP.Root.Stop.Authenticate(ctx, c); return err }, Logf: t.Logf}
1037 done := make(chan error, 1)
1038 go func() { done <- control.Serve(controlCtx, listener) }()
1039 defer func() { cancelControl(); lifecycleCheck(t, "control shutdown", <-done) }()
1040
1041 lifecycleCheck(t, "controller ready", os.WriteFile("/run/carlos-control/lifecycle-ready", []byte("ready"), 0644))
1042 <-ctx.Done()
1043}
1044
1045func lifecycleTraceWake(t *testing.T, w *ClaimWakeAssembly) {
1046 if old := w.AfterAdmission; old != nil {
1047 w.AfterAdmission = func(ctx context.Context, p, o hostclaim.Observed) error {
1048 e := old(ctx, p, o)
1049 t.Log("after admission", e)
1050 return e
1051 }
1052 }
1053 if old := w.Prelaunch; old != nil {
1054 w.Prelaunch = func(ctx context.Context, o hostclaim.Observed, work hostwrite.Work, in ClaimLaunchInputs) (*activator.RoleLaunch, error) {
1055 l, e := old(ctx, o, work, in)
1056 t.Log("prelaunch", e)
1057 return l, e
1058 }
1059 }
1060
1061 if dispatch, ok := w.Inputs.(ClaimOwnerDispatch); ok {
1062 if serving, ok := dispatch.Serving.Inputs.(*claimServingOwners); ok && serving.emptyRecovery != nil {
1063 check := serving.emptyRecovery
1064 serving.emptyRecovery = func(ctx context.Context, o hostclaim.Observed, a []hostclaim.AdmissionRecord) (bool, error) {
1065 yes, err := check(ctx, o, a)
1066 t.Logf("empty recovery holder=%+v admissions=%+v yes=%v err=%v", o.Claim.Holder, a, yes, err)
1067 return yes, err
1068 }
1069 if f, ok := serving.Sources.Files.(*movesFiles); ok {
1070 f.emptyRecovery = serving.emptyRecovery
1071 }
1072 }
1073 }
1074 if old := w.ServingPrerequisites; old != nil {
1075 w.ServingPrerequisites = func(ctx context.Context, rt registry.Route, o hostclaim.Observed) error {
1076 e := old(ctx, rt, o)
1077 t.Log("files prerequisites", e)
1078 return e
1079 }
1080 }
1081 if old := w.recoveryPrior; old != nil {
1082 w.recoveryPrior = func(ctx context.Context, o hostclaim.Observed) (hostclaim.Observed, bool, error) {
1083 p, b, e := old(ctx, o)
1084 t.Logf("recovery prior=%+v pending=%v error=%v", p.Claim, b, e)
1085 return p, b, e
1086 }
1087 }
1088
1089 recover := w.RecoverArtifacts
1090 if recover != nil {
1091 w.RecoverArtifacts = func(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs) error {
1092 e := recover(ctx, o, in)
1093 t.Log("recover artifacts", e)
1094 return e
1095 }
1096 }
1097 prepare := w.PrepareState
1098 if prepare != nil {
1099 w.PrepareState = func(ctx context.Context, o hostclaim.Observed, work hostwrite.Work, in ClaimLaunchInputs) error {
1100 e := prepare(ctx, o, work, in)
1101 t.Log("prepare state", e)
1102 return e
1103 }
1104 }
1105 before := w.BeforeLegacyInstall
1106 if before != nil {
1107 w.BeforeLegacyInstall = func(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs, work hostwrite.Work) error {
1108 e := before(ctx, o, in, work)
1109 t.Logf("legacy install empty=%v admission=%+v error=%v", in.Empty, in.Admission, e)
1110 return e
1111 }
1112 }
1113 enroll := w.BeforeInstallLocked
1114 if enroll != nil {
1115 w.BeforeInstallLocked = func(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs) error {
1116 e := enroll(ctx, o, in)
1117 t.Log("enroll", e)
1118 return e
1119 }
1120 }
1121}
1122
1123func lifecycleRetry(ctx context.Context, fn func(context.Context) error) error {
1124 bounded, cancel := context.WithTimeout(ctx, 30*time.Second)
1125 defer cancel()
1126 for {
1127 err := fn(bounded)
1128 if err == nil {
1129 return nil
1130 }
1131 select {
1132 case <-bounded.Done():
1133 return errors.Join(err, bounded.Err())
1134 case <-time.After(100 * time.Millisecond):
1135 }
1136 }
1137}
1138