| 1 | //go:build linux && rootrunner |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "crypto/ed25519" |
| 8 | "encoding/json" |
| 9 | "errors" |
| 10 | "os" |
| 11 | "os/exec" |
| 12 | "path/filepath" |
| 13 | "strconv" |
| 14 | "strings" |
| 15 | "testing" |
| 16 | "time" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/hostbox" |
| 19 | "github.com/carlosframework/platform/internal/hostclaim" |
| 20 | execlease "github.com/carlosframework/platform/internal/hostexec" |
| 21 | "github.com/carlosframework/platform/internal/hostops" |
| 22 | "github.com/carlosframework/platform/internal/provision" |
| 23 | "github.com/carlosframework/platform/internal/registry" |
| 24 | "github.com/carlosframework/platform/internal/release" |
| 25 | "github.com/carlosframework/platform/internal/rootparticipation" |
| 26 | "github.com/carlosframework/platform/internal/rootstate" |
| 27 | ) |
| 28 | |
| 29 | func lifecycleDeleteDriver(t *testing.T, phase string, raw release.Store, key ed25519.PrivateKey, actor hostbox.Actor, pointer release.SystemPointer) func() { |
| 30 | t.Helper() |
| 31 | var operation *hostops.OperationHandle |
| 32 | var controller hostops.Controller |
| 33 | ctx := t.Context() |
| 34 | serial := 0 |
| 35 | return func() { |
| 36 | if phase != "2" && phase != "crash" { |
| 37 | t.Fatal("unexpected deletion before reboot") |
| 38 | } |
| 39 | if operation == nil { |
| 40 | var control *hostops.ControlWriter |
| 41 | controller, _, control = lifecycleController(t, raw, key, actor, pointer) |
| 42 | admission, err := hostops.ReadCanonicalAdmission(ctx, raw, "bab", "lifecycle") |
| 43 | lifecycleCheck(t, "delete admission", err) |
| 44 | operation, err = control.BeginExclusive(ctx, "bab", "lifecycle", admission.Value.AppID, hostops.AppOperation{ID: "delete-lifecycle", Kind: hostops.OpDeleting, AppID: admission.Value.AppID, Intent: map[string]string{"host": lifecycleHost}}, hostops.RegistrationOwner{Console: "console", Enrollment: strings.Repeat("d", 32), Controller: "console-controller"}) |
| 45 | lifecycleCheck(t, "exclusive deletion", err) |
| 46 | ctx = operation.Lifecycle(ctx) |
| 47 | } |
| 48 | obs, err := hostclaim.Read(ctx, controller.Store, lifecycleHost) |
| 49 | lifecycleCheck(t, "delete observation", err) |
| 50 | if obs.Claim.Free() { |
| 51 | return |
| 52 | } |
| 53 | switch obs.Claim.Op.Kind { |
| 54 | case hostclaim.KindServing: |
| 55 | _, err = controller.BeginRetire(ctx, obs.Claim.Holder.Key, "runner", obs.Claim.Holder.Incarnation) |
| 56 | case hostclaim.KindDeleting: |
| 57 | serial++ |
| 58 | _, err = operation.Runner(time.Now).Run(ctx, hostops.Effect{ID: "retire-" + strconv.Itoa(serial), Resource: "record:" + lifecycleHost, Action: "retire"}, func(call context.Context) (string, error) { |
| 59 | e := controller.Retire(call, lifecycleHost, obs.Claim.Op.ID) |
| 60 | if errors.Is(e, hostops.ErrPending) { |
| 61 | current, rerr := hostclaim.Read(call, controller.Store, lifecycleHost) |
| 62 | if rerr == nil && current.Claim.Op.Kind == hostclaim.KindParking { |
| 63 | return "parked", nil |
| 64 | } |
| 65 | return "", errors.Join(hostops.ErrNotApplied, e) |
| 66 | } |
| 67 | return "retired", e |
| 68 | }) |
| 69 | case hostclaim.KindParking: |
| 70 | if len(obs.Claim.Obligations) == 0 { |
| 71 | err = controller.Free(ctx, lifecycleHost, obs.Claim.Op.ID) |
| 72 | } |
| 73 | default: |
| 74 | t.Fatal("unexpected retirement phase", obs.Claim.Op.Kind) |
| 75 | } |
| 76 | if err != nil && !errors.Is(err, hostops.ErrPending) && !errors.Is(err, hostops.ErrNotApplied) { |
| 77 | lifecycleCheck(t, "delete transition", err) |
| 78 | } |
| 79 | } |
| 80 | } |
| 81 | |
| 82 | func lifecycleVerifyRetiredPins(t *testing.T, store release.Store, key ed25519.PrivateKey) { |
| 83 | t.Helper() |
| 84 | ctx := t.Context() |
| 85 | lifecycleActiveSibling(t) |
| 86 | recordKey := provision.RecordKey("bab", "lifecycle", lifecycleHost) |
| 87 | _, _, err := store.Get(ctx, recordKey) |
| 88 | if !errors.Is(err, os.ErrNotExist) { |
| 89 | t.Fatal("deleted record retained", err) |
| 90 | } |
| 91 | settleDeadline := time.Now().Add(time.Minute) |
| 92 | for { |
| 93 | raw, e := os.ReadFile(filepath.Join(rootparticipation.Namespace, lifecycleHost+".json")) |
| 94 | lifecycleCheck(t, "final participation record", e) |
| 95 | var record execlease.ExecutionRecord |
| 96 | lifecycleCheck(t, "final participation decode", json.Unmarshal(raw, &record)) |
| 97 | if record.Status == "settled" { |
| 98 | break |
| 99 | } |
| 100 | if time.Now().After(settleDeadline) { |
| 101 | t.Fatalf("retired participation remains %s", record.Status) |
| 102 | } |
| 103 | time.Sleep(100 * time.Millisecond) |
| 104 | } |
| 105 | journals, err := hostbox.SavedRetirementJournals(claimRetireJournalDir("", "")) |
| 106 | lifecycleCheck(t, "final retirement journals", err) |
| 107 | if len(journals) == 0 { |
| 108 | t.Fatal("retirement has no durable journal") |
| 109 | } |
| 110 | for _, j := range journals { |
| 111 | if j.Inventory.Host != lifecycleHost || !j.Complete { |
| 112 | t.Fatal("retirement journal incomplete", j.ID) |
| 113 | } |
| 114 | for _, step := range j.Steps { |
| 115 | if !step.Done { |
| 116 | t.Fatal("retirement artifact incomplete", step.Artifact) |
| 117 | } |
| 118 | if step.Artifact.Kind == hostbox.ArtifactPath { |
| 119 | a := step.Artifact |
| 120 | if _, e := os.Lstat(filepath.Join("/data", a.Name)); !errors.Is(e, os.ErrNotExist) { |
| 121 | t.Fatal("retired source remains", e) |
| 122 | } |
| 123 | parked := filepath.Join(".parked", j.Inventory.Captured.Incarnation, lifecycleHost, j.ID, a.Root, a.Name) |
| 124 | id, e := hostbox.CaptureInventoryIdentity(a.Identity.MountPoint, parked) |
| 125 | t.Logf("parked mount evidence before=%+v after=%+v", a.Identity, id) |
| 126 | if e != nil || !a.Identity.Same(id) { |
| 127 | t.Fatal("parked artifact lost its recorded mount or identity", e) |
| 128 | } |
| 129 | } |
| 130 | } |
| 131 | } |
| 132 | source := productionRootGenerationPins().(generationPinSource) |
| 133 | retire := source.retire |
| 134 | source.retire = func(ctx context.Context, g *release.RootGenerations) error { |
| 135 | err := retire(ctx, g) |
| 136 | t.Log("pins retirement", err) |
| 137 | if err != nil { |
| 138 | lifecycleRetirementDiagnostic(t, ctx, g) |
| 139 | lifecycleCrashDiagnostic(t) |
| 140 | } |
| 141 | return err |
| 142 | } |
| 143 | reconcile := source.reconcile |
| 144 | source.reconcile = func(ctx context.Context, r []release.GenerationReservation) (release.GenerationEvidence, error) { |
| 145 | e, err := reconcile(ctx, r) |
| 146 | t.Log("pins reconciliation", err) |
| 147 | if err != nil { |
| 148 | inv, readErr := generationInventoryAt(ctx, rootparticipation.Namespace, "/var/lib/carlos-claims/authority/fences", "/var/lib/carlos-claims/journals.db") |
| 149 | t.Log("pins raw inventory", readErr) |
| 150 | inv.Handovers, readErr = productionGenerationHandovers(ctx) |
| 151 | t.Log("pins handovers", readErr) |
| 152 | inv.Retirements, readErr = productionGenerationRetirements(ctx) |
| 153 | t.Log("pins retirements", readErr) |
| 154 | _, readErr = generationRetirements(ctx, inv) |
| 155 | t.Log("pins retirement map", readErr) |
| 156 | _, readErr = productionGenerationRegistryInventory(ctx, inv) |
| 157 | t.Log("pins registry", readErr) |
| 158 | lifecycleInventoryDiagnostic(t, ctx, inv, r) |
| 159 | } |
| 160 | return e, err |
| 161 | } |
| 162 | lock, err := rootstate.Open(rootReleasePrivate, false, false) |
| 163 | lifecycleCheck(t, "release lock", err) |
| 164 | defer lock.Close() |
| 165 | unknown := "plan14-pin-unknown.service" |
| 166 | stopUnknown := func() { exec.Command("systemctl", "stop", unknown).Run() } |
| 167 | t.Cleanup(stopUnknown) |
| 168 | out, err := exec.Command("systemd-run", "--unit="+unknown, "--slice=carlos-instances.slice", "--property=Type=exec", "/bin/sleep", "60").CombinedOutput() |
| 169 | lifecycleCheck(t, "unrecorded containment probe: "+string(out), err) |
| 170 | if err := rootstate.WithRelease(ctx, lock, func() error { _, err := source.StagedPins(ctx, release.OptDir()); return err }); err == nil { |
| 171 | t.Fatal("unrecorded managed process did not prevent generation certification") |
| 172 | } |
| 173 | stopUnknown() |
| 174 | lifecycleCheck(t, "production pins after actual retirement", rootstate.WithRelease(ctx, lock, func() error { |
| 175 | pins, err := source.StagedPins(ctx, release.OptDir()) |
| 176 | if err != nil { |
| 177 | return err |
| 178 | } |
| 179 | pinned, err := pins.Refers(filepath.Dir(release.VersionPath(release.OptDir(), "bab", "v1", "lifecycle"))) |
| 180 | if err != nil { |
| 181 | return err |
| 182 | } |
| 183 | if !pinned { |
| 184 | t.Fatal("active sibling failed to pin the shared generation tree") |
| 185 | } |
| 186 | sibling, err := pins.Refers(release.VersionPath(release.OptDir(), "bab", "v1", "sibling")) |
| 187 | if err != nil || !sibling { |
| 188 | t.Fatal("active sibling executable is not pinned", err) |
| 189 | } |
| 190 | retired, err := pins.Refers(release.VersionPath(release.OptDir(), "bab", "v1", "lifecycle")) |
| 191 | if err != nil || retired { |
| 192 | t.Fatal("retired executable still has a direct pin", err) |
| 193 | } |
| 194 | return nil |
| 195 | })) |
| 196 | lifecycleActiveSibling(t) |
| 197 | body := []byte("signed sibling") |
| 198 | manifest := release.Manifest{App: "sibling", Version: "v9", Kind: "binary", Artifacts: map[string]release.Artifact{release.AdoptPlatform(): {}}} |
| 199 | lifecycleCheck(t, "ship sibling", release.Ship(ctx, store, "bab", manifest, map[string][]byte{release.AdoptPlatform(): body})) |
| 200 | lifecycleCheck(t, "promote sibling", release.Promote(ctx, store, key, "bab", "sibling", "stable", "v9")) |
| 201 | lifecycleCheck(t, "stage sibling", release.StageArtifact(release.OptDir(), "bab", "v9", "sibling", body)) |
| 202 | path := release.StagedManifestPath(release.OptDir(), "bab", "v9", "sibling") |
| 203 | lifecycleCheck(t, "damage sibling manifest", os.WriteFile(path, []byte("broken sibling manifest"), 0644)) |
| 204 | lifecycleCheck(t, "repair sibling with production pins", rootstate.WithRelease(ctx, lock, func() error { |
| 205 | proof := release.NewRootProofGate(filepath.Join(t.TempDir(), "policy.json"), filepath.Join(t.TempDir(), "bakes")) |
| 206 | _, e := release.StageHeldVerifiedWithPins(ctx, store, key.Public().(ed25519.PublicKey), nil, release.OptDir(), registry.Route{Host: "sibling.example", Account: "bab", App: "sibling", Kind: "instance", Channel: "stable", Status: "landing", HoldOp: "repair"}, "v9", proof, source) |
| 207 | return e |
| 208 | })) |
| 209 | repaired, err := os.ReadFile(path) |
| 210 | if err != nil || string(repaired) == "broken sibling manifest" { |
| 211 | t.Fatal("sibling repair failed", err) |
| 212 | } |
| 213 | _, _, err = store.Get(ctx, recordKey) |
| 214 | if !errors.Is(err, os.ErrNotExist) { |
| 215 | t.Fatal("record resurrected", err) |
| 216 | } |
| 217 | } |
| 218 | |
amadan