| 1 | //go:build linux && rootrunner |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "crypto/sha256" |
| 8 | "encoding/hex" |
| 9 | "encoding/json" |
| 10 | "errors" |
| 11 | "fmt" |
| 12 | "os" |
| 13 | "path/filepath" |
| 14 | "reflect" |
| 15 | "testing" |
| 16 | |
| 17 | "github.com/carlosframework/platform/internal/hostbox" |
| 18 | "github.com/carlosframework/platform/internal/hostguard" |
| 19 | "golang.org/x/sys/unix" |
| 20 | ) |
| 21 | |
| 22 | const inventoryRebootRoot = "/var/lib/carlos-e53-inventory" |
| 23 | |
| 24 | type inventoryRebootFixture struct { |
| 25 | Inventory hostbox.StopInventory |
| 26 | File, Link, Retry hostbox.Artifact |
| 27 | Journal hostbox.RetirementJournal |
| 28 | } |
| 29 | |
| 30 | func inventoryRebootParker() hostbox.TypedParker { |
| 31 | return hostbox.TypedParker{Roots: map[string]string{"data": inventoryRebootRoot}, ParkedRoot: inventoryRebootRoot, Incarnation: "inc", Host: "inventory.example", Retirement: "retire"} |
| 32 | } |
| 33 | |
| 34 | func TestRootRebootInventoryIdentityPhase1(t *testing.T) { |
| 35 | if os.Geteuid() != 0 { |
| 36 | t.Fatal("requires root") |
| 37 | } |
| 38 | if err := os.Mkdir(inventoryRebootRoot, 0700); err != nil { |
| 39 | t.Fatal(err) |
| 40 | } |
| 41 | for _, dir := range []string{"instances", "instances/inventory.example"} { |
| 42 | if err := os.Mkdir(filepath.Join(inventoryRebootRoot, dir), 0700); err != nil { |
| 43 | t.Fatal(err) |
| 44 | } |
| 45 | } |
| 46 | for _, name := range []string{"db", "retry"} { |
| 47 | if err := os.WriteFile(filepath.Join(inventoryRebootRoot, name), []byte(name), 0600); err != nil { |
| 48 | t.Fatal(err) |
| 49 | } |
| 50 | } |
| 51 | if err := os.Symlink("db", filepath.Join(inventoryRebootRoot, "live")); err != nil { |
| 52 | t.Fatal(err) |
| 53 | } |
| 54 | p := inventoryRebootParker() |
| 55 | f := inventoryRebootFixture{Inventory: hostbox.StopInventory{Complete: true, Host: "inventory.example", Incarnation: "inc", InstallationStamp: 1, Box: "box", Enrollment: "enrollment", DirectoryRevision: "1"}} |
| 56 | for _, v := range []struct { |
| 57 | name string |
| 58 | kind hostbox.ArtifactKind |
| 59 | out *hostbox.Artifact |
| 60 | }{{"db", hostbox.ArtifactPath, &f.File}, {"live", hostbox.ArtifactLiveLink, &f.Link}, {"retry", hostbox.ArtifactPath, &f.Retry}} { |
| 61 | var err error |
| 62 | *v.out, err = p.CaptureArtifact("data", v.name, "owner", v.kind) |
| 63 | if err != nil { |
| 64 | t.Fatal(err) |
| 65 | } |
| 66 | if v.out.Identity.Handle == "" { |
| 67 | t.Fatal("persistent handle required on runner") |
| 68 | } |
| 69 | } |
| 70 | // A lost reply after rename must verify the destination after reboot. |
| 71 | if err := p.Park(t.Context(), "retire", f.Retry); err != nil { |
| 72 | t.Fatal(err) |
| 73 | } |
| 74 | id, err := hostbox.CaptureInventoryIdentity(inventoryRebootRoot, "instances/inventory.example") |
| 75 | if err != nil { |
| 76 | t.Fatal(err) |
| 77 | } |
| 78 | owner := sha256.Sum256([]byte(fmt.Sprintf("%s\x00%s\x00%d\x00%s", f.Inventory.Host, f.Inventory.Incarnation, f.Inventory.InstallationStamp, "box"))) |
| 79 | f.Inventory.Entries = []hostbox.StopInventoryEntry{{Kind: "path", ManagedRoot: "data", RelativePath: "instances/inventory.example", ArtifactID: "state", Ownership: hex.EncodeToString(owner[:]), Materialization: "materialized", Device: id.Device, Inode: id.Inode, Type: unix.S_IFDIR, Identity: id}} |
| 80 | db, err := openRebootInventory(t, filepath.Join(inventoryRebootRoot, "inventory.db")) |
| 81 | if err != nil { |
| 82 | t.Fatal(err) |
| 83 | } |
| 84 | if err := db.WriteStopInventory(t.Context(), f.Inventory); err != nil { |
| 85 | t.Fatal(err) |
| 86 | } |
| 87 | if err := db.Close(); err != nil { |
| 88 | t.Fatal(err) |
| 89 | } |
| 90 | f.Journal, err = hostbox.NewRetirementJournal("retire", hostbox.RetirementInventory{Host: "inventory.example", Captured: hostguard.Captured{Incarnation: "inc", Stamp: 1}, Ownership: "owner", Artifacts: []hostbox.Artifact{f.File, f.Link, f.Retry}}) |
| 91 | if err != nil { |
| 92 | t.Fatal(err) |
| 93 | } |
| 94 | target := hostguard.FenceTarget{Host: "inventory.example", Incarnation: "inc", Stamp: 1} |
| 95 | retirement := inventoryRebootRetirement(p) |
| 96 | retirement.Filesystem = inventoryRebootInterrupt{p} |
| 97 | if err := retirement.Run(t.Context(), target, f.Journal); !errors.Is(err, os.ErrClosed) { |
| 98 | t.Fatal("retirement interruption", err) |
| 99 | } |
| 100 | raw, err := json.Marshal(f) |
| 101 | if err != nil { |
| 102 | t.Fatal(err) |
| 103 | } |
| 104 | if err := writeFileDurable(filepath.Join(inventoryRebootRoot, "fixture.json"), raw, 0600); err != nil { |
| 105 | t.Fatal(err) |
| 106 | } |
| 107 | t.Logf("saved boot=%s device=%d mount=%d fs=%s", id.Boot, id.Device, id.Mount, id.Filesystem) |
| 108 | } |
| 109 | |
| 110 | func TestRootRebootInventoryIdentityPhase2(t *testing.T) { |
| 111 | if os.Geteuid() != 0 { |
| 112 | t.Fatal("requires root") |
| 113 | } |
| 114 | defer os.RemoveAll(inventoryRebootRoot) |
| 115 | raw, err := os.ReadFile(filepath.Join(inventoryRebootRoot, "fixture.json")) |
| 116 | var f inventoryRebootFixture |
| 117 | if err != nil || json.Unmarshal(raw, &f) != nil { |
| 118 | t.Fatal("load", err) |
| 119 | } |
| 120 | db, err := openRebootInventory(t, filepath.Join(inventoryRebootRoot, "inventory.db")) |
| 121 | if err != nil { |
| 122 | t.Fatal(err) |
| 123 | } |
| 124 | defer db.Close() |
| 125 | snapshot, revision, err := db.Snapshot(t.Context(), f.Inventory.Host, hostguard.Captured{Incarnation: "inc", Stamp: 1}) |
| 126 | if err != nil || revision != 1 || !reflect.DeepEqual(snapshot, f.Inventory) { |
| 127 | t.Fatal("persistent inventory snapshot", err) |
| 128 | } |
| 129 | reader, err := hostbox.OpenGenerationLaunchReader(filepath.Join(inventoryRebootRoot, "inventory.db")) |
| 130 | if err != nil { |
| 131 | t.Fatal(err) |
| 132 | } |
| 133 | defer reader.Close() |
| 134 | payloads, err := reader.GenerationPayloads(t.Context()) |
| 135 | if err != nil || len(payloads) != 1 || !reflect.DeepEqual(payloads[0].Inventory, f.Inventory) { |
| 136 | t.Fatal("persistent payload", err) |
| 137 | } |
| 138 | e := f.Inventory.Entries[0] |
| 139 | now, err := hostbox.CaptureInventoryIdentity(inventoryRebootRoot, e.RelativePath) |
| 140 | if err != nil || now.Boot == e.Identity.Boot { |
| 141 | t.Fatal("requires actual reboot", err) |
| 142 | } |
| 143 | t.Logf("new boot=%s device=%d mount=%d fs=%s", now.Boot, now.Device, now.Mount, now.Filesystem) |
| 144 | // Force renumbered values too, so a runner retaining its numbers cannot |
| 145 | // accidentally make a raw-device comparison pass this acceptance. |
| 146 | e.Device += 1000 |
| 147 | e.Identity.Device = e.Device |
| 148 | e.Identity.Mount += 1000 |
| 149 | f.Inventory.Entries[0] = e |
| 150 | h := ClaimRootHTTP{Roots: map[string]string{"data": inventoryRebootRoot}} |
| 151 | a := ClaimRoleAssembly{ManagedRoots: h.Roots} |
| 152 | check := func() error { |
| 153 | if err := h.verifyInventoryPath(e); err != nil { |
| 154 | return fmt.Errorf("root inventory: %w", err) |
| 155 | } |
| 156 | if _, _, err := installationCoverage(h.Roots, f.Inventory, false); err != nil { |
| 157 | return fmt.Errorf("stop coverage: %w", err) |
| 158 | } |
| 159 | if err := generationPayloadFiles(t.Context(), inventoryRebootRoot, []hostbox.GenerationPayload{{Inventory: f.Inventory, Revision: 1}}); err != nil { |
| 160 | return fmt.Errorf("payload: %w", err) |
| 161 | } |
| 162 | changed, err := a.prepareStateInventory(ClaimLaunchInputs{DBPath: filepath.Join(inventoryRebootRoot, e.RelativePath, "app.db")}, &f.Inventory, hostbox.Actor{Box: "box"}) |
| 163 | if changed { |
| 164 | return fmt.Errorf("rewrote retained inventory") |
| 165 | } |
| 166 | return err |
| 167 | } |
| 168 | if err := check(); err != nil { |
| 169 | t.Fatal(err) |
| 170 | } |
| 171 | if err := h.verifyNewInventoryIdentities(hostbox.StopInventory{}, f.Inventory); err == nil { |
| 172 | t.Error("old-boot evidence accepted as new inventory") |
| 173 | } |
| 174 | path := filepath.Join(inventoryRebootRoot, e.RelativePath) |
| 175 | if err := unix.Mount(path, path, "", unix.MS_BIND, ""); err != nil { |
| 176 | t.Fatal(err) |
| 177 | } |
| 178 | if err := check(); err == nil { |
| 179 | t.Error("accepted substituted mount") |
| 180 | } |
| 181 | if err := unix.Unmount(path, 0); err != nil { |
| 182 | t.Fatal(err) |
| 183 | } |
| 184 | if err := os.Chown(path, 65534, 65534); err != nil { |
| 185 | t.Fatal(err) |
| 186 | } |
| 187 | if err := check(); err == nil { |
| 188 | t.Error("accepted changed ownership") |
| 189 | } |
| 190 | if err := os.Chown(path, int(e.Identity.UID), int(e.Identity.GID)); err != nil { |
| 191 | t.Fatal(err) |
| 192 | } |
| 193 | if err := unix.Mount("tmpfs", path, "tmpfs", 0, "mode=0700"); err != nil { |
| 194 | t.Fatal(err) |
| 195 | } |
| 196 | if err := check(); err == nil { |
| 197 | t.Error("accepted different filesystem") |
| 198 | } |
| 199 | if err := unix.Unmount(path, 0); err != nil { |
| 200 | t.Fatal(err) |
| 201 | } |
| 202 | if err := check(); err != nil { |
| 203 | t.Fatal("restored original", err) |
| 204 | } |
| 205 | if err := os.Chown(inventoryRebootRoot, 65534, 65534); err != nil { |
| 206 | t.Fatal(err) |
| 207 | } |
| 208 | if err := check(); err == nil { |
| 209 | t.Error("accepted untrusted root ancestry") |
| 210 | } |
| 211 | if err := hostbox.VerifyInventoryIdentityForOwner(inventoryRebootRoot, e, 65534); err != nil { |
| 212 | t.Fatal("explicit managed-root owner refused after reboot", err) |
| 213 | } |
| 214 | if err := hostbox.VerifyInventoryIdentityForOwner(inventoryRebootRoot, e, 65533); err == nil { |
| 215 | t.Fatal("wrong managed-root owner accepted") |
| 216 | } |
| 217 | if err := os.Chmod(inventoryRebootRoot, 0777); err != nil { |
| 218 | t.Fatal(err) |
| 219 | } |
| 220 | if err := hostbox.VerifyInventoryIdentityForOwner(inventoryRebootRoot, e, 65534); err == nil { |
| 221 | t.Fatal("writable managed-root accepted") |
| 222 | } |
| 223 | if err := os.Chmod(inventoryRebootRoot, 0700); err != nil { |
| 224 | t.Fatal(err) |
| 225 | } |
| 226 | if err := os.Chown(inventoryRebootRoot, 0, 0); err != nil { |
| 227 | t.Fatal(err) |
| 228 | } |
| 229 | |
| 230 | p := inventoryRebootParker() |
| 231 | for _, v := range []*hostbox.Artifact{&f.File, &f.Link, &f.Retry} { |
| 232 | v.Dev += 1000 |
| 233 | v.Identity.Device = v.Dev |
| 234 | v.Identity.Mount += 1000 |
| 235 | } |
| 236 | // Existing-file replacement must survive refused retirement. |
| 237 | if err := os.Rename(filepath.Join(inventoryRebootRoot, "db"), filepath.Join(inventoryRebootRoot, "saved")); err != nil { |
| 238 | t.Fatal(err) |
| 239 | } |
| 240 | if err := os.WriteFile(filepath.Join(inventoryRebootRoot, "db"), []byte("replacement"), 0600); err != nil { |
| 241 | t.Fatal(err) |
| 242 | } |
| 243 | if err := p.Park(t.Context(), "retire", f.File); err == nil { |
| 244 | t.Fatal("parked replacement") |
| 245 | } |
| 246 | if raw, err := os.ReadFile(filepath.Join(inventoryRebootRoot, "db")); err != nil || string(raw) != "replacement" { |
| 247 | t.Fatal("replacement changed", err) |
| 248 | } |
| 249 | if err := os.Remove(filepath.Join(inventoryRebootRoot, "db")); err != nil { |
| 250 | t.Fatal(err) |
| 251 | } |
| 252 | if err := os.Rename(filepath.Join(inventoryRebootRoot, "saved"), filepath.Join(inventoryRebootRoot, "db")); err != nil { |
| 253 | t.Fatal(err) |
| 254 | } |
| 255 | for _, v := range []hostbox.Artifact{f.File, f.Link, f.Retry} { |
| 256 | if err := p.Park(context.Background(), "retire", v); err != nil { |
| 257 | t.Fatalf("park %s: %v", v.Name, err) |
| 258 | } |
| 259 | if err := p.Park(context.Background(), "retire", v); err != nil { |
| 260 | t.Fatalf("replay %s: %v", v.Name, err) |
| 261 | } |
| 262 | } |
| 263 | retirement := inventoryRebootRetirement(p) |
| 264 | target := hostguard.FenceTarget{Host: "inventory.example", Incarnation: "inc", Stamp: 1} |
| 265 | if err := retirement.Run(t.Context(), target, f.Journal); err != nil { |
| 266 | t.Fatal("durable retirement recovery", err) |
| 267 | } |
| 268 | if err := retirement.Run(t.Context(), target, f.Journal); err != nil { |
| 269 | t.Fatal("durable retirement replay", err) |
| 270 | } |
| 271 | if target, err := os.Readlink(filepath.Join(inventoryRebootRoot, ".parked/inc/inventory.example/retire/data/live")); err != nil || target != "db" { |
| 272 | t.Fatal("link changed", err) |
| 273 | } |
| 274 | if retained, err := os.ReadFile(filepath.Join(inventoryRebootRoot, "fixture.json")); err != nil || string(retained) != string(raw) { |
| 275 | t.Fatal("rewrote durable evidence", err) |
| 276 | } |
| 277 | } |
| 278 | |
| 279 | // Cleanup authority is a fixture; persistence, inode checks and journal replay |
| 280 | // use the production implementations across the real reboot. |
| 281 | type inventoryRebootAuthority struct{} |
| 282 | |
| 283 | func (inventoryRebootAuthority) WithCleanup(_ context.Context, _ hostguard.FenceTarget, fn func() error) error { |
| 284 | return fn() |
| 285 | } |
| 286 | |
| 287 | type inventoryRebootRows struct{} |
| 288 | |
| 289 | func (inventoryRebootRows) RetireRows(context.Context, string, []hostbox.Artifact) error { |
| 290 | return os.ErrPermission |
| 291 | } |
| 292 | |
| 293 | type inventoryRebootInterrupt struct{ parker hostbox.TypedParker } |
| 294 | |
| 295 | func (p inventoryRebootInterrupt) Park(ctx context.Context, id string, a hostbox.Artifact) error { |
| 296 | if a.Name != "retry" { |
| 297 | return os.ErrClosed |
| 298 | } |
| 299 | return p.parker.Park(ctx, id, a) |
| 300 | } |
| 301 | func inventoryRebootRetirement(p hostbox.TypedParker) hostbox.TypedRetirement { |
| 302 | return hostbox.TypedRetirement{JournalDir: filepath.Join(inventoryRebootRoot, "retirements"), Authority: inventoryRebootAuthority{}, Filesystem: p, Rows: inventoryRebootRows{}, Ack: func(_ context.Context, j hostbox.RetirementJournal) error { |
| 303 | if !j.Complete { |
| 304 | return os.ErrPermission |
| 305 | } |
| 306 | return nil |
| 307 | }} |
| 308 | } |
| 309 | |
amadan