| 1 | //go:build linux |
| 2 | |
| 3 | package hostbox |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "encoding/json" |
| 8 | "errors" |
| 9 | "os" |
| 10 | "os/exec" |
| 11 | "path/filepath" |
| 12 | "testing" |
| 13 | "time" |
| 14 | |
| 15 | "github.com/carlosframework/platform/internal/hostclaim" |
| 16 | "github.com/carlosframework/platform/internal/hostguard" |
| 17 | "github.com/carlosframework/platform/internal/registry" |
| 18 | "github.com/carlosframework/platform/internal/release" |
| 19 | "github.com/carlosframework/platform/internal/rootstate" |
| 20 | ) |
| 21 | |
| 22 | type plan14ColdManifest struct { |
| 23 | Base, Data, Locks string |
| 24 | Prior hostguard.FenceTarget |
| 25 | Next ColdSuccessor |
| 26 | } |
| 27 | type plan14ColdDisk struct { |
| 28 | rootstate.Store |
| 29 | dir string |
| 30 | before, after func([]byte) |
| 31 | } |
| 32 | |
| 33 | func (d *plan14ColdDisk) Read(n string) ([]byte, error) { return os.ReadFile(filepath.Join(d.dir, n)) } |
| 34 | func (d *plan14ColdDisk) Write(n string, b []byte) error { |
| 35 | if d.before != nil { |
| 36 | d.before(b) |
| 37 | } |
| 38 | f, err := os.CreateTemp(d.dir, "pending-") |
| 39 | if err != nil { |
| 40 | return err |
| 41 | } |
| 42 | defer os.Remove(f.Name()) |
| 43 | _, err = f.Write(b) |
| 44 | if err == nil { |
| 45 | err = f.Sync() |
| 46 | } |
| 47 | err = errors.Join(err, f.Close()) |
| 48 | if err != nil { |
| 49 | return err |
| 50 | } |
| 51 | if err = os.Rename(f.Name(), filepath.Join(d.dir, n)); err != nil { |
| 52 | return err |
| 53 | } |
| 54 | parent, err := os.Open(d.dir) |
| 55 | if err != nil { |
| 56 | return err |
| 57 | } |
| 58 | err = errors.Join(parent.Sync(), parent.Close()) |
| 59 | if err != nil { |
| 60 | return err |
| 61 | } |
| 62 | if d.after != nil { |
| 63 | d.after(b) |
| 64 | } |
| 65 | return nil |
| 66 | } |
| 67 | func (d *plan14ColdDisk) Names() ([]string, error) { |
| 68 | es, err := os.ReadDir(d.dir) |
| 69 | var names []string |
| 70 | for _, e := range es { |
| 71 | names = append(names, e.Name()) |
| 72 | } |
| 73 | return names, err |
| 74 | } |
| 75 | func (d *plan14ColdDisk) Lock(context.Context, string) (func(), error) { return func() {}, nil } |
| 76 | func (d *plan14ColdDisk) Close() error { return nil } |
| 77 | func plan14ColdSetup(t *testing.T) plan14ColdManifest { |
| 78 | t.Setenv("CARLOS_HOST_CLAIM", "1") |
| 79 | source, store, initial := workFixture(t) |
| 80 | ctx := t.Context() |
| 81 | base := t.TempDir() |
| 82 | rootDir := filepath.Join(base, "root") |
| 83 | if err := os.Mkdir(rootDir, 0700); err != nil { |
| 84 | t.Fatal(err) |
| 85 | } |
| 86 | disk := &plan14ColdDisk{dir: rootDir} |
| 87 | admit := func(old hostclaim.Observed, generation string) hostclaim.Observed { |
| 88 | next, err := hostclaim.Update(ctx, store, old, hostclaim.AdmitWriter(hostclaim.Admission{ObservedOpID: old.Claim.Op.ID, ObservedPhase: old.Claim.Op.Phase, ObservedSession: old.Claim.Holder.Session, ObservedGeneration: old.Claim.Holder.Generation, Box: "box", Region: "ie", Mode: "cold", NewGeneration: generation})) |
| 89 | if err != nil { |
| 90 | t.Fatal(err) |
| 91 | } |
| 92 | return next |
| 93 | } |
| 94 | old := admit(initial, "first") |
| 95 | if err := source.Guard.Activate(ctx, old.Host, "box", nil); err != nil { |
| 96 | t.Fatal(err) |
| 97 | } |
| 98 | source.Guard.Root = hostguard.NewRootAuthority(disk) |
| 99 | holder := old.Claim.Holder |
| 100 | if err := source.Guard.EstablishRoot(ctx, hostguard.RootInstallation{Host: old.Host, Incarnation: holder.Incarnation, Stamp: holder.Stamp, Account: holder.Account, App: holder.App, Record: holder.Key, Box: "box", Enrollment: "enrollment", Migration: 1, InventoryRevision: 1}, func(*hostclaim.Claim) error { return nil }); err != nil { |
| 101 | t.Fatal(err) |
| 102 | } |
| 103 | prior := hostguard.FenceTarget{FenceID: "fence", Region: "ie", Host: old.Host, Incarnation: old.Claim.Holder.Incarnation, Stamp: old.Claim.Holder.Stamp, Session: old.Claim.Holder.Session, Box: "box", Enrollment: "enrollment", BootID: "boot", ControllerID: "controller", ClaimETag: old.ETag, DirectoryRevision: "1", Op: old.Claim.Op.ID, Phase: old.Claim.Op.Phase} |
| 104 | if _, err := source.Guard.BeginFence(ctx, prior); err != nil { |
| 105 | t.Fatal(err) |
| 106 | } |
| 107 | if err := source.Guard.CompleteFence(ctx, prior, []byte("verified-root-receipt"), time.Now().UTC(), func(context.Context, hostguard.FenceTarget, []byte, time.Time) error { return nil }); err != nil { |
| 108 | t.Fatal(err) |
| 109 | } |
| 110 | reg, err := registry.Open(filepath.Join(base, "registry.db")) |
| 111 | if err != nil { |
| 112 | t.Fatal(err) |
| 113 | } |
| 114 | t.Cleanup(func() { reg.Close() }) |
| 115 | inventory, err := OpenStopInventoryDB(filepath.Join(base, "inventory.db")) |
| 116 | if err != nil { |
| 117 | t.Fatal(err) |
| 118 | } |
| 119 | t.Cleanup(func() { inventory.Close() }) |
| 120 | launches, err := OpenLaunchJournal(filepath.Join(base, "launches.db")) |
| 121 | if err != nil { |
| 122 | t.Fatal(err) |
| 123 | } |
| 124 | t.Cleanup(func() { launches.Close() }) |
| 125 | owner := registry.ClaimOwnership{ClaimIncarnation: prior.Incarnation, ClaimStamp: prior.Stamp, ClaimOwner: "old-owner", ClaimRowID: "old-route"} |
| 126 | dbPath := filepath.Join(source.Guard.DataDir, old.Host+".db") |
| 127 | if err = os.WriteFile(dbPath, []byte("untrusted-local-database"), 0600); err != nil { |
| 128 | t.Fatal(err) |
| 129 | } |
| 130 | if err = reg.Upsert(registry.Route{Host: old.Host, Kind: "instance", Socket: "/run/example.sock", DBPath: dbPath, ClaimOwnership: owner, HoldOp: "held-operation"}); err != nil { |
| 131 | t.Fatal(err) |
| 132 | } |
| 133 | if err = reg.UpsertTimer(registry.Timer{Host: old.Host, Name: "daily", Kind: "cron", Spec: "0 0 * * *", NextAt: 123, Attempts: 2}); err != nil { |
| 134 | t.Fatal(err) |
| 135 | } |
| 136 | _, err = reg.GetTimer(old.Host, "daily") |
| 137 | if err != nil { |
| 138 | t.Fatal(err) |
| 139 | } |
| 140 | parker := TypedParker{CapturePersistent: true, Roots: map[string]string{"data": source.Guard.DataDir}, ParkedRoot: source.Guard.DataDir, Incarnation: prior.Incarnation, Host: prior.Host, Retirement: "capture"} |
| 141 | artifact, err := parker.CaptureArtifact("data", old.Host+".db", owner.ClaimOwner, ArtifactPath) |
| 142 | if err != nil { |
| 143 | t.Fatal(err) |
| 144 | } |
| 145 | in := StopInventory{Complete: true, Host: old.Host, Incarnation: prior.Incarnation, InstallationStamp: prior.Stamp, Box: "box", Enrollment: "enrollment", DirectoryRevision: "1", Entries: []StopInventoryEntry{{Kind: "route", ArtifactID: owner.ClaimRowID, ManagedRoot: "registry", RelativePath: old.Host, RowIdentity: owner.ClaimRowID, Ownership: owner.ClaimOwner, Materialization: "materialized"}, {Kind: "path", ArtifactID: "database", ManagedRoot: "data", RelativePath: artifact.Name, Ownership: artifact.Owner, Materialization: "materialized", Identity: artifact.Identity, Device: artifact.Dev, Inode: artifact.Ino, Type: artifact.Type}}} |
| 146 | if err = inventory.WriteStopInventory(ctx, in); err != nil { |
| 147 | t.Fatal(err) |
| 148 | } |
| 149 | successor := admit(old, "second") |
| 150 | next := ColdSuccessor{Host: old.Host, Incarnation: prior.Incarnation, Stamp: successor.Claim.Holder.Stamp, Session: successor.Claim.Holder.Session, Generation: successor.Claim.Holder.Generation, Box: "box", Enrollment: "enrollment", DirectoryRevision: "1", ETag: successor.ETag, Op: successor.Claim.Op.ID, Phase: successor.Claim.Op.Phase} |
| 151 | handover := ColdHandover{Guard: source.Guard, Claims: store, Registry: reg, Inventory: inventory, Launches: launches, Roots: map[string]string{"data": source.Guard.DataDir}, JournalDir: filepath.Join(base, "handover"), VerifyReceipt: func(context.Context, hostguard.FenceReceipt) error { return nil }} |
| 152 | |
| 153 | durable := release.NewDirStore(filepath.Join(base, "deployment")) |
| 154 | keys, err := store.List(ctx, "") |
| 155 | if err != nil { |
| 156 | t.Fatal(err) |
| 157 | } |
| 158 | for _, key := range keys { |
| 159 | body, _, err := store.Get(ctx, key) |
| 160 | if err != nil { |
| 161 | t.Fatal(err) |
| 162 | } |
| 163 | if _, err = durable.Put(ctx, key, body, ""); err != nil { |
| 164 | t.Fatal(err) |
| 165 | } |
| 166 | } |
| 167 | manifest := plan14ColdManifest{Base: base, Data: source.Guard.DataDir, Locks: source.Guard.LockDir, Prior: prior, Next: next} |
| 168 | raw, _ := json.Marshal(manifest) |
| 169 | if err = os.WriteFile(filepath.Join(base, "fixture.json"), raw, 0600); err != nil { |
| 170 | t.Fatal(err) |
| 171 | } |
| 172 | _ = handover |
| 173 | return manifest |
| 174 | } |
| 175 | func plan14ColdOpen(t *testing.T, m plan14ColdManifest) (ColdHandover, *plan14ColdDisk) { |
| 176 | t.Helper() |
| 177 | reg, err := registry.Open(filepath.Join(m.Base, "registry.db")) |
| 178 | if err != nil { |
| 179 | t.Fatal(err) |
| 180 | } |
| 181 | t.Cleanup(func() { reg.Close() }) |
| 182 | inventory, err := OpenStopInventoryDB(filepath.Join(m.Base, "inventory.db")) |
| 183 | if err != nil { |
| 184 | t.Fatal(err) |
| 185 | } |
| 186 | t.Cleanup(func() { inventory.Close() }) |
| 187 | launches, err := OpenLaunchJournal(filepath.Join(m.Base, "launches.db")) |
| 188 | if err != nil { |
| 189 | t.Fatal(err) |
| 190 | } |
| 191 | t.Cleanup(func() { launches.Close() }) |
| 192 | disk := &plan14ColdDisk{dir: filepath.Join(m.Base, "root")} |
| 193 | store := release.NewDirStore(filepath.Join(m.Base, "deployment")) |
| 194 | g := hostguard.Guard{DataDir: m.Data, LockDir: m.Locks, Claims: hostguard.StoreReader{Store: store}, Root: hostguard.NewRootAuthorityWithInventory(disk, func(ctx context.Context, host string, c hostguard.Captured) (uint64, error) { |
| 195 | _, revision, err := inventory.Snapshot(ctx, host, c) |
| 196 | return revision, err |
| 197 | })} |
| 198 | return ColdHandover{Guard: g, Claims: store, Registry: reg, Inventory: inventory, Launches: launches, Roots: map[string]string{"data": m.Data}, JournalDir: filepath.Join(m.Base, "handover"), VerifyReceipt: func(_ context.Context, r hostguard.FenceReceipt) error { |
| 199 | if string(r.Bytes) != "verified-root-receipt" || r.Target != m.Prior { |
| 200 | return ErrChanged |
| 201 | } |
| 202 | return nil |
| 203 | }}, disk |
| 204 | } |
| 205 | func TestPlan14ColdHandoverProcessBoundaries(t *testing.T) { |
| 206 | if phase := os.Getenv("PLAN14_COLD_PHASE"); phase != "" { |
| 207 | t.Setenv("CARLOS_HOST_CLAIM", "1") |
| 208 | raw, err := os.ReadFile(filepath.Join(os.Getenv("PLAN14_COLD_DIR"), "fixture.json")) |
| 209 | if err != nil { |
| 210 | t.Fatal(err) |
| 211 | } |
| 212 | var m plan14ColdManifest |
| 213 | if err = json.Unmarshal(raw, &m); err != nil { |
| 214 | t.Fatal(err) |
| 215 | } |
| 216 | h, disk := plan14ColdOpen(t, m) |
| 217 | h.checkpoint = func(point string) { |
| 218 | if phase == point { |
| 219 | os.Exit(73) |
| 220 | } |
| 221 | } |
| 222 | classify := func(raw []byte) string { |
| 223 | var r struct { |
| 224 | Barrier *hostguard.FenceTarget |
| 225 | State struct { |
| 226 | Stamp uint64 `json:"active_stamp"` |
| 227 | } |
| 228 | } |
| 229 | if json.Unmarshal(raw, &r) != nil { |
| 230 | t.Fatal("bad root") |
| 231 | } |
| 232 | if r.State.Stamp == 0 { |
| 233 | return "retired" |
| 234 | } |
| 235 | if r.Barrier != nil { |
| 236 | return "barrier" |
| 237 | } |
| 238 | if r.State.Stamp == m.Next.Stamp { |
| 239 | return "commit" |
| 240 | } |
| 241 | return "old" |
| 242 | } |
| 243 | disk.before = func(raw []byte) { |
| 244 | if phase == "before-root-"+classify(raw) { |
| 245 | os.Exit(73) |
| 246 | } |
| 247 | } |
| 248 | disk.after = func(raw []byte) { |
| 249 | if phase == "after-root-"+classify(raw) { |
| 250 | os.Exit(73) |
| 251 | } |
| 252 | } |
| 253 | err = h.Run(t.Context(), m.Prior, m.Next) |
| 254 | t.Fatal("boundary not reached", phase, err) |
| 255 | } |
| 256 | for _, phase := range []string{"intent", "parked", "inventory-intent", "rows", "inventory-materialized", "complete-journal", "before-root-barrier", "after-root-barrier", "before-root-retired", "after-root-retired", "before-root-commit", "after-root-commit"} { |
| 257 | t.Run(phase, func(t *testing.T) { |
| 258 | m := plan14ColdSetup(t) |
| 259 | cmd := exec.CommandContext(t.Context(), os.Args[0], "-test.run=^TestPlan14ColdHandoverProcessBoundaries$") |
| 260 | cmd.Dir = m.Base |
| 261 | cmd.Env = append(os.Environ(), "PLAN14_COLD_PHASE="+phase, "PLAN14_COLD_DIR="+m.Base) |
| 262 | output, err := cmd.CombinedOutput() |
| 263 | exit, ok := err.(*exec.ExitError) |
| 264 | if !ok || exit.ExitCode() != 73 { |
| 265 | t.Fatalf("crash: %v %s", err, output) |
| 266 | } |
| 267 | h, _ := plan14ColdOpen(t, m) |
| 268 | effects := 0 |
| 269 | allow := func(*hostclaim.Claim) error { return nil } |
| 270 | prior := hostguard.Captured{Incarnation: m.Prior.Incarnation, Stamp: m.Prior.Stamp} |
| 271 | next := hostguard.Captured{Incarnation: m.Next.Incarnation, Stamp: m.Next.Stamp} |
| 272 | if err = h.Guard.With(t.Context(), m.Prior.Host, prior, allow, func() error { effects++; return nil }); err == nil || effects != 0 { |
| 273 | t.Fatal("predecessor granted", err, effects) |
| 274 | } |
| 275 | if phase != "after-root-commit" { |
| 276 | if err = h.Guard.With(t.Context(), m.Next.Host, next, allow, func() error { effects++; return nil }); err == nil || effects != 0 { |
| 277 | t.Fatal("unfinished successor granted", err, effects) |
| 278 | } |
| 279 | } |
| 280 | if err = h.Recover(t.Context()); err != nil { |
| 281 | t.Fatal("resume", err) |
| 282 | } |
| 283 | if err = h.Run(t.Context(), m.Prior, m.Next); err != nil { |
| 284 | t.Fatal("replay", err) |
| 285 | } |
| 286 | if err = h.Guard.With(t.Context(), m.Next.Host, next, allow, func() error { effects++; return nil }); err != nil || effects != 1 { |
| 287 | t.Fatal("successor", err, effects) |
| 288 | } |
| 289 | receipt, err := h.Guard.FenceReceiptFor(t.Context(), m.Prior) |
| 290 | if err != nil || h.VerifyReceipt(t.Context(), receipt) != nil || receipt.Target.BootID != "boot" { |
| 291 | t.Fatal("old receipt changed", receipt, err) |
| 292 | } |
| 293 | if _, err = os.Stat(filepath.Join(m.Data, m.Prior.Host+".db")); !os.IsNotExist(err) { |
| 294 | t.Fatal("old data reused", err) |
| 295 | } |
| 296 | row, err := h.Registry.Get(m.Next.Host) |
| 297 | if err != nil || row.ClaimStamp != m.Next.Stamp { |
| 298 | t.Fatal("rows", row, err) |
| 299 | } |
| 300 | view, err := h.Guard.Root.View(t.Context(), m.Next.Host, next) |
| 301 | if err != nil || view.Barrier != nil || view.Installation.InventoryRevision == 0 { |
| 302 | t.Fatal("root commit", view, err) |
| 303 | } |
| 304 | }) |
| 305 | } |
| 306 | } |
| 307 | |
amadan