| 1 | //go:build linux |
| 2 | |
| 3 | package hostbox |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "database/sql" |
| 8 | "errors" |
| 9 | "fmt" |
| 10 | "os" |
| 11 | "path/filepath" |
| 12 | "reflect" |
| 13 | "testing" |
| 14 | "time" |
| 15 | |
| 16 | "github.com/carlosframework/platform/internal/hostclaim" |
| 17 | "github.com/carlosframework/platform/internal/hostguard" |
| 18 | "github.com/carlosframework/platform/internal/provision" |
| 19 | "github.com/carlosframework/platform/internal/registry" |
| 20 | ) |
| 21 | |
| 22 | func servingHandoverFixture(t *testing.T, deviceDelta ...uint64) (ServingHandover, hostguard.FenceTarget, ColdSuccessor) { |
| 23 | t.Helper() |
| 24 | return servingHandoverInitialFixture(t, false, false, deviceDelta...) |
| 25 | } |
| 26 | |
| 27 | func servingHandoverInitialFixture(t *testing.T, first, omitPath bool, deviceDelta ...uint64) (ServingHandover, hostguard.FenceTarget, ColdSuccessor) { |
| 28 | t.Helper() |
| 29 | t.Setenv("CARLOS_HOST_CLAIM", "1") |
| 30 | source, store, initial := workFixture(t) |
| 31 | admit := func(o hostclaim.Observed, g string) hostclaim.Observed { |
| 32 | n, err := hostclaim.Update(t.Context(), store, o, hostclaim.AdmitWriter(hostclaim.Admission{ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: "box", Region: "ie", Mode: "cold", NewGeneration: g})) |
| 33 | if err != nil { |
| 34 | t.Fatal(err) |
| 35 | } |
| 36 | return n |
| 37 | } |
| 38 | old := initial |
| 39 | if !first { |
| 40 | old = admit(initial, "first") |
| 41 | } |
| 42 | base := t.TempDir() |
| 43 | disk := &plan14ColdDisk{dir: t.TempDir()} |
| 44 | reg, err := registry.Open(filepath.Join(base, "registry.db")) |
| 45 | if err != nil { |
| 46 | t.Fatal(err) |
| 47 | } |
| 48 | t.Cleanup(func() { reg.Close() }) |
| 49 | inv, err := OpenStopInventoryDB(filepath.Join(base, "inventory.db")) |
| 50 | if err != nil { |
| 51 | t.Fatal(err) |
| 52 | } |
| 53 | t.Cleanup(func() { inv.Close() }) |
| 54 | launches, err := OpenLaunchJournal(filepath.Join(base, "launch.db")) |
| 55 | if err != nil { |
| 56 | t.Fatal(err) |
| 57 | } |
| 58 | t.Cleanup(func() { launches.Close() }) |
| 59 | guard := source.Guard |
| 60 | guard.Root = hostguard.NewRootAuthorityWithInventory(disk, func(ctx context.Context, host string, c hostguard.Captured) (uint64, error) { |
| 61 | _, r, e := inv.Snapshot(ctx, host, c) |
| 62 | return r, e |
| 63 | }) |
| 64 | holder := old.Claim.Holder |
| 65 | owner := ownershipFor(old.Host, holder.Incarnation, holder.Stamp, "box") |
| 66 | row := registry.Route{Host: old.Host, Kind: "instance", Account: holder.Account, App: holder.App, DBPath: provision.NestedDBPath(guard.DataDir, old.Host), Socket: provision.NestedSocket(old.Host), ClaimOwnership: registry.ClaimOwnership{ClaimIncarnation: holder.Incarnation, ClaimStamp: holder.Stamp, ClaimOwner: owner, ClaimRowID: "old-route"}} |
| 67 | if err = reg.Upsert(row); err != nil { |
| 68 | t.Fatal(err) |
| 69 | } |
| 70 | dir := filepath.Dir(row.DBPath) |
| 71 | if err = os.MkdirAll(dir, 0700); err != nil { |
| 72 | t.Fatal(err) |
| 73 | } |
| 74 | if err = os.WriteFile(filepath.Join(dir, "env"), []byte("configuration"), 0600); err != nil { |
| 75 | t.Fatal(err) |
| 76 | } |
| 77 | roots := map[string]string{"data": guard.DataDir} |
| 78 | rel, _ := filepath.Rel(guard.DataDir, dir) |
| 79 | a, err := (TypedParker{Roots: roots, ParkedRoots: roots, Incarnation: holder.Incarnation, Host: old.Host, Retirement: "capture"}).CaptureArtifact("data", rel, owner, ArtifactPath) |
| 80 | if err != nil { |
| 81 | t.Fatal(err) |
| 82 | } |
| 83 | if len(deviceDelta) > 0 { |
| 84 | a.Dev += deviceDelta[0] |
| 85 | } |
| 86 | in := StopInventory{Complete: true, Host: old.Host, Incarnation: holder.Incarnation, InstallationStamp: holder.Stamp, Box: "box", Enrollment: "enrollment", DirectoryRevision: "1", Entries: []StopInventoryEntry{ |
| 87 | {Kind: "route", ArtifactID: "old-route", ManagedRoot: "registry", RelativePath: old.Host, RowIdentity: "old-route", Ownership: owner, Materialization: "materialized"}, |
| 88 | {Kind: "path", ArtifactID: "state", ManagedRoot: "data", RelativePath: rel, Ownership: owner, Materialization: "materialized", Device: a.Dev, Inode: a.Ino, Type: a.Type}, |
| 89 | }} |
| 90 | if omitPath { |
| 91 | in.Entries = in.Entries[:1] |
| 92 | } |
| 93 | if err = inv.WriteStopInventory(t.Context(), in); err != nil { |
| 94 | t.Fatal(err) |
| 95 | } |
| 96 | _, rev, err := inv.Snapshot(t.Context(), old.Host, hostguard.Captured{Incarnation: holder.Incarnation, Stamp: holder.Stamp}) |
| 97 | if err != nil { |
| 98 | t.Fatal(err) |
| 99 | } |
| 100 | installation := hostguard.RootInstallation{Host: old.Host, Incarnation: holder.Incarnation, Stamp: holder.Stamp, Account: holder.Account, App: holder.App, Record: holder.Key, Box: "box", Enrollment: "enrollment", Migration: 1, InventoryRevision: rev} |
| 101 | if err = guard.EstablishRoot(t.Context(), installation, func(*hostclaim.Claim) error { return nil }); err != nil { |
| 102 | t.Fatal(err) |
| 103 | } |
| 104 | prior := hostguard.FenceTarget{FenceID: "stop", Region: "ie", Host: old.Host, Incarnation: holder.Incarnation, Stamp: holder.Stamp, Session: holder.Session, Box: "box", Enrollment: "enrollment", BootID: "boot", ControllerID: "controller", ClaimETag: old.ETag, DirectoryRevision: "1"} |
| 105 | if !first { |
| 106 | if _, err = guard.BeginFence(t.Context(), prior); err != nil { |
| 107 | t.Fatal(err) |
| 108 | } |
| 109 | if err = guard.CompleteFence(t.Context(), prior, []byte("verified stop"), time.Now().UTC(), func(context.Context, hostguard.FenceTarget, []byte, time.Time) error { return nil }); err != nil { |
| 110 | t.Fatal(err) |
| 111 | } |
| 112 | } else { |
| 113 | prior.FenceID = "" |
| 114 | } |
| 115 | successor := admit(old, "second") |
| 116 | n := successor.Claim.Holder |
| 117 | next := ColdSuccessor{Host: old.Host, Incarnation: n.Incarnation, Stamp: n.Stamp, Session: n.Session, Generation: n.Generation, Box: "box", Enrollment: "enrollment", DirectoryRevision: "1", ETag: successor.ETag} |
| 118 | h := ServingHandover{ColdHandover: ColdHandover{Guard: guard, Claims: store, Registry: reg, Inventory: inv, Launches: launches, Roots: roots, JournalDir: filepath.Join(base, "handover"), VerifyReceipt: func(_ context.Context, r hostguard.FenceReceipt) error { |
| 119 | if r.Target != prior || string(r.Bytes) != "verified stop" { |
| 120 | return ErrChanged |
| 121 | } |
| 122 | return nil |
| 123 | }}, VerifyDirectory: func(_ context.Context, p hostguard.FenceTarget, e StopInventoryEntry, a Artifact) error { |
| 124 | if p.BootID != "boot" || a.Dev != e.Device { |
| 125 | return ErrChanged |
| 126 | } |
| 127 | return nil |
| 128 | }, Validate: func(_ context.Context, c *hostclaim.Claim, p hostguard.FenceTarget, _ ColdSuccessor, _ StopInventory, i hostguard.RootInstallation) error { |
| 129 | if p != prior || i != installation || !reflect.DeepEqual(c, successor.Claim) { |
| 130 | return ErrChanged |
| 131 | } |
| 132 | return nil |
| 133 | }} |
| 134 | return h, prior, next |
| 135 | } |
| 136 | |
| 137 | func TestServingHandoverRecoversEveryBoundary(t *testing.T) { |
| 138 | for _, point := range []string{"intent", "barrier", "inventory", "rows", "complete-journal"} { |
| 139 | t.Run(point, func(t *testing.T) { |
| 140 | h, p, n := servingHandoverFixture(t) |
| 141 | old, _, err := h.Inventory.Snapshot(t.Context(), p.Host, hostguard.Captured{Incarnation: p.Incarnation, Stamp: p.Stamp}) |
| 142 | if err != nil { |
| 143 | t.Fatal(err) |
| 144 | } |
| 145 | crash := errors.New("injected interruption") |
| 146 | h.checkpoint = func(got string) error { |
| 147 | if got == point { |
| 148 | return crash |
| 149 | } |
| 150 | return nil |
| 151 | } |
| 152 | if err = h.Run(t.Context(), p, n); !errors.Is(err, crash) { |
| 153 | t.Fatal(err) |
| 154 | } |
| 155 | _ = h.Guard.Activate(t.Context(), p.Host, p.Box, nil) |
| 156 | allowed := func(*hostclaim.Claim) error { return nil } |
| 157 | if err = h.Guard.With(t.Context(), p.Host, hostguard.Captured{Incarnation: n.Incarnation, Stamp: n.Stamp}, allowed, func() error { t.Fatal("successor ran during partial handover"); return nil }); err == nil { |
| 158 | t.Fatal("partial transfer authorized") |
| 159 | } |
| 160 | h.checkpoint = nil |
| 161 | if err = h.Recover(t.Context()); err != nil { |
| 162 | t.Fatal("recover", err) |
| 163 | } |
| 164 | if err = h.Run(t.Context(), p, n); err != nil { |
| 165 | t.Fatal("replay", err) |
| 166 | } |
| 167 | if err = h.Guard.With(t.Context(), p.Host, hostguard.Captured{Incarnation: n.Incarnation, Stamp: n.Stamp}, allowed, func() error { return nil }); err != nil { |
| 168 | t.Fatal("completed transfer", err) |
| 169 | } |
| 170 | successor, err := h.Inventory.ReadInstallation(t.Context(), n.Host, n.Incarnation, n.Stamp) |
| 171 | if err != nil { |
| 172 | t.Fatal(err) |
| 173 | } |
| 174 | for _, e := range successor.Entries { |
| 175 | if e.Kind == "path" && (e.Identity.Boot == "" || VerifyInventoryIdentity(h.Roots[e.ManagedRoot], e) != nil) { |
| 176 | t.Fatal("successor lost verified directory identity", e) |
| 177 | } |
| 178 | } |
| 179 | retained, _, err := h.Inventory.Snapshot(t.Context(), p.Host, hostguard.Captured{Incarnation: p.Incarnation, Stamp: p.Stamp}) |
| 180 | if err != nil || !reflect.DeepEqual(retained, old) { |
| 181 | t.Fatal("old inventory", err) |
| 182 | } |
| 183 | b, err := os.ReadFile(filepath.Join(filepath.Dir(provision.NestedDBPath(h.Guard.DataDir, p.Host)), "env")) |
| 184 | if err != nil || string(b) != "configuration" { |
| 185 | t.Fatal("source lost", err) |
| 186 | } |
| 187 | }) |
| 188 | } |
| 189 | } |
| 190 | |
| 191 | func TestServingHandoverRefusesOldDestructiveRecovery(t *testing.T) { |
| 192 | h, p, n := servingHandoverFixture(t) |
| 193 | old := h.ColdHandover |
| 194 | old.OperationsOnly = true |
| 195 | if err := old.Run(t.Context(), p, n); !errors.Is(err, ErrUnsupported) { |
| 196 | t.Fatal(err) |
| 197 | } |
| 198 | if _, err := os.Stat(filepath.Dir(provision.NestedDBPath(h.Guard.DataDir, p.Host))); err != nil { |
| 199 | t.Fatal(err) |
| 200 | } |
| 201 | } |
| 202 | |
| 203 | func TestServingHandoverStaleJournalDoesNotBlockHealthy(t *testing.T) { |
| 204 | h, p, n := servingHandoverFixture(t) |
| 205 | h.checkpoint = func(point string) error { |
| 206 | if point == "intent" { |
| 207 | return errors.New("interrupt") |
| 208 | } |
| 209 | return nil |
| 210 | } |
| 211 | if h.Run(t.Context(), p, n) == nil { |
| 212 | t.Fatal("no interrupted intent") |
| 213 | } |
| 214 | stale := filepath.Join(h.JournalDir, "000-stale.json") |
| 215 | if err := os.WriteFile(stale, []byte("stale journal"), 0600); err != nil { |
| 216 | t.Fatal(err) |
| 217 | } |
| 218 | h.checkpoint = nil |
| 219 | if h.Recover(t.Context()) == nil { |
| 220 | t.Fatal("stale journal was not reported") |
| 221 | } |
| 222 | if b, err := os.ReadFile(stale); err != nil || string(b) != "stale journal" { |
| 223 | t.Fatal("stale evidence changed", err) |
| 224 | } |
| 225 | if err := h.Guard.With(t.Context(), p.Host, hostguard.Captured{Incarnation: n.Incarnation, Stamp: n.Stamp}, func(*hostclaim.Claim) error { return nil }, func() error { return nil }); err != nil { |
| 226 | t.Fatal("healthy journal not recovered", err) |
| 227 | } |
| 228 | } |
| 229 | |
| 230 | func TestServingHandoverRebindsOnlyProvenDirectory(t *testing.T) { |
| 231 | for _, proven := range []bool{false, true} { |
| 232 | t.Run(fmt.Sprint(proven), func(t *testing.T) { |
| 233 | h, prior, next := servingHandoverFixture(t, 1) |
| 234 | in, _, err := h.Inventory.Snapshot(t.Context(), prior.Host, hostguard.Captured{Incarnation: prior.Incarnation, Stamp: prior.Stamp}) |
| 235 | if err != nil { |
| 236 | t.Fatal(err) |
| 237 | } |
| 238 | var live uint64 |
| 239 | for _, e := range in.Entries { |
| 240 | if e.Kind == "path" { |
| 241 | live = e.Device - 1 |
| 242 | } |
| 243 | } |
| 244 | calls := 0 |
| 245 | h.VerifyDirectory = func(_ context.Context, p hostguard.FenceTarget, e StopInventoryEntry, a Artifact) error { |
| 246 | calls++ |
| 247 | if p != prior || a.Dev != live || e.Device != live+1 || !proven { |
| 248 | return ErrChanged |
| 249 | } |
| 250 | return nil |
| 251 | } |
| 252 | err = h.Run(t.Context(), prior, next) |
| 253 | if !proven { |
| 254 | if err == nil { |
| 255 | t.Fatal("unproven device accepted") |
| 256 | } |
| 257 | return |
| 258 | } |
| 259 | if err != nil { |
| 260 | t.Fatal(err) |
| 261 | } |
| 262 | out, err := h.Inventory.ReadInstallation(t.Context(), next.Host, next.Incarnation, next.Stamp) |
| 263 | if err != nil { |
| 264 | t.Fatal(err) |
| 265 | } |
| 266 | for _, e := range out.Entries { |
| 267 | if e.Kind == "path" && e.Device != live { |
| 268 | t.Fatal("successor kept stale device") |
| 269 | } |
| 270 | } |
| 271 | if calls < 2 { |
| 272 | t.Fatal("proof not rechecked", calls) |
| 273 | } |
| 274 | }) |
| 275 | } |
| 276 | } |
| 277 | |
| 278 | func TestServingHandoverEqualNumbersRequireProof(t *testing.T) { |
| 279 | for _, boundary := range []string{"initial", "intent", "barrier", "inventory-write", "complete-journal"} { |
| 280 | t.Run(boundary, func(t *testing.T) { |
| 281 | h, p, n := servingHandoverFixture(t) |
| 282 | changed := boundary == "initial" |
| 283 | calls := 0 |
| 284 | h.VerifyDirectory = func(_ context.Context, _ hostguard.FenceTarget, e StopInventoryEntry, a Artifact) error { |
| 285 | calls++ |
| 286 | if e.Device != a.Dev || e.Inode != a.Ino || e.Type != a.Type { |
| 287 | t.Fatal("numeric identities differ") |
| 288 | } |
| 289 | if changed { |
| 290 | return ErrChanged |
| 291 | } |
| 292 | return nil |
| 293 | } |
| 294 | if boundary == "inventory-write" { |
| 295 | validate := h.Validate |
| 296 | h.Validate = func(ctx context.Context, c *hostclaim.Claim, p hostguard.FenceTarget, n ColdSuccessor, in StopInventory, i hostguard.RootInstallation) error { |
| 297 | err := validate(ctx, c, p, n, in, i) |
| 298 | if calls == 3 { |
| 299 | changed = true |
| 300 | } |
| 301 | return err |
| 302 | } |
| 303 | } else if boundary != "initial" { |
| 304 | h.checkpoint = func(point string) error { |
| 305 | if point == boundary { |
| 306 | return errors.New("interrupted") |
| 307 | } |
| 308 | return nil |
| 309 | } |
| 310 | if h.Run(t.Context(), p, n) == nil { |
| 311 | t.Fatal("interruption missed") |
| 312 | } |
| 313 | h.checkpoint = nil |
| 314 | changed = true |
| 315 | } |
| 316 | before, err := h.Registry.Get(p.Host) |
| 317 | if err != nil { |
| 318 | t.Fatal(err) |
| 319 | } |
| 320 | inventory, invErr := h.Inventory.ReadInstallation(t.Context(), n.Host, n.Incarnation, n.Stamp) |
| 321 | if err := h.Run(t.Context(), p, n); !errors.Is(err, ErrChanged) { |
| 322 | t.Fatal("unproven directory accepted", err) |
| 323 | } |
| 324 | if calls == 0 { |
| 325 | t.Fatal("proof skipped") |
| 326 | } |
| 327 | after, err := h.Registry.Get(p.Host) |
| 328 | if err != nil || !reflect.DeepEqual(before, after) { |
| 329 | t.Fatal("registry changed", err) |
| 330 | } |
| 331 | got, gotErr := h.Inventory.ReadInstallation(t.Context(), n.Host, n.Incarnation, n.Stamp) |
| 332 | if !reflect.DeepEqual(inventory, got) || !errors.Is(gotErr, invErr) { |
| 333 | t.Fatal("inventory changed", gotErr) |
| 334 | } |
| 335 | if boundary != "complete-journal" && !errors.Is(gotErr, sql.ErrNoRows) { |
| 336 | t.Fatal("successor inventory exists", gotErr) |
| 337 | } |
| 338 | }) |
| 339 | } |
| 340 | } |
| 341 | |
| 342 | func TestServingHandoverFreshRouteWithoutState(t *testing.T) { |
| 343 | for _, first := range []bool{true, false} { |
| 344 | t.Run(fmt.Sprint(first), func(t *testing.T) { |
| 345 | h, p, n := servingHandoverInitialFixture(t, first, true) |
| 346 | err := h.Run(t.Context(), p, n) |
| 347 | if !first { |
| 348 | if !errors.Is(err, ErrTypedInventory) { |
| 349 | t.Fatal("running predecessor without state", err) |
| 350 | } |
| 351 | return |
| 352 | } |
| 353 | if err != nil { |
| 354 | t.Fatal("first route handover", err) |
| 355 | } |
| 356 | if err := h.Run(t.Context(), p, n); err != nil { |
| 357 | t.Fatal("replay", err) |
| 358 | } |
| 359 | in, err := h.Inventory.ReadInstallation(t.Context(), n.Host, n.Incarnation, n.Stamp) |
| 360 | if err != nil || len(in.Entries) != 1 || in.Entries[0].Kind != "route" { |
| 361 | t.Fatal("unexpected state authority", in, err) |
| 362 | } |
| 363 | }) |
| 364 | } |
| 365 | } |
| 366 | |
| 367 | func TestServingHandoverBarrierIsNotPendingStop(t *testing.T) { |
| 368 | h, prior, next := servingHandoverFixture(t) |
| 369 | crashed := errors.New("crash after durable barrier") |
| 370 | h.checkpoint = func(point string) error { |
| 371 | if point == "barrier" { |
| 372 | return crashed |
| 373 | } |
| 374 | return nil |
| 375 | } |
| 376 | if err := h.Run(t.Context(), prior, next); !errors.Is(err, crashed) { |
| 377 | t.Fatal(err) |
| 378 | } |
| 379 | all, err := h.Guard.PendingFences(t.Context()) |
| 380 | if err != nil || len(all) != 1 { |
| 381 | t.Fatal(all, err) |
| 382 | } |
| 383 | stops, err := h.Guard.PendingStopFences(t.Context()) |
| 384 | if err != nil || len(stops) != 0 { |
| 385 | t.Fatal("handover exposed as a stop", stops, err) |
| 386 | } |
| 387 | h.checkpoint = nil |
| 388 | if err := h.Recover(t.Context()); err != nil { |
| 389 | t.Fatal("recover handover", err) |
| 390 | } |
| 391 | all, err = h.Guard.PendingFences(t.Context()) |
| 392 | if err != nil || len(all) != 0 { |
| 393 | t.Fatal(all, err) |
| 394 | } |
| 395 | } |
| 396 | |
amadan