| 1 | //go:build linux |
| 2 | |
| 3 | package hostbox |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "errors" |
| 8 | "os" |
| 9 | "path/filepath" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/hostclaim" |
| 14 | "github.com/carlosframework/platform/internal/hostguard" |
| 15 | "github.com/carlosframework/platform/internal/registry" |
| 16 | ) |
| 17 | |
| 18 | func TestColdHandoverParksFilesRestoresOwnedRowsAndNeverReplaysOverStartedSuccessor(t *testing.T) { |
| 19 | coldHandoverTimers(t, false) |
| 20 | } |
| 21 | |
| 22 | func TestPlan14ColdHandoverAbandonedTimers(t *testing.T) { |
| 23 | coldHandoverTimers(t, true) |
| 24 | } |
| 25 | |
| 26 | func coldHandoverTimers(t *testing.T, abandoned bool) { |
| 27 | t.Setenv("CARLOS_HOST_CLAIM", "1") |
| 28 | source, store, initial := workFixture(t) |
| 29 | ctx := t.Context() |
| 30 | admit := func(old hostclaim.Observed, generation string) hostclaim.Observed { |
| 31 | next, err := hostclaim.Update(ctx, store, old, hostclaim.AdmitWriter(hostclaim.Admission{ObservedOpID: old.Claim.Op.ID, ObservedPhase: old.Claim.Op.Phase, ObservedSession: old.Claim.Holder.Session, ObservedGeneration: old.Claim.Holder.Generation, Box: "box", Region: "ie", Mode: "cold", NewGeneration: generation})) |
| 32 | if err != nil { |
| 33 | t.Fatal(err) |
| 34 | } |
| 35 | return next |
| 36 | } |
| 37 | old := admit(initial, "first") |
| 38 | if err := source.Guard.Activate(ctx, old.Host, "box", nil); err != nil { |
| 39 | t.Fatal(err) |
| 40 | } |
| 41 | prior := hostguard.FenceTarget{FenceID: "fence", Region: "ie", Host: old.Host, Incarnation: old.Claim.Holder.Incarnation, Stamp: old.Claim.Holder.Stamp, Session: old.Claim.Holder.Session, Box: "box", Enrollment: "enrollment", BootID: "boot", ControllerID: "controller", ClaimETag: old.ETag, DirectoryRevision: "1", Op: old.Claim.Op.ID, Phase: old.Claim.Op.Phase} |
| 42 | if _, err := source.Guard.BeginFence(ctx, prior); err != nil { |
| 43 | t.Fatal(err) |
| 44 | } |
| 45 | if err := source.Guard.CompleteFence(ctx, prior, []byte("verified-root-receipt"), time.Now().UTC(), func(context.Context, hostguard.FenceTarget, []byte, time.Time) error { return nil }); err != nil { |
| 46 | t.Fatal(err) |
| 47 | } |
| 48 | base := t.TempDir() |
| 49 | reg, err := registry.Open(filepath.Join(base, "registry.db")) |
| 50 | if err != nil { |
| 51 | t.Fatal(err) |
| 52 | } |
| 53 | defer reg.Close() |
| 54 | inventory, err := OpenStopInventoryDB(filepath.Join(base, "inventory.db")) |
| 55 | if err != nil { |
| 56 | t.Fatal(err) |
| 57 | } |
| 58 | defer inventory.Close() |
| 59 | launches, err := OpenLaunchJournal(filepath.Join(base, "launches.db")) |
| 60 | if err != nil { |
| 61 | t.Fatal(err) |
| 62 | } |
| 63 | defer launches.Close() |
| 64 | owner := registry.ClaimOwnership{ClaimIncarnation: prior.Incarnation, ClaimStamp: prior.Stamp, ClaimOwner: "old-owner", ClaimRowID: "old-route"} |
| 65 | dbPath := filepath.Join(source.Guard.DataDir, old.Host+".db") |
| 66 | if err = os.WriteFile(dbPath, []byte("untrusted-local-database"), 0600); err != nil { |
| 67 | t.Fatal(err) |
| 68 | } |
| 69 | if err = reg.Upsert(registry.Route{Host: old.Host, Kind: "instance", Socket: "/run/example.sock", DBPath: dbPath, ClaimOwnership: owner, HoldOp: "held-operation"}); err != nil { |
| 70 | t.Fatal(err) |
| 71 | } |
| 72 | if err = reg.UpsertTimer(registry.Timer{Host: old.Host, Name: "daily", Kind: "cron", Spec: "0 0 * * *", NextAt: 123, Attempts: 2}); err != nil { |
| 73 | t.Fatal(err) |
| 74 | } |
| 75 | timer, err := reg.GetTimer(old.Host, "daily") |
| 76 | if err != nil { |
| 77 | t.Fatal(err) |
| 78 | } |
| 79 | parker := TypedParker{CapturePersistent: true, Roots: map[string]string{"data": source.Guard.DataDir}, ParkedRoot: source.Guard.DataDir, Incarnation: prior.Incarnation, Host: prior.Host, Retirement: "capture"} |
| 80 | artifact, err := parker.CaptureArtifact("data", old.Host+".db", owner.ClaimOwner, ArtifactPath) |
| 81 | if err != nil { |
| 82 | t.Fatal(err) |
| 83 | } |
| 84 | in := StopInventory{Complete: true, Host: old.Host, Incarnation: prior.Incarnation, InstallationStamp: prior.Stamp, Box: "box", Enrollment: "enrollment", DirectoryRevision: "1", Entries: []StopInventoryEntry{{Kind: "route", ArtifactID: owner.ClaimRowID, ManagedRoot: "registry", RelativePath: old.Host, RowIdentity: owner.ClaimRowID, Ownership: owner.ClaimOwner, Materialization: "materialized"}, {Kind: "path", ArtifactID: "database", ManagedRoot: "data", RelativePath: artifact.Name, Ownership: artifact.Owner, Materialization: "materialized", Identity: artifact.Identity, Device: artifact.Dev, Inode: artifact.Ino, Type: artifact.Type}}} |
| 85 | if abandoned { |
| 86 | for _, name := range []string{"abandoned", "withdrawn"} { |
| 87 | in.Entries = append(in.Entries, StopInventoryEntry{Kind: "timer", ArtifactID: name, ManagedRoot: "registry", RelativePath: old.Host + "/" + name, Ownership: owner.ClaimOwner, RowIdentity: name, Materialization: "intended", TimerRetired: name == "withdrawn"}) |
| 88 | } |
| 89 | } |
| 90 | if err = inventory.WriteStopInventory(ctx, in); err != nil { |
| 91 | t.Fatal(err) |
| 92 | } |
| 93 | successor := admit(old, "second") |
| 94 | next := ColdSuccessor{Host: old.Host, Incarnation: prior.Incarnation, Stamp: successor.Claim.Holder.Stamp, Session: successor.Claim.Holder.Session, Generation: successor.Claim.Holder.Generation, Box: "box", Enrollment: "enrollment", DirectoryRevision: "1", ETag: successor.ETag, Op: successor.Claim.Op.ID, Phase: successor.Claim.Op.Phase} |
| 95 | handover := ColdHandover{Guard: source.Guard, Claims: store, Registry: reg, Inventory: inventory, Launches: launches, Roots: map[string]string{"data": source.Guard.DataDir}, JournalDir: filepath.Join(base, "handover"), VerifyReceipt: func(context.Context, hostguard.FenceReceipt) error { return nil }} |
| 96 | |
| 97 | intentCrash := errors.New("crash after durable intent before successor barrier") |
| 98 | handover.afterIntent = func() error { return intentCrash } |
| 99 | if err = handover.Run(ctx, prior, next); !errors.Is(err, intentCrash) { |
| 100 | t.Fatal(err) |
| 101 | } |
| 102 | pending, err := source.Guard.PendingFences(ctx) |
| 103 | if err != nil || len(pending) != 0 { |
| 104 | t.Fatalf("barrier exists before durable intent recovery: %+v %v", pending, err) |
| 105 | } |
| 106 | handover.afterIntent = nil |
| 107 | crash := errors.New("crash after old installation parked") |
| 108 | handover.afterPark = func() error { return crash } |
| 109 | if err = handover.Recover(ctx); !errors.Is(err, crash) { |
| 110 | t.Fatal(err) |
| 111 | } |
| 112 | if err = source.Guard.Activate(ctx, old.Host, "box", nil); err != nil { |
| 113 | t.Fatal(err) |
| 114 | } |
| 115 | if err = source.Guard.With(ctx, old.Host, hostguard.Captured{Incarnation: next.Incarnation, Stamp: next.Stamp}, func(*hostclaim.Claim) error { return nil }, func() error { return nil }); !errors.Is(err, hostguard.ErrFenced) { |
| 116 | t.Fatalf("crash exposed successor: %v", err) |
| 117 | } |
| 118 | if err = reg.Close(); err != nil { |
| 119 | t.Fatal(err) |
| 120 | } |
| 121 | reg, err = registry.Open(filepath.Join(base, "registry.db")) |
| 122 | if err != nil { |
| 123 | t.Fatal(err) |
| 124 | } |
| 125 | defer reg.Close() |
| 126 | if err = inventory.Close(); err != nil { |
| 127 | t.Fatal(err) |
| 128 | } |
| 129 | inventory, err = OpenStopInventoryDB(filepath.Join(base, "inventory.db")) |
| 130 | if err != nil { |
| 131 | t.Fatal(err) |
| 132 | } |
| 133 | defer inventory.Close() |
| 134 | handover.Registry = reg |
| 135 | handover.Inventory = inventory |
| 136 | handover.afterPark = nil |
| 137 | if err = handover.Recover(ctx); err != nil { |
| 138 | t.Fatal(err) |
| 139 | } |
| 140 | if _, err = os.Stat(dbPath); !os.IsNotExist(err) { |
| 141 | t.Fatalf("old DB reused: %v", err) |
| 142 | } |
| 143 | got, err := reg.Get(old.Host) |
| 144 | if err != nil || got.ClaimStamp != next.Stamp || got.ClaimRowID == owner.ClaimRowID || got.HoldOp != "held-operation" { |
| 145 | t.Fatalf("route ownership not restored: %+v %v", got, err) |
| 146 | } |
| 147 | after, err := reg.GetTimer(old.Host, "daily") |
| 148 | if err != nil || after.Registration != timer.Registration || after.NextAt != timer.NextAt || after.Attempts != timer.Attempts || after.ClaimRowID == timer.ClaimRowID || after.ClaimStamp != next.Stamp { |
| 149 | t.Fatalf("timer bookkeeping not retained: %+v %v", after, err) |
| 150 | } |
| 151 | retained, err := inventory.ReadInstallation(ctx, old.Host, prior.Incarnation, prior.Stamp) |
| 152 | if err != nil || len(retained.Entries) != len(in.Entries) { |
| 153 | t.Fatal("predecessor inventory discarded") |
| 154 | } |
| 155 | if err = os.WriteFile(dbPath, []byte("new-runtime-data"), 0600); err != nil { |
| 156 | t.Fatal(err) |
| 157 | } |
| 158 | if err = reg.SetVersions(old.Host, "new-live-version", ""); err != nil { |
| 159 | t.Fatal(err) |
| 160 | } |
| 161 | if err = handover.Run(ctx, prior, next); err != nil { |
| 162 | t.Fatal(err) |
| 163 | } |
| 164 | if err = handover.Recover(ctx); err != nil { |
| 165 | t.Fatal(err) |
| 166 | } |
| 167 | body, err := os.ReadFile(dbPath) |
| 168 | if err != nil || string(body) != "new-runtime-data" { |
| 169 | t.Fatal("completed replay touched live successor") |
| 170 | } |
| 171 | got, err = reg.Get(old.Host) |
| 172 | if err != nil || got.Version != "new-live-version" { |
| 173 | t.Fatal("completed replay overwrote new route") |
| 174 | } |
| 175 | } |
| 176 | |
amadan