| 1 | //go:build linux |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "errors" |
| 8 | "fmt" |
| 9 | "github.com/carlosframework/platform/internal/hostclaim" |
| 10 | "github.com/carlosframework/platform/internal/hostwrite" |
| 11 | "github.com/carlosframework/platform/internal/park" |
| 12 | "github.com/carlosframework/platform/internal/registry" |
| 13 | "os" |
| 14 | "path/filepath" |
| 15 | "testing" |
| 16 | "time" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/hostbox" |
| 19 | "github.com/carlosframework/platform/internal/hostguard" |
| 20 | "github.com/carlosframework/platform/internal/provision" |
| 21 | "github.com/carlosframework/platform/internal/rootstate" |
| 22 | ) |
| 23 | |
| 24 | func TestPlan14RootInventoryEstablishment(t *testing.T) { |
| 25 | t.Parallel() |
| 26 | for _, attack := range []string{"none", "record etag", "claim etag", "account", "revision", "foreign root"} { |
| 27 | t.Run(attack, func(t *testing.T) { |
| 28 | f := newRecoveryFixture(t) |
| 29 | o := f.read() |
| 30 | h := o.Claim.Holder |
| 31 | rec := provision.Record{Schema: 1, Host: o.Host, Account: h.Account, App: h.App, Channel: "stable", CreatedAt: f.now} |
| 32 | plan14PutRecord(t, f.claims, rec) |
| 33 | _, etag, err := f.claims.Get(t.Context(), h.Key) |
| 34 | if err != nil { |
| 35 | t.Fatalf("%v: %v", err, errors.Unwrap(err)) |
| 36 | } |
| 37 | reg := restartTestRegistry(t) |
| 38 | journals, err := hostbox.NewJournalService(f.journal) |
| 39 | if err != nil { |
| 40 | t.Fatalf("%v: %v", err, errors.Unwrap(err)) |
| 41 | } |
| 42 | guard := f.guard |
| 43 | guard.Root = hostguard.NewRootAuthority(&rollbackMemory{files: map[string][]byte{}}) |
| 44 | roots := map[string]string{"data": t.TempDir(), "opt": t.TempDir(), "units": t.TempDir()} |
| 45 | actor := hostbox.Actor{Box: "box-1", Enrollment: "en", Revision: 1, Binding: "1", Region: "ie"} |
| 46 | census := &hostbox.EmptyInstallationCensus{Inventory: journals.Inventory, Registry: reg, Launches: f.journal, Box: actor.Box, Enrollment: actor.Enrollment, DirectoryRevision: "1", Roots: roots} |
| 47 | root := ClaimRootHTTP{Journals: journals, Launches: f.journal, Roots: roots, InventoryActor: func(context.Context) (hostbox.Actor, error) { return actor, nil }, Reconcile: hostbox.ReconcileAdapter{Registry: reg, Inventory: journals.Inventory, Census: census, Box: actor.Box, Enrollment: actor.Enrollment}, Stop: ClaimStopHTTP{Coordinator: ClaimStopCoordinator{Guard: guard, Evidence: f.claims}}} |
| 48 | expected := hostbox.InventoryExpectation{RecordKey: h.Key, RecordETag: etag, Target: hostguard.FenceTarget{Host: o.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Box: actor.Box, Enrollment: actor.Enrollment, ClaimETag: o.ETag, Op: o.Claim.Op.ID, Phase: o.Claim.Op.Phase}} |
| 49 | in := hostbox.StopInventory{Complete: true, Host: o.Host, Incarnation: h.Incarnation, InstallationStamp: h.Stamp, Box: actor.Box, Enrollment: actor.Enrollment, DirectoryRevision: "1"} |
| 50 | switch attack { |
| 51 | case "record etag": |
| 52 | expected.RecordETag = "other" |
| 53 | case "claim etag": |
| 54 | expected.Target.ClaimETag = "other" |
| 55 | case "account": |
| 56 | expected.RecordKey = "apps/bap/jam/instances/" + o.Host + ".json" |
| 57 | case "revision": |
| 58 | expected.Revision = 8 |
| 59 | case "foreign root": |
| 60 | in.Entries = []hostbox.StopInventoryEntry{{Kind: "path", ManagedRoot: "opt", RelativePath: "secret", Ownership: "forged", ArtifactID: "forged", Materialization: "intended"}} |
| 61 | } |
| 62 | var request hostbox.JournalRequest |
| 63 | _, client := hostbox.RemoteJournals(func(ctx context.Context, q hostbox.JournalRequest) (hostbox.JournalResponse, error) { |
| 64 | request = q |
| 65 | return root.commitRootInventory(ctx, q) |
| 66 | }) |
| 67 | _, err = client.WriteExpected(t.Context(), in, expected) |
| 68 | if attack != "none" { |
| 69 | if err == nil { |
| 70 | t.Fatal("untrusted installation established") |
| 71 | } |
| 72 | if _, e := guard.Capture(o.Host); e == nil { |
| 73 | t.Fatal("refusal wrote root installation") |
| 74 | } |
| 75 | return |
| 76 | } |
| 77 | if err != nil { |
| 78 | t.Fatalf("%v: %v", err, errors.Unwrap(err)) |
| 79 | } |
| 80 | v, err := guard.Root.View(t.Context(), o.Host, hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp}) |
| 81 | if err != nil || v.Installation.InventoryRevision != 1 || v.Installation.Record != h.Key { |
| 82 | t.Fatalf("installation snapshot: %+v %v", v, err) |
| 83 | } |
| 84 | if _, err := root.commitRootInventory(t.Context(), request); err != nil { |
| 85 | t.Fatal("lost response replay", err) |
| 86 | } |
| 87 | _, rev, err := journals.Inventory.Snapshot(t.Context(), o.Host, v.Captured) |
| 88 | if err != nil || rev != 1 { |
| 89 | t.Fatalf("replay changed revision: %d %v", rev, err) |
| 90 | } |
| 91 | }) |
| 92 | } |
| 93 | } |
| 94 | |
| 95 | func TestPlan14InventoryRegistrationCompletesWithoutAgentLock(t *testing.T) { |
| 96 | t.Parallel() |
| 97 | testInventoryRegistration(t, false) |
| 98 | } |
| 99 | func TestPlan14RootInventoryPersistentProducer(t *testing.T) { testInventoryRegistration(t, true) } |
| 100 | func TestPlan14RootInventoryRejectsOldBootMaterialization(t *testing.T) { |
| 101 | t.Parallel() |
| 102 | testInventoryRegistration(t, true, true) |
| 103 | } |
| 104 | |
| 105 | func testInventoryRegistration(t *testing.T, persistent bool, oldBoot ...bool) { |
| 106 | testInventoryRegistrationWith(t, persistent, oldBoot, nil, nil) |
| 107 | } |
| 108 | func testInventoryRegistrationWith(t *testing.T, persistent bool, oldBoot []bool, authority rootstate.Store, after func(*testing.T, *recoveryFixture, ClaimRootHTTP, registry.Route, hostbox.Actor)) { |
| 109 | cases := []bool{false, true} |
| 110 | if after != nil { |
| 111 | cases = []bool{false} |
| 112 | } |
| 113 | for _, stale := range cases { |
| 114 | t.Run(fmt.Sprint(stale), func(t *testing.T) { |
| 115 | f := newRecoveryFixture(t) |
| 116 | if after != nil { |
| 117 | o := f.read() |
| 118 | if _, err := hostclaim.Update(t.Context(), f.claims, o, hostclaim.AdmitWriter(hostclaim.Admission{ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: "box-1", Region: "ie", Mode: "cold", NewGeneration: "first-generation"})); err != nil { |
| 119 | t.Fatal(err) |
| 120 | } |
| 121 | } |
| 122 | obs := f.read() |
| 123 | holder := obs.Claim.Holder |
| 124 | rec := provision.Record{Schema: 1, Host: obs.Host, Account: holder.Account, App: holder.App, Channel: "stable", CreatedAt: f.now} |
| 125 | plan14PutRecord(t, f.claims, rec) |
| 126 | reg := restartTestRegistry(t) |
| 127 | journals, err := hostbox.NewJournalService(f.journal) |
| 128 | if err != nil { |
| 129 | t.Fatal(err) |
| 130 | } |
| 131 | rootGuard := f.guard |
| 132 | rootGuard.Root = hostguard.NewRootAuthority(&rollbackDisk{dir: t.TempDir()}) |
| 133 | if authority != nil { |
| 134 | rootGuard.Root = hostguard.NewRootAuthority(authority) |
| 135 | } |
| 136 | roots := map[string]string{"data": f.guard.DataDir, "opt": t.TempDir(), "units": t.TempDir()} |
| 137 | actor := hostbox.Actor{Box: "box-1", Enrollment: "en", Revision: 1, Binding: "1", Region: "ie"} |
| 138 | census := &hostbox.EmptyInstallationCensus{Inventory: journals.Inventory, Registry: reg, Launches: f.journal, Box: actor.Box, Enrollment: actor.Enrollment, DirectoryRevision: "1", Roots: roots, SharedPaths: map[string][]string{"data": {"hostguard"}}} |
| 139 | root := ClaimRootHTTP{Journals: journals, Launches: f.journal, Roots: roots, InventoryActor: func(context.Context) (hostbox.Actor, error) { return actor, nil }, Reconcile: hostbox.ReconcileAdapter{Registry: reg, Inventory: journals.Inventory, Census: census, Box: actor.Box, Enrollment: actor.Enrollment}, Stop: ClaimStopHTTP{Coordinator: ClaimStopCoordinator{Guard: rootGuard, Evidence: f.claims}}} |
| 140 | if persistent { |
| 141 | root.inventoryIdentity = rootstate.ObjectIdentity |
| 142 | } |
| 143 | _, remote := hostbox.RemoteJournals(func(ctx context.Context, q hostbox.JournalRequest) (hostbox.JournalResponse, error) { |
| 144 | if q.Op == "inventory-write" { |
| 145 | if persistent { |
| 146 | for i, e := range q.Inventory.Entries { |
| 147 | if e.Kind == "path" { |
| 148 | q.Inventory.Entries[i].Identity = rootstate.FileIdentity{Boot: e.Identity.Boot, Filesystem: "fake", Handle: "fake"} |
| 149 | if len(oldBoot) > 0 && e.Materialization == "materialized" { |
| 150 | q.Inventory.Entries[i].Identity.Boot = "previous-boot" |
| 151 | } |
| 152 | } |
| 153 | } |
| 154 | } |
| 155 | var out hostbox.JournalResponse |
| 156 | var err error |
| 157 | if persistent { |
| 158 | _, sender := hostbox.RemoteJournals(func(ctx context.Context, signed hostbox.JournalRequest) (hostbox.JournalResponse, error) { |
| 159 | var e error |
| 160 | out, e = root.commitRootInventory(ctx, signed) |
| 161 | if e == nil { |
| 162 | if _, replayErr := root.commitRootInventory(ctx, signed); replayErr != nil { |
| 163 | t.Fatal("root identity replay", replayErr) |
| 164 | } |
| 165 | } |
| 166 | return out, e |
| 167 | }) |
| 168 | _, err = sender.WriteExpected(ctx, q.Inventory, hostbox.InventoryExpectation{Revision: q.Revision, RecordKey: q.RecordKey, RecordETag: q.RecordETag, Target: q.Target}) |
| 169 | out.ID = q.ID |
| 170 | } else { |
| 171 | out, err = root.commitRootInventory(ctx, q) |
| 172 | } |
| 173 | if stale && q.Revision == 1 && err == nil { |
| 174 | rec.RecordSeq++ |
| 175 | plan14PutRecord(t, f.claims, rec) |
| 176 | } |
| 177 | return out, err |
| 178 | } |
| 179 | return journals.Execute(ctx, actor.Box+"/"+actor.Enrollment, q) |
| 180 | }) |
| 181 | guard := f.guard |
| 182 | guard.FenceView = rootGuard.Root.View |
| 183 | source := hostbox.WorkSource{Guard: guard, Box: actor.Box, Region: "ie", Now: time.Now, Policy: func(context.Context, *hostclaim.Claim, hostbox.Selection, hostwrite.Identity) error { return nil }, Local: func(context.Context, *hostclaim.Claim, hostbox.Selection) error { return nil }} |
| 184 | adapter := hostbox.ReconcileAdapter{Registry: reg, Inventory: remote, Box: actor.Box, Enrollment: actor.Enrollment, DirectoryRevision: "1", Census: census, Reconciler: hostbox.Reconciler{Claims: f.claims, Guard: guard, Box: actor.Box, Region: "ie", Now: time.Now, Writes: hostwrite.Config{Enabled: true, Selector: source}, Local: park.Local{Guard: guard, Claims: f.claims, DataDir: guard.DataDir, Box: actor.Box}, CheckRecord: func(context.Context, hostbox.RecordSelection) error { return nil }, CheckLocal: func(context.Context, hostclaim.Observed, hostbox.RecordSelection) error { return nil }}} |
| 185 | row := registry.Route{Host: rec.Host, Account: rec.Account, App: rec.App, Kind: "instance", DBPath: provision.NestedDBPath(guard.DataDir, rec.Host), Socket: provision.NestedSocket(rec.Host)} |
| 186 | ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) |
| 187 | defer cancel() |
| 188 | writes := 0 |
| 189 | err = adapter.Install(ctx, provision.ClaimRecord{Host: rec.Host, Key: holder.Key, Account: rec.Account, App: rec.App, Family: "instance", CreatedAt: rec.CreatedAt, Value: rec, Route: row}, func(owner registry.ClaimOwnership) error { |
| 190 | writes++ |
| 191 | row.ClaimOwnership = owner |
| 192 | return reg.Upsert(row) |
| 193 | }) |
| 194 | if stale { |
| 195 | if err == nil || writes != 0 { |
| 196 | t.Fatalf("stale response applied: %v %d", err, writes) |
| 197 | } |
| 198 | return |
| 199 | } |
| 200 | if err != nil || writes != 1 { |
| 201 | t.Fatalf("registration failed or blocked: %v (%v), effects=%d", err, errors.Unwrap(err), writes) |
| 202 | } |
| 203 | _, revision, err := journals.Inventory.Snapshot(t.Context(), rec.Host, hostguard.Captured{Incarnation: holder.Incarnation, Stamp: holder.Stamp}) |
| 204 | if err != nil || revision != 3 { |
| 205 | t.Fatalf("registration did not complete all phases: %d %v", revision, err) |
| 206 | } |
| 207 | if err := os.Mkdir(filepath.Join(guard.DataDir, "instances"), 0700); err != nil { |
| 208 | t.Fatal(err) |
| 209 | } |
| 210 | work, err := source.Select(t.Context(), rec.Host, "spawn") |
| 211 | if err != nil { |
| 212 | t.Fatal(err) |
| 213 | } |
| 214 | roles := &ClaimRoleAssembly{Inventory: remote, ManagedRoots: roots, Guard: guard, Claims: f.claims} |
| 215 | err = roles.prepareStateRemote(ctx, f.read(), work, ClaimLaunchInputs{DBPath: row.DBPath}, actor) |
| 216 | if len(oldBoot) > 0 { |
| 217 | if err == nil { |
| 218 | t.Fatal("old-boot numeric identity admitted new materialization") |
| 219 | } |
| 220 | saved, _, snapshotErr := journals.Inventory.Snapshot(t.Context(), rec.Host, hostguard.Captured{Incarnation: holder.Incarnation, Stamp: holder.Stamp}) |
| 221 | if snapshotErr != nil { |
| 222 | t.Fatal(snapshotErr) |
| 223 | } |
| 224 | for _, e := range saved.Entries { |
| 225 | if e.Kind == "path" && e.Materialization == "materialized" { |
| 226 | t.Fatal("refused old boot wrote materialized authority") |
| 227 | } |
| 228 | } |
| 229 | return |
| 230 | } |
| 231 | if err != nil { |
| 232 | t.Fatalf("state registration failed or blocked: %v", err) |
| 233 | } |
| 234 | saved, revision, err := journals.Inventory.Snapshot(t.Context(), rec.Host, hostguard.Captured{Incarnation: holder.Incarnation, Stamp: holder.Stamp}) |
| 235 | if persistent && err == nil { |
| 236 | for _, e := range saved.Entries { |
| 237 | if e.Kind == "path" && (e.Identity.Boot == "" || e.Identity.Filesystem == "fake" || e.Identity.Handle == "fake") { |
| 238 | t.Fatal("agent minted root identity", e) |
| 239 | } |
| 240 | } |
| 241 | } |
| 242 | if err != nil || revision != 5 { |
| 243 | t.Fatalf("state phases missing: %d %v", revision, err) |
| 244 | } |
| 245 | if after != nil { |
| 246 | after(t, f, root, row, actor) |
| 247 | } |
| 248 | |
| 249 | }) |
| 250 | } |
| 251 | } |
| 252 | |
amadan