| 1 | //go:build linux |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "os" |
| 8 | "path/filepath" |
| 9 | "testing" |
| 10 | "time" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/hostactors" |
| 13 | "github.com/carlosframework/platform/internal/hostbox" |
| 14 | "github.com/carlosframework/platform/internal/hostclaim" |
| 15 | "github.com/carlosframework/platform/internal/hostguard" |
| 16 | "github.com/carlosframework/platform/internal/hostwrite" |
| 17 | "github.com/carlosframework/platform/internal/park" |
| 18 | "github.com/carlosframework/platform/internal/provision" |
| 19 | "github.com/carlosframework/platform/internal/registry" |
| 20 | ) |
| 21 | |
| 22 | // bindingInstallWorld is one box's agent and root halves over one roster: the |
| 23 | // agent's ReconcileAdapter pinned to a verified actor's binding, installing |
| 24 | // through the remote journal into root's real commitRootInventory, and root's |
| 25 | // inventory actor the production identity, fresh on every read (agent mode's |
| 26 | // actorForRoot). |
| 27 | type bindingInstallWorld struct { |
| 28 | roster *bindingRosterFixture |
| 29 | identity hostbox.IdentityProvider |
| 30 | f *recoveryFixture |
| 31 | rec provision.Record |
| 32 | route registry.Route |
| 33 | reg *registry.Registry |
| 34 | journals *hostbox.JournalService |
| 35 | roots map[string]string |
| 36 | guard hostguard.Guard |
| 37 | rootAuth *hostguard.RootAuthority |
| 38 | // rebuilt by pin |
| 39 | pinned hostbox.Actor |
| 40 | root ClaimRootHTTP |
| 41 | census *hostbox.EmptyInstallationCensus |
| 42 | adapter hostbox.ReconcileAdapter |
| 43 | remote *hostbox.StopInventoryDB |
| 44 | source hostbox.WorkSource |
| 45 | } |
| 46 | |
| 47 | func newBindingInstallWorld(t *testing.T) *bindingInstallWorld { |
| 48 | t.Helper() |
| 49 | w := &bindingInstallWorld{roster: newBindingRosterFixture(t, "box-1", "ie")} |
| 50 | w.identity = w.roster.identity() |
| 51 | w.f = newRecoveryFixture(t) |
| 52 | obs := w.f.read() |
| 53 | holder := obs.Claim.Holder |
| 54 | w.rec = provision.Record{Schema: 1, Host: obs.Host, Account: holder.Account, App: holder.App, Channel: "stable", CreatedAt: w.f.now} |
| 55 | plan14PutRecord(t, w.f.claims, w.rec) |
| 56 | w.reg = restartTestRegistry(t) |
| 57 | var err error |
| 58 | if w.journals, err = hostbox.NewJournalService(w.f.journal); err != nil { |
| 59 | t.Fatal(err) |
| 60 | } |
| 61 | w.rootAuth = hostguard.NewRootAuthority(&rollbackDisk{dir: t.TempDir()}) |
| 62 | w.roots = map[string]string{"data": w.f.guard.DataDir, "opt": t.TempDir(), "units": t.TempDir()} |
| 63 | w.route = registry.Route{Host: w.rec.Host, Account: w.rec.Account, App: w.rec.App, Kind: "instance", DBPath: provision.NestedDBPath(w.f.guard.DataDir, w.rec.Host), Socket: provision.NestedSocket(w.rec.Host)} |
| 64 | w.start(t) |
| 65 | return w |
| 66 | } |
| 67 | |
| 68 | // start is an agent and root start: Verify once, pin the binding into the |
| 69 | // agent's adapter and census exactly as ClaimAssembly.Build does |
| 70 | // (claimassembly.go, Reconcile.DirectoryRevision and the census), and wire |
| 71 | // root with the fresh production identity. |
| 72 | func (w *bindingInstallWorld) start(t *testing.T) { |
| 73 | t.Helper() |
| 74 | pinned, err := w.identity.Verify(t.Context()) |
| 75 | if err != nil { |
| 76 | t.Fatal(err) |
| 77 | } |
| 78 | if !hostactors.ValidBinding(pinned.Binding) { |
| 79 | t.Fatalf("pinned binding %q", pinned.Binding) |
| 80 | } |
| 81 | w.pinned = pinned |
| 82 | rootGuard := w.f.guard |
| 83 | rootGuard.Root = w.rootAuth |
| 84 | w.census = &hostbox.EmptyInstallationCensus{Inventory: w.journals.Inventory, Registry: w.reg, Launches: w.f.journal, Box: pinned.Box, Enrollment: pinned.Enrollment, DirectoryRevision: pinned.Binding, Roots: w.roots, SharedPaths: map[string][]string{"data": {"hostguard"}}} |
| 85 | w.root = ClaimRootHTTP{Journals: w.journals, Launches: w.f.journal, Roots: w.roots, InventoryActor: w.identity.Verify, Reconcile: hostbox.ReconcileAdapter{Registry: w.reg, Inventory: w.journals.Inventory, Census: w.census, Box: pinned.Box, Enrollment: pinned.Enrollment}, Stop: ClaimStopHTTP{Coordinator: ClaimStopCoordinator{Guard: rootGuard, Evidence: w.f.claims}}} |
| 86 | root := w.root |
| 87 | _, w.remote = hostbox.RemoteJournals(func(ctx context.Context, q hostbox.JournalRequest) (hostbox.JournalResponse, error) { |
| 88 | if q.Op == "inventory-write" { |
| 89 | return root.commitRootInventory(ctx, q) |
| 90 | } |
| 91 | return w.journals.Execute(ctx, pinned.Box+"/"+pinned.Enrollment, q) |
| 92 | }) |
| 93 | w.guard = w.f.guard |
| 94 | w.guard.FenceView = w.rootAuth.View |
| 95 | // The production policy, unit-shape refusal included (G10): an allow-all |
| 96 | // policy here hid the refusal of every fresh install. |
| 97 | policy, _ := witnessPolicy(t, w.reg) |
| 98 | w.source = hostbox.WorkSource{Guard: w.guard, Box: pinned.Box, Region: "ie", Now: time.Now, Policy: policy, Local: func(context.Context, *hostclaim.Claim, hostbox.Selection) error { return nil }} |
| 99 | w.adapter = hostbox.ReconcileAdapter{Registry: w.reg, Inventory: w.remote, Box: pinned.Box, Enrollment: pinned.Enrollment, DirectoryRevision: pinned.Binding, Census: w.census, Reconciler: hostbox.Reconciler{Claims: w.f.claims, Guard: w.guard, Box: pinned.Box, Region: "ie", Now: time.Now, Writes: hostwrite.Config{Enabled: true, Selector: w.source}, Local: park.Local{Guard: w.guard, Claims: w.f.claims, DataDir: w.guard.DataDir, Box: pinned.Box}, CheckRecord: func(context.Context, hostbox.RecordSelection) error { return nil }, CheckLocal: func(context.Context, hostclaim.Observed, hostbox.RecordSelection) error { return nil }}} |
| 100 | } |
| 101 | |
| 102 | func (w *bindingInstallWorld) install(t *testing.T) { |
| 103 | t.Helper() |
| 104 | holder := w.f.read().Claim.Holder |
| 105 | ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) |
| 106 | defer cancel() |
| 107 | writes := 0 |
| 108 | err := w.adapter.Install(ctx, provision.ClaimRecord{Host: w.rec.Host, Key: holder.Key, Account: w.rec.Account, App: w.rec.App, Family: "instance", CreatedAt: w.rec.CreatedAt, Value: w.rec, Route: w.route}, func(owner registry.ClaimOwnership) error { |
| 109 | writes++ |
| 110 | w.route.ClaimOwnership = owner |
| 111 | return w.reg.Upsert(w.route) |
| 112 | }) |
| 113 | if err != nil || writes != 1 { |
| 114 | t.Fatalf("install: %v, effects=%d", err, writes) |
| 115 | } |
| 116 | } |
| 117 | |
| 118 | func (w *bindingInstallWorld) inventory(t *testing.T) (hostbox.StopInventory, uint64) { |
| 119 | t.Helper() |
| 120 | holder := w.f.read().Claim.Holder |
| 121 | in, revision, err := w.journals.Inventory.Snapshot(t.Context(), w.rec.Host, hostguard.Captured{Incarnation: holder.Incarnation, Stamp: holder.Stamp}) |
| 122 | if err != nil { |
| 123 | t.Fatal(err) |
| 124 | } |
| 125 | return in, revision |
| 126 | } |
| 127 | |
| 128 | // L2 and L3 (roster binding): install at the binding B0 through the remote |
| 129 | // path into root's commitRootInventory, then six real roster writes that move |
| 130 | // the write counter (hold reserve and release, activation reserve and cancel, |
| 131 | // an enrollment attempt, a role change). Without a restart (L2), and after |
| 132 | // the agent and root start again and re-pin (L3), every comparison still |
| 133 | // passes: the agent's census, a second inventory write through root |
| 134 | // (prepareStateRemote), the regional and move-park bound identity and |
| 135 | // VerifyRegional, and Verify's fresh actor is SameBinding with the pin. |
| 136 | // |
| 137 | // Root's inventory actor is the fresh production identity, as in agent mode. |
| 138 | // Instance-control mode's startup-pinned inventory actor becomes fresh in |
| 139 | // phase B (task 5); L4 covers it there. |
| 140 | func TestClaimBindingSurvivesRosterWritesWithoutRestart(t *testing.T) { |
| 141 | testClaimBindingSurvivesRosterWrites(t, false) |
| 142 | } |
| 143 | |
| 144 | func TestClaimBindingSurvivesRosterWritesAfterRestart(t *testing.T) { |
| 145 | testClaimBindingSurvivesRosterWrites(t, true) |
| 146 | } |
| 147 | |
| 148 | func testClaimBindingSurvivesRosterWrites(t *testing.T, restart bool) { |
| 149 | ctx := t.Context() |
| 150 | w := newBindingInstallWorld(t) |
| 151 | b0 := w.pinned.Binding |
| 152 | w.install(t) |
| 153 | in, revision := w.inventory(t) |
| 154 | if in.DirectoryRevision != b0 || revision != 3 { |
| 155 | t.Fatalf("installed stamp %q (want the binding %q), inventory revision %d", in.DirectoryRevision, b0, revision) |
| 156 | } |
| 157 | |
| 158 | before := w.roster.read().Roster.Revision |
| 159 | w.roster.sixWrites() |
| 160 | fresh, err := w.identity.Verify(ctx) |
| 161 | if err != nil { |
| 162 | t.Fatal("Verify after six roster writes:", err) |
| 163 | } |
| 164 | if fresh.Revision != before+6 || !hostbox.SameBinding(fresh, w.pinned) || fresh.Binding != b0 { |
| 165 | t.Fatalf("fresh actor revision %d (pinned %d), binding %q (pinned %q)", fresh.Revision, w.pinned.Revision, fresh.Binding, b0) |
| 166 | } |
| 167 | pinned := w.pinned |
| 168 | if restart { |
| 169 | w.start(t) |
| 170 | if w.pinned.Binding != b0 || w.pinned.Revision == pinned.Revision { |
| 171 | t.Fatalf("restart pinned %q at revision %d", w.pinned.Binding, w.pinned.Revision) |
| 172 | } |
| 173 | } |
| 174 | |
| 175 | // A create's hold stays held through the comparisons below, as the |
| 176 | // create's own hold does while its box installs. |
| 177 | held := w.roster.reserveHold() |
| 178 | if fresh, err = w.identity.Verify(ctx); err != nil || fresh.Binding != b0 { |
| 179 | t.Fatalf("Verify with a hold held: binding %q, %v", fresh.Binding, err) |
| 180 | } |
| 181 | holder := w.f.read().Claim.Holder |
| 182 | if err := w.census.CheckLocal(ctx, w.f.read(), hostbox.RecordSelection{Host: w.rec.Host, Incarnation: holder.Incarnation}); err != nil { |
| 183 | t.Fatal("agent census after roster writes:", err) |
| 184 | } |
| 185 | |
| 186 | // The second inventory write: state registration through root. |
| 187 | if err := os.Mkdir(filepath.Join(w.guard.DataDir, "instances"), 0700); err != nil { |
| 188 | t.Fatal(err) |
| 189 | } |
| 190 | work, err := w.source.Select(ctx, w.rec.Host, "spawn") |
| 191 | if err != nil { |
| 192 | t.Fatal(err) |
| 193 | } |
| 194 | roles := &ClaimRoleAssembly{Inventory: w.remote, ManagedRoots: w.roots, Guard: w.guard, Claims: w.f.claims} |
| 195 | if err := roles.prepareStateRemote(ctx, w.f.read(), work, ClaimLaunchInputs{DBPath: w.route.DBPath}, fresh); err != nil { |
| 196 | t.Fatal("state registration after roster writes:", err) |
| 197 | } |
| 198 | if in, revision = w.inventory(t); in.DirectoryRevision != b0 || revision != 5 { |
| 199 | t.Fatalf("after state registration: stamp %q, revision %d", in.DirectoryRevision, revision) |
| 200 | } |
| 201 | |
| 202 | // The regional request store's and the move parks' bound identity, and |
| 203 | // the regional enrollment check, against the actor this start pinned. |
| 204 | if err := claimBoundIdentity(w.identity.Verify, w.pinned)(ctx); err != nil { |
| 205 | t.Fatal("bound identity after roster writes:", err) |
| 206 | } |
| 207 | if err := (claimRegionalEnrollment{identity: w.identity, actor: w.pinned}).VerifyRegional(ctx, w.pinned.Region, w.pinned.Bucket, hostbox.RequestLimit); err != nil { |
| 208 | t.Fatal("VerifyRegional after roster writes:", err) |
| 209 | } |
| 210 | |
| 211 | // The hold is released and more writes land: nothing pins the counter. |
| 212 | w.roster.releaseHold(held) |
| 213 | w.roster.holdChurn() |
| 214 | if err := claimBoundIdentity(w.identity.Verify, w.pinned)(ctx); err != nil { |
| 215 | t.Fatal("bound identity after a further hold:", err) |
| 216 | } |
| 217 | if err := w.census.CheckLocal(ctx, w.f.read(), hostbox.RecordSelection{Host: w.rec.Host, Incarnation: holder.Incarnation}); err != nil { |
| 218 | t.Fatal("census after a further hold:", err) |
| 219 | } |
| 220 | |
| 221 | // A binding change is still refused by the same comparisons: quarantine |
| 222 | // and readmission (direct signed write, see readmit) move the binding. |
| 223 | w.roster.quarantine("loss_1") |
| 224 | w.roster.readmit() |
| 225 | readmitted, err := w.identity.Verify(ctx) |
| 226 | if err != nil { |
| 227 | t.Fatal(err) |
| 228 | } |
| 229 | if hostbox.SameBinding(readmitted, w.pinned) { |
| 230 | t.Fatal("readmitted actor is SameBinding with the startup pin") |
| 231 | } |
| 232 | if err := claimBoundIdentity(w.identity.Verify, w.pinned)(ctx); err == nil { |
| 233 | t.Fatal("bound identity accepted a readmitted box") |
| 234 | } |
| 235 | if err := (claimRegionalEnrollment{identity: w.identity, actor: w.pinned}).VerifyRegional(ctx, w.pinned.Region, w.pinned.Bucket, hostbox.RequestLimit); err == nil { |
| 236 | t.Fatal("VerifyRegional accepted a readmitted box") |
| 237 | } |
| 238 | } |
| 239 | |
amadan