amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package host
2
3import (
4 "context"
5 "fmt"
6 "testing"
7 "time"
8
9 "github.com/carlosframework/platform/internal/hostbox"
10 "github.com/carlosframework/platform/internal/hostclaim"
11 "github.com/carlosframework/platform/internal/hostguard"
12 "github.com/carlosframework/platform/internal/provision"
13)
14
15func plan14InstallRegistration(t *testing.T, kind, phase string) (*recoveryFixture, ClaimRootHTTP, hostbox.JournalRequest) {
16 t.Helper()
17 f := newRecoveryFixture(t)
18 if _, err := hostclaim.Update(t.Context(), f.claims, f.read(), func(c *hostclaim.Claim) error {
19 if kind != hostclaim.KindServing {
20 c.Op = hostclaim.Op{ID: "install-op", Kind: kind, Phase: phase, Exec: "box-1", Participants: []string{"box-0", "box-1"}, Intent: map[string]string{"source": "box-0", "target": "box-1", "to": "ie", "region": "ie", "source_box": "box-0", "target_box": "box-1"}, Journal: map[string]string{"install_stamp": fmt.Sprint(c.Holder.Stamp)}}
21 }
22 return nil
23 }); err != nil {
24 t.Fatal(err)
25 }
26 o := f.read()
27 h := o.Claim.Holder
28 rec := provision.Record{Schema: 1, Host: o.Host, Account: h.Account, App: h.App, Channel: "stable", Region: "ie", CreatedAt: f.now}
29 if kind == hostclaim.KindLanding {
30 rec.Move = &provision.MoveState{ID: o.Claim.Op.ID, Incarnation: h.Incarnation, From: "uk", To: "ie", State: provision.MoveLanding, LandingPhase: phase}
31 }
32 plan14PutRecord(t, f.claims, rec)
33 _, etag, err := f.claims.Get(t.Context(), h.Key)
34 if err != nil {
35 t.Fatal(err)
36 }
37 reg := restartTestRegistry(t)
38 journals, err := hostbox.NewJournalService(f.journal)
39 if err != nil {
40 t.Fatal(err)
41 }
42 guard := f.guard
43 guard.Root = hostguard.NewRootAuthority(&rollbackDisk{dir: t.TempDir()})
44 actor := hostbox.Actor{Box: "box-1", Enrollment: "en", Region: "ie", Revision: 1, Binding: "1"}
45 roots := map[string]string{"data": guard.DataDir, "opt": t.TempDir(), "units": t.TempDir()}
46 census := &hostbox.EmptyInstallationCensus{Inventory: journals.Inventory, Registry: reg, Launches: f.journal, Box: actor.Box, Enrollment: actor.Enrollment, DirectoryRevision: "1", Roots: roots, SharedPaths: map[string][]string{"data": {"hostguard"}}}
47 root := ClaimRootHTTP{Journals: journals, Launches: f.journal, Roots: roots, InventoryActor: func(context.Context) (hostbox.Actor, error) { return actor, nil }, Reconcile: hostbox.ReconcileAdapter{Registry: reg, Inventory: journals.Inventory, Census: census}, Stop: ClaimStopHTTP{Coordinator: ClaimStopCoordinator{Guard: guard, Evidence: f.claims}}}
48 q := hostbox.JournalRequest{V: 3, ID: "registration-1", Op: "inventory-write", RecordKey: h.Key, RecordETag: etag, Target: hostguard.FenceTarget{Host: o.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Box: actor.Box, Enrollment: actor.Enrollment, ClaimETag: o.ETag, Op: o.Claim.Op.ID, Phase: o.Claim.Op.Phase}, Inventory: hostbox.StopInventory{Complete: true, Host: o.Host, Incarnation: h.Incarnation, InstallationStamp: h.Stamp, Box: actor.Box, Enrollment: actor.Enrollment, DirectoryRevision: "1"}}
49 _, client := hostbox.RemoteJournals(func(_ context.Context, prepared hostbox.JournalRequest) (hostbox.JournalResponse, error) {
50 q = prepared
51 return hostbox.JournalResponse{V: 1, ID: prepared.ID, Revision: 1}, nil
52 })
53 if _, err := client.WriteExpected(t.Context(), q.Inventory, hostbox.InventoryExpectation{RecordKey: q.RecordKey, RecordETag: q.RecordETag, Target: q.Target}); err != nil {
54 t.Fatal(err)
55 }
56 return f, root, q
57}
58
59func TestPlan14FirstRegistrationOperations(t *testing.T) {
60 t.Parallel()
61 for _, kind := range []string{hostclaim.KindServing, hostclaim.KindRelocating, hostclaim.KindLanding} {
62 for _, attack := range []string{"none", "phase", "box", "session", "enrollment", "record", "op", "stamp", "agent row"} {
63 t.Run(kind+"/"+attack, func(t *testing.T) {
64 phase := hostclaim.PhaseInstalling
65 if kind == hostclaim.KindLanding {
66 phase = provision.LandChecking
67 }
68 f, root, q := plan14InstallRegistration(t, kind, phase)
69 switch attack {
70 case "phase":
71 q.Target.Phase = "wrong"
72 case "box":
73 q.Target.Box = "box-0"
74 case "session":
75 q.Target.Session++
76 case "enrollment":
77 q.Target.Enrollment = "other"
78 case "record":
79 q.RecordETag = "other"
80 case "op":
81 q.Target.Op = "other"
82 case "stamp":
83 q.Target.Stamp++
84 case "agent row":
85 rt, _, err := rootClaimRecord(t.Context(), f.claims, f.read().Claim, root.Roots["data"])
86 if err != nil {
87 t.Fatal(err)
88 }
89 if err = root.Reconcile.Registry.Upsert(rt); err != nil {
90 t.Fatal(err)
91 }
92 }
93 out, err := root.commitRootInventory(t.Context(), q)
94 if attack != "none" {
95 if err == nil {
96 t.Fatal("unauthorized registration")
97 }
98 return
99 }
100 if err != nil {
101 t.Fatal(err)
102 }
103 if out.Revision != 1 {
104 t.Fatal(out)
105 }
106 if _, err = root.commitRootInventory(t.Context(), q); err != nil {
107 t.Fatal("replay", err)
108 }
109 if kind == hostclaim.KindLanding {
110 row, err := root.Reconcile.Registry.Get(q.Target.Host)
111 if err != nil || row == nil || row.HoldOp != q.Target.Op || row.Status != "landing" || row.DBPath != provision.DefaultDBPath(root.Roots["data"], q.Target.Host) || !row.ClaimOwnership.Valid() {
112 t.Fatalf("root held row: %+v %v", row, err)
113 }
114 }
115 })
116 }
117 }
118}
119
120func TestPlan14FirstRegistrationRefusesLaterPhases(t *testing.T) {
121 t.Parallel()
122 for kind, phases := range map[string][]string{hostclaim.KindLanding: {provision.LandConfigured, provision.LandRestored, provision.LandProbing}, hostclaim.KindRelocating: {hostclaim.PhaseRelocatingProbing, hostclaim.PhaseProbeFailed}, hostclaim.KindRestoring: {provision.LandConfigured}, hostclaim.KindReverting: {provision.LandConfigured}} {
123 for _, phase := range phases {
124 t.Run(kind+"/"+phase, func(t *testing.T) {
125 _, root, q := plan14InstallRegistration(t, kind, phase)
126 if _, err := root.commitRootInventory(t.Context(), q); err == nil {
127 t.Fatal("skipped first registration")
128 }
129 })
130 }
131 }
132}
133
134func plan14PreparedInventory(t *testing.T, root ClaimRootHTTP, q hostbox.JournalRequest) hostbox.JournalRequest {
135 t.Helper()
136 _, client := hostbox.RemoteJournals(func(_ context.Context, p hostbox.JournalRequest) (hostbox.JournalResponse, error) {
137 q = p
138 return hostbox.JournalResponse{V: 1, ID: p.ID, Revision: p.Revision + 1}, nil
139 })
140 if _, err := client.WriteExpected(t.Context(), q.Inventory, hostbox.InventoryExpectation{Revision: q.Revision, RecordKey: q.RecordKey, RecordETag: q.RecordETag, Target: q.Target}); err != nil {
141 t.Fatal(err)
142 }
143 return q
144}
145
146func TestPlan14LandingConfiguredAdoptsOnlyRootRow(t *testing.T) {
147 t.Parallel()
148 for _, attack := range []string{"none", "row id", "row missing", "op", "session", "record", "enrollment", "probe session"} {
149 t.Run(attack, func(t *testing.T) {
150 f, root, q := plan14InstallRegistration(t, hostclaim.KindLanding, provision.LandChecking)
151 if _, err := root.commitRootInventory(t.Context(), q); err != nil {
152 t.Fatal(err)
153 }
154 prior, err := root.Reconcile.Registry.Get(q.Target.Host)
155 if err != nil {
156 t.Fatal(err)
157 }
158 o, err := hostclaim.Update(t.Context(), f.claims, f.read(), func(c *hostclaim.Claim) error {
159 c.Op.Phase = provision.LandConfigured
160 if attack == "probe session" {
161 c.Op.Phase = provision.LandProbing
162 }
163 if attack == "op" {
164 c.Op.ID = "different"
165 }
166 if attack == "session" {
167 c.Holder.Session++
168 }
169 return nil
170 })
171 if err != nil {
172 t.Fatal(err)
173 }
174 rec, _, err := provision.FindRecordByHost(t.Context(), f.claims, o.Host)
175 if err != nil {
176 t.Fatal(err)
177 }
178 if attack == "probe session" {
179 h := o.Claim.Holder
180 o, err = hostclaim.Update(t.Context(), f.claims, o, hostclaim.AdmitWriter(hostclaim.Admission{ObservedOpID: o.Claim.Op.ID, ObservedPhase: o.Claim.Op.Phase, ObservedSession: h.Session, ObservedGeneration: h.Generation, Box: "box-1", Mode: "cold", NewGeneration: "landing-generation", KeepStamp: true, Coverage: "sealed", Region: "ie", PredecessorRegion: "uk"}))
181 if err != nil {
182 t.Fatal(err)
183 }
184 }
185 rec.Move.LandingPhase = o.Claim.Op.Phase
186 rec.Move.ID = o.Claim.Op.ID
187 plan14PutRecord(t, f.claims, *rec)
188 _, q.RecordETag, err = f.claims.Get(t.Context(), q.RecordKey)
189 if err != nil {
190 t.Fatal(err)
191 }
192 q.Target.ClaimETag, q.Target.Phase, q.Target.Op, q.Target.Session = o.ETag, o.Claim.Op.Phase, o.Claim.Op.ID, o.Claim.Holder.Session
193 q.Inventory, q.Revision, err = root.Journals.Inventory.Snapshot(t.Context(), o.Host, hostguard.Captured{Incarnation: q.Target.Incarnation, Stamp: q.Target.Stamp})
194 if err != nil {
195 t.Fatal(err)
196 }
197 switch attack {
198 case "row id":
199 prior.ClaimRowID = "forged"
200 if err = root.Reconcile.Registry.Delete(o.Host); err == nil {
201 err = root.Reconcile.Registry.Upsert(*prior)
202 }
203 case "row missing":
204 err = root.Reconcile.Registry.Delete(o.Host)
205 case "record":
206 q.RecordETag = "stale"
207 case "enrollment":
208 q.Target.Enrollment = "stale"
209 }
210 if err != nil {
211 t.Fatal(err)
212 }
213 q = plan14PreparedInventory(t, root, q)
214 out, err := root.commitRootInventory(t.Context(), q)
215 if attack != "none" && attack != "probe session" {
216 if err == nil {
217 t.Fatal("adopted changed installation")
218 }
219 return
220 }
221 if err != nil {
222 t.Fatal(err)
223 }
224 if out.Revision != 2 {
225 t.Fatal(out)
226 }
227 row, err := root.Reconcile.Registry.Get(o.Host)
228 if err != nil || row.ClaimOwnership != prior.ClaimOwnership {
229 t.Fatalf("identity changed: %+v %v", row, err)
230 }
231 if _, err := root.commitRootInventory(t.Context(), q); err != nil {
232 t.Fatal("configured replay", err)
233 }
234 })
235 }
236}
237
238func TestPlan14LandingRegistrationOutsideHostLock(t *testing.T) {
239 t.Parallel()
240 for _, stale := range []bool{false, true} {
241 t.Run(fmt.Sprint(stale), func(t *testing.T) {
242 f, root, _ := plan14InstallRegistration(t, hostclaim.KindLanding, provision.LandChecking)
243 if err := f.guard.Activate(t.Context(), f.read().Host, "box-1", nil); err != nil {
244 t.Fatal(err)
245 }
246 previous, err := f.guard.Capture(f.read().Host)
247 if err != nil {
248 t.Fatal(err)
249 }
250 if _, err := hostclaim.Update(t.Context(), f.claims, f.read(), func(c *hostclaim.Claim) error { c.Holder.Stamp++; return nil }); err != nil {
251 t.Fatal(err)
252 }
253 guard := f.guard
254 guard.FenceView = root.Stop.Coordinator.Guard.Root.View
255 _, remote := hostbox.RemoteJournals(func(ctx context.Context, q hostbox.JournalRequest) (hostbox.JournalResponse, error) {
256 if q.Op != "inventory-write" {
257 return root.Journals.Execute(ctx, "box-1/en", q)
258 }
259 out, err := root.commitRootInventory(ctx, q)
260 if err == nil && stale {
261 rec, _, e := provision.FindRecordByHost(ctx, f.claims, q.Target.Host)
262 if e != nil {
263 return out, e
264 }
265 rec.RecordSeq++
266 plan14PutRecord(t, f.claims, *rec)
267 }
268 return out, err
269 })
270 a := hostbox.ReconcileAdapter{Inventory: remote, Box: "box-1", Enrollment: "en", DirectoryRevision: "1", Reconciler: hostbox.Reconciler{Claims: f.claims, Guard: guard}}
271 ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)
272 defer cancel()
273 err = a.RegisterLanding(ctx, f.read())
274 if stale {
275 if err == nil {
276 t.Fatal("activated after record changed during RPC")
277 }
278 return
279 }
280 if err != nil {
281 t.Fatal("registration blocked or refused", err)
282 }
283 if err := a.RegisterLanding(ctx, f.read()); err != nil {
284 t.Fatal("checking retry", err)
285 }
286 captured, err := f.guard.Capture(f.read().Host)
287 if err != nil || captured != previous {
288 t.Fatalf("registration hid previous stay: %+v, want %+v: %v", captured, previous, err)
289 }
290 if err := guard.Activate(ctx, f.read().Host, "box-1", nil); err != nil {
291 t.Fatal("ordered activation", err)
292 }
293 })
294 }
295}
296
297type plan14RegistrationCrash struct {
298 *rollbackDisk
299 writes int
300}
301
302func (s *plan14RegistrationCrash) Write(name string, body []byte) error {
303 s.writes++
304 if s.writes == 2 {
305 return fmt.Errorf("injected root commit crash")
306 }
307 return s.rollbackDisk.Write(name, body)
308}
309
310func TestPlan14LandingRegistrationRecoversRootCommit(t *testing.T) {
311 _, root, q := plan14InstallRegistration(t, hostclaim.KindLanding, provision.LandChecking)
312 store := &plan14RegistrationCrash{rollbackDisk: &rollbackDisk{dir: t.TempDir()}}
313 root.Stop.Coordinator.Guard.Root = hostguard.NewRootAuthority(store)
314 if _, err := root.commitRootInventory(t.Context(), q); err == nil {
315 t.Fatal("missed injected crash")
316 }
317 before, err := root.Reconcile.Registry.Get(q.Target.Host)
318 if err != nil || before == nil {
319 t.Fatalf("row before recovery: %v %v", before, err)
320 }
321 if err := root.recoverRootInventory(t.Context()); err != nil {
322 t.Fatal(err)
323 }
324 after, err := root.Reconcile.Registry.Get(q.Target.Host)
325 if err != nil || after == nil || before.ClaimOwnership != after.ClaimOwnership {
326 t.Fatalf("recovery reminted row: %v %v", after, err)
327 }
328 view, err := root.Stop.Coordinator.Guard.Root.View(t.Context(), q.Target.Host, hostguard.Captured{Incarnation: q.Target.Incarnation, Stamp: q.Target.Stamp})
329 if err != nil || view.Installation.InventoryRevision != 1 || view.Installation.Registration.RowID != before.ClaimRowID {
330 t.Fatalf("root recovery: %+v %v", view, err)
331 }
332}
333
334func TestPlan14FirstRegistrationUsesExistingInstallAuthority(t *testing.T) {
335 t.Parallel()
336 for _, mode := range []string{"never started", "writer", "missing writer", "foreign writer", "replacement", "existing install"} {
337 t.Run(mode, func(t *testing.T) {
338 kind, phase := hostclaim.KindServing, ""
339 if mode == "replacement" || mode == "existing install" {
340 kind, phase = hostclaim.KindRelocating, hostclaim.PhaseInstalled
341 }
342 f, root, q := plan14InstallRegistration(t, kind, phase)
343 o, err := hostclaim.Update(t.Context(), f.claims, f.read(), func(c *hostclaim.Claim) error {
344 switch mode {
345 case "writer":
346 c.Holder.WriterBox, c.Holder.Session = "box-1", 1
347 case "missing writer":
348 c.Holder.WriterBox, c.Holder.Session = "", 1
349 case "foreign writer":
350 c.Holder.WriterBox, c.Holder.Session = "box-0", 1
351 case "existing install":
352 c.Op.Journal["local_install"] = "already-installed"
353 }
354 return nil
355 })
356 if err != nil {
357 t.Fatal(err)
358 }
359 q.Target.ClaimETag, q.Target.Session = o.ETag, o.Claim.Holder.Session
360 q = plan14PreparedInventory(t, root, q)
361 _, err = root.commitRootInventory(t.Context(), q)
362 refuse := mode == "missing writer" || mode == "foreign writer" || mode == "existing install"
363 if (err != nil) != refuse {
364 t.Fatalf("refuse=%v: %v", refuse, err)
365 }
366 })
367 }
368}
369