amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1//go:build linux
2
3package host
4
5import (
6 "bytes"
7 "context"
8 "encoding/json"
9 "errors"
10 "fmt"
11 "os"
12 "path/filepath"
13 "reflect"
14 "testing"
15 "time"
16
17 "github.com/carlosframework/platform/internal/activator"
18 "github.com/carlosframework/platform/internal/genselect"
19 "github.com/carlosframework/platform/internal/hostbox"
20 "github.com/carlosframework/platform/internal/hostclaim"
21 "github.com/carlosframework/platform/internal/hostguard"
22 "github.com/carlosframework/platform/internal/hostwrite"
23 "github.com/carlosframework/platform/internal/lease/leasetest"
24 "github.com/carlosframework/platform/internal/provision"
25 "github.com/carlosframework/platform/internal/registry"
26 "github.com/carlosframework/platform/internal/release"
27 "github.com/superfly/ltx"
28 "golang.org/x/sys/unix"
29)
30
31// This is the explicit Task 0 root seam, not a production parking witness.
32// It retains immutable replies across controller restarts so the agent's
33// enrollment/completion order can be tested independently of Task 0R.
34type servingParkFixture struct {
35 continuePending bool
36 t *testing.T
37 f *recoveryFixture
38 db string
39 records map[string]servingParkAuthorization
40 calls []string
41 inside bool
42 failComplete bool
43 afterPark func()
44}
45
46func (p *servingParkFixture) park(current, prior servingArtifactIdentity, legacy bool) (servingParkAuthorization, error) {
47 if p.inside {
48 p.t.Fatal("root called while Work held")
49 }
50 p.calls = append(p.calls, "park")
51 for _, a := range p.records {
52 if a.Successor == current {
53 return a, nil
54 }
55 }
56 if p.continuePending {
57 for id, a := range p.records {
58 a.Successor = current
59 db, _ := filepath.Rel(p.f.guard.DataDir, p.db)
60 a.TagsFrom = p.f.read().Claim.Inventory[current.Box].Paths[db]
61 p.records[id] = a
62 return a, nil
63 }
64 }
65 if !legacy && prior == (servingArtifactIdentity{}) {
66 return servingParkAuthorization{}, hostbox.ErrChanged
67 }
68 a := servingParkAuthorization{TagsFrom: prior.Stamp, Predecessor: prior, Successor: current, OriginalSuccessor: current, Legacy: legacy}
69 a.ID = servingRetirementID(prior, current, legacy)
70 var stat unix.Statx_t
71 if err := unix.Statx(unix.AT_FDCWD, filepath.Dir(p.db), unix.AT_SYMLINK_NOFOLLOW, unix.STATX_BASIC_STATS|unix.STATX_MNT_ID, &stat); err != nil {
72 return a, err
73 }
74 a.Directory = servingArtifactDirectory{Device: unix.Mkdev(stat.Dev_major, stat.Dev_minor), Inode: stat.Ino, Mount: stat.Mnt_id}
75 dest := filepath.Join(p.f.guard.DataDir, "test-parks", a.ID)
76 if err := os.MkdirAll(dest, 0700); err != nil {
77 return a, err
78 }
79 for _, suffix := range []string{"", ".gen", "-wal", "-shm"} {
80 if err := os.Rename(p.db+suffix, filepath.Join(dest, filepath.Base(p.db)+suffix)); err != nil && !errors.Is(err, os.ErrNotExist) {
81 return a, err
82 }
83 }
84 p.records[a.ID] = a
85 if p.afterPark != nil {
86 p.afterPark()
87 }
88 return a, nil
89}
90func (p *servingParkFixture) Park(_ context.Context, current, prior servingArtifactIdentity) (servingParkAuthorization, error) {
91 return p.park(current, prior, false)
92}
93func (p *servingParkFixture) ParkLegacy(_ context.Context, current servingArtifactIdentity, _ servingLegacyPredecessor) (servingParkAuthorization, error) {
94 return p.park(current, servingArtifactIdentity{}, true)
95}
96func (p *servingParkFixture) Complete(ctx context.Context, id string, current servingArtifactIdentity) error {
97 if p.inside {
98 p.t.Fatal("completion under Work")
99 }
100 p.calls = append(p.calls, "complete")
101 a, ok := p.records[id]
102 if !ok || a.Successor != current {
103 return hostbox.ErrChanged
104 }
105 o, err := hostclaim.Read(ctx, p.f.claims, current.Host)
106 if err != nil {
107 return err
108 }
109 db, _ := filepath.Rel(p.f.guard.DataDir, p.db)
110 if o.Claim.Inventory[current.Box].Paths[db] != current.Stamp || o.Claim.Inventory[current.Box].Paths[db+".gen"] != current.Stamp {
111 p.t.Fatal("root completion before paired enrollment")
112 }
113 if p.failComplete {
114 return hostbox.ErrChanged
115 }
116 return nil
117}
118
119type servingRetirementWork struct {
120 p *servingParkFixture
121 r *claimServingArtifactRetirement
122 admitted hostclaim.Observed
123}
124
125func (w servingRetirementWork) With(ctx context.Context, _ hostwrite.Identity, fn func() error) error {
126 if w.p.inside {
127 w.p.t.Fatal("recursive Work")
128 }
129 h := w.admitted.Claim.Holder
130 return w.r.Guard.With(ctx, w.admitted.Host, hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp}, func(c *hostclaim.Claim) error {
131 if !claimSameWriter(w.admitted, hostclaim.Observed{Host: w.admitted.Host, Claim: c}, nil) {
132 return hostbox.ErrChanged
133 }
134 return nil
135 }, func() error { w.p.inside = true; defer func() { w.p.inside = false }(); return fn() })
136}
137
138func servingRetirementFixture(t *testing.T, hosts ...string) (*recoveryFixture, *claimServingArtifactRetirement, *servingParkFixture, ClaimLaunchInputs) {
139 t.Helper()
140 f := newRecoveryFixture(t, hosts...)
141 db := provision.NestedDBPath(f.guard.DataDir, f.hostName())
142 if err := os.MkdirAll(filepath.Dir(db), 0700); err != nil {
143 t.Fatal(err)
144 }
145 p := &servingParkFixture{t: t, f: f, db: db, records: make(map[string]servingParkAuthorization)}
146 r := &claimServingArtifactRetirement{Claims: f.claims, Guard: f.guard, DataDir: f.guard.DataDir, Box: "box-1", Region: "ie", Root: p}
147 return f, r, p, ClaimLaunchInputs{DBPath: db}
148}
149func servingAdmit(t *testing.T, f *recoveryFixture, in *ClaimLaunchInputs, generation string) hostclaim.Observed {
150 t.Helper()
151 o := f.read()
152 in.Admission = hostclaim.Admission{ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: "box-1", Region: "ie", PredecessorRegion: "ie", Mode: "cold", NewGeneration: generation}
153 admitted, err := hostclaim.Update(t.Context(), f.claims, o, hostclaim.AdmitWriter(in.Admission))
154 if err != nil {
155 t.Fatal(err)
156 }
157 if err := f.guard.Activate(t.Context(), recoveryHost, "box-1", nil); err != nil {
158 t.Fatal(err)
159 }
160 return admitted
161}
162func servingStop(t *testing.T, f *recoveryFixture, o hostclaim.Observed, boots ...string) hostguard.FenceReceipt {
163 t.Helper()
164 return servingStopFor(t, f, o, hostbox.Actor{Box: "box-1", Region: "ie", Enrollment: "enrollment", Revision: 7, Binding: "7"}, boots...)
165}
166
167func servingStopFor(t *testing.T, f *recoveryFixture, o hostclaim.Observed, actor hostbox.Actor, boots ...string) hostguard.FenceReceipt {
168 t.Helper()
169 boot := "boot"
170 if len(boots) != 0 {
171 boot = boots[0]
172 }
173 target := recoveryTuple(o, claimStopLocal{Actor: actor, BootID: boot, ControllerID: "controller"})
174 target.FenceID, target.ClaimGen = "stop-"+o.Claim.Holder.Generation, o.Claim.Gen
175 if _, err := f.guard.BeginFence(t.Context(), target); err != nil {
176 t.Fatal(err)
177 }
178 s := StopReceipt{V: 1, ID: target.FenceID, Kind: StopReceiptStopped, Region: target.Region, ClaimETag: target.ClaimETag, ClaimGen: target.ClaimGen, FenceID: target.FenceID, Host: target.Host, Incarnation: target.Incarnation, InstallationStamp: target.Stamp, WriterSession: target.Session, Box: target.Box, Enrollment: target.Enrollment, BootID: target.BootID, ControllerID: target.ControllerID, DirectoryRevision: testReceiptNumeral(target), Binding: target.DirectoryRevision, EvidenceAt: f.now}
179 body := mustMarshalStopReceipt(s)
180 receipt := hostguard.FenceReceipt{Target: target, Bytes: body, EvidenceAt: f.now}
181 if err := f.guard.CompleteFence(t.Context(), target, body, f.now, func(ctx context.Context, _ hostguard.FenceTarget, _ []byte, _ time.Time) error {
182 return verifyRetirementReceipt(ctx, receipt)
183 }); err != nil {
184 t.Fatal(err)
185 }
186 return receipt
187}
188
189func TestServingRetirementEnrollmentAndProductionRestore(t *testing.T) {
190 f, r, p, in := servingRetirementFixture(t)
191 plan := plan12RestoreBaseline(t, f)
192 if err := os.WriteFile(in.DBPath, []byte("legacy source"), 0600); err != nil {
193 t.Fatal(err)
194 }
195 files := filepath.Join(filepath.Dir(in.DBPath), "files")
196 if err := os.Mkdir(files, 0700); err != nil {
197 t.Fatal(err)
198 }
199 env := filepath.Join(filepath.Dir(in.DBPath), "env")
200 if err := os.WriteFile(env, []byte("KEEP=1"), 0600); err != nil {
201 t.Fatal(err)
202 }
203 for _, generation := range []string{"G", "G2"} {
204 admitted := servingAdmit(t, f, &in, generation)
205 work := servingRetirementWork{p: p, r: r, admitted: admitted}
206 restore := hostbox.ColdRestore{Reader: f.data, Bucket: f.bucket, Generation: generation, Work: work,
207 BeforeLegacyInstall: func(ctx context.Context, _, _ string) error { return r.BeforeLegacyInstall(ctx, admitted, in, work) },
208 BeforeInstallLocked: func(ctx context.Context, _, _ string) error { return r.EnrollLocked(ctx, admitted, in) },
209 RetireArtifacts: func(ctx context.Context, generation string) error { return r.Retire(ctx, admitted, in, generation) },
210 }
211 if err := r.Recover(t.Context(), admitted, in); err != nil {
212 t.Fatal(err)
213 }
214 if err := restore.Install(t.Context(), in.DBPath, plan); err != nil {
215 t.Fatal(err)
216 }
217 if got := mwReadDB(t, in.DBPath); got != "selected recovery baseline" {
218 t.Fatal(got)
219 }
220 marker, err := hostbox.ReadGenerationMarker(in.DBPath + ".gen")
221 if err != nil || marker != generation {
222 t.Fatalf("marker %q: %v", marker, err)
223 }
224 servingStop(t, f, f.read())
225 }
226 if body, err := os.ReadFile(env); err != nil || string(body) != "KEEP=1" {
227 t.Fatalf("env: %s %v", body, err)
228 }
229 if _, err := os.Stat(files); err != nil {
230 t.Fatal(err)
231 }
232 if len(p.records) != 2 {
233 t.Fatalf("park records: %v", p.records)
234 }
235}
236
237func TestServingRetirementRefusesUnprovedEnrollment(t *testing.T) {
238 f, r, p, in := servingRetirementFixture(t)
239 admitted := servingAdmit(t, f, &in, "G")
240 if err := r.EnrollLocked(t.Context(), admitted, in); !errors.Is(err, hostbox.ErrChanged) {
241 t.Fatal(err)
242 }
243 if len(p.calls) != 0 {
244 t.Fatal(p.calls)
245 }
246 for _, a := range []string{"", ".gen", "-wal", "-shm"} {
247 t.Run("replacement"+a, func(t *testing.T) {
248 if err := r.BeforeLegacyInstall(t.Context(), admitted, in, servingRetirementWork{p, r, admitted}); err != nil {
249 t.Fatal(err)
250 }
251 if err := os.WriteFile(in.DBPath+a, []byte("unexpected"), 0600); err != nil {
252 t.Fatal(err)
253 }
254 if err := r.EnrollLocked(t.Context(), admitted, in); !errors.Is(err, hostbox.ErrChanged) {
255 t.Fatal(err)
256 }
257 if err := os.Remove(in.DBPath + a); err != nil {
258 t.Fatal(err)
259 }
260 })
261 }
262}
263
264func TestServingRetirementCrashBetweenTagsAndCompletion(t *testing.T) {
265 f, r, p, in := servingRetirementFixture(t)
266 admitted := servingAdmit(t, f, &in, "G")
267 p.failComplete = true
268 if err := r.BeforeLegacyInstall(t.Context(), admitted, in, servingRetirementWork{p, r, admitted}); !errors.Is(err, hostbox.ErrChanged) {
269 t.Fatal(err)
270 }
271 if err := r.EnrollLocked(t.Context(), admitted, in); !errors.Is(err, hostbox.ErrChanged) {
272 t.Fatal(err)
273 }
274 p.failComplete = false
275 restarted := &claimServingArtifactRetirement{Claims: r.Claims, Guard: r.Guard, DataDir: r.DataDir, Box: r.Box, Region: r.Region, Root: p}
276 before := f.read().ETag
277 if err := restarted.Recover(t.Context(), admitted, in); err != nil {
278 t.Fatal(err)
279 }
280 if f.read().ETag != before {
281 t.Fatal("recovery rewrote current tags")
282 }
283 if err := restarted.EnrollLocked(t.Context(), admitted, in); err != nil {
284 t.Fatal(err)
285 }
286 if len(p.records) != 1 {
287 t.Fatal("recovery invented another retirement")
288 }
289}
290
291func TestServingRetirementAuthenticatesEveryReceiptField(t *testing.T) {
292 t.Parallel()
293 fields := []string{"Region", "ClaimETag", "ClaimGen", "ObligationID", "FenceID", "Host", "Incarnation", "InstallationStamp", "WriterSession", "OperationID", "Phase", "Box", "Enrollment", "BootID", "ControllerID", "DirectoryRevision", "EvidenceAt"}
294 for _, field := range fields {
295 t.Run(field, func(t *testing.T) {
296 f, r, p, in := servingRetirementFixture(t)
297 old := servingAdmit(t, f, &in, "G")
298 if err := r.BeforeLegacyInstall(t.Context(), old, in, servingRetirementWork{p, r, old}); err != nil {
299 t.Fatal(err)
300 }
301 receipt := servingStop(t, f, f.read())
302 admitted := servingAdmit(t, f, &in, "G2")
303 var body StopReceipt
304 if err := json.Unmarshal(receipt.Bytes, &body); err != nil {
305 t.Fatal(err)
306 }
307 v := reflect.ValueOf(&body).Elem().FieldByName(field)
308 switch v.Kind() {
309 case reflect.String:
310 v.SetString(v.String() + "bad")
311 case reflect.Uint64:
312 v.SetUint(v.Uint() + 1)
313 default:
314 v.Set(reflect.ValueOf(f.now.Add(time.Second)))
315 }
316 receipt.Bytes = mustMarshalStopReceipt(body)
317 r.Guard.FenceView = func(_ context.Context, host string, c hostguard.Captured) (hostguard.FenceView, error) {
318 return hostguard.FenceView{Schema: 1, Host: host, Captured: c, Installation: hostguard.RootInstallation{Host: host, Migration: 1}, Receipts: []hostguard.FenceReceipt{receipt}}, nil
319 }
320 p.calls = nil
321 before := f.read().ETag
322 if err := r.Retire(t.Context(), admitted, in, "G"); !errors.Is(err, ErrClaimStopReceipt) {
323 t.Fatalf("forged %s: %v", field, err)
324 }
325 if len(p.calls) != 0 || f.read().ETag != before {
326 t.Fatal("forged receipt caused effects")
327 }
328 })
329 }
330}
331
332func TestServingRetirementRefusesWrongTagsAndWriter(t *testing.T) {
333 t.Parallel()
334 for _, name := range []string{"missing-one", "zero", "mixed", "future", "foreign-box", "foreign-generation", "stale-location", "flat", "ancestor", "outside-path", "noncold", "duplicate-admission", "missing-receipt", "wrong-session"} {
335 t.Run(name, func(t *testing.T) {
336 f, r, p, in := servingRetirementFixture(t)
337 old := servingAdmit(t, f, &in, "G")
338 if err := r.BeforeLegacyInstall(t.Context(), old, in, servingRetirementWork{p, r, old}); err != nil {
339 t.Fatal(err)
340 }
341 if name != "missing-receipt" {
342 servingStop(t, f, f.read())
343 }
344 admitted := servingAdmit(t, f, &in, "G2")
345 db, _ := filepath.Rel(r.DataDir, in.DBPath)
346 if name == "outside-path" {
347 in.DBPath = filepath.Join(r.DataDir, "foreign.db")
348 } else if name == "stale-location" {
349 admitted.Claim.Holder.Location += "/stale"
350 } else {
351 o := f.read()
352 _, err := hostclaim.Update(t.Context(), f.claims, o, func(c *hostclaim.Claim) error {
353 e := c.Inventory[r.Box]
354 switch name {
355 case "missing-one":
356 delete(e.Paths, db+".gen")
357 case "zero":
358 e.Paths[db], e.Paths[db+".gen"] = 0, 0
359 case "mixed":
360 e.Paths[db]++
361 case "future":
362 e.Paths[db], e.Paths[db+".gen"] = c.Gen+50, c.Gen+50
363 case "foreign-box":
364 c.Holder.Admissions[0].Box = "foreign"
365 case "foreign-generation":
366 c.Holder.Admissions[0].Generation = "foreign"
367 case "flat":
368 e.Layout = "flat"
369 case "ancestor":
370 e.Paths[filepath.Dir(db)] = old.Claim.Holder.Stamp
371 case "noncold":
372 c.Holder.Admissions[0].Mode = "warm"
373 case "duplicate-admission":
374 c.Holder.Admissions = append(c.Holder.Admissions, c.Holder.Admissions[0])
375 case "wrong-session":
376 c.Holder.Admissions[0].Session++
377 }
378 c.Inventory[r.Box] = e
379 return nil
380 })
381 // Some invalid claims are refused by the writer itself. That also
382 // prevents retirement; never persist malformed bytes to bypass it.
383 if err != nil {
384 return
385 }
386 }
387 before := f.read().ETag
388 p.calls = nil
389 if err := r.Retire(t.Context(), admitted, in, "G"); err == nil {
390 t.Fatal("invalid authority accepted")
391 }
392 if len(p.calls) != 0 {
393 t.Fatal(p.calls)
394 }
395 if f.read().ETag != before {
396 t.Fatal("invalid authority retagged inventory")
397 }
398 })
399 }
400}
401
402type servingCASFailure struct {
403 release.Store
404 afterCommit bool
405 calls int
406}
407
408func (s *servingCASFailure) Put(ctx context.Context, key string, b []byte, etag string) (string, error) {
409 s.calls++
410 if s.afterCommit {
411 if _, err := s.Store.Put(ctx, key, b, etag); err != nil {
412 return "", err
413 }
414 }
415 return "", fmt.Errorf("injected lost CAS reply")
416}
417func TestServingEnrollmentResolvesOnlyExactCommittedCAS(t *testing.T) {
418 for _, committed := range []bool{false, true} {
419 t.Run(fmt.Sprint(committed), func(t *testing.T) {
420 f, r, p, in := servingRetirementFixture(t)
421 admitted := servingAdmit(t, f, &in, "G")
422 r.Claims = &servingCASFailure{Store: f.claims, afterCommit: committed}
423 err := r.BeforeLegacyInstall(t.Context(), admitted, in, servingRetirementWork{p, r, admitted})
424 if (err == nil) != committed {
425 t.Fatalf("committed %v: %v", committed, err)
426 }
427 if !committed {
428 if err := r.EnrollLocked(t.Context(), admitted, in); err == nil {
429 t.Fatal("uncommitted enrollment admitted")
430 }
431 r.Claims = f.claims
432 if err := r.BeforeLegacyInstall(t.Context(), admitted, in, servingRetirementWork{p, r, admitted}); err != nil {
433 t.Fatal(err)
434 }
435 }
436 if len(p.records) != 1 {
437 t.Fatal("CAS retry reparks legacy source")
438 }
439 })
440 }
441}
442
443func TestServingRetirementPreservesOperationDispatch(t *testing.T) {
444 _, r, _, _ := servingRetirementFixture(t)
445 var calls []string
446 w := &ClaimWakeAssembly{
447 RetireArtifacts: func(_ context.Context, o hostclaim.Observed, _ ClaimLaunchInputs, _ string) error {
448 calls = append(calls, "retire:"+o.Claim.Op.Kind)
449 return nil
450 },
451 RecoverArtifacts: func(_ context.Context, o hostclaim.Observed, _ ClaimLaunchInputs) error {
452 calls = append(calls, "recover:"+o.Claim.Op.Kind)
453 return nil
454 },
455 }
456 if err := bindServingArtifactRetirement(w, r); err != nil {
457 t.Fatal(err)
458 }
459 for _, kind := range []string{hostclaim.KindLanding, hostclaim.KindRestoring, hostclaim.KindReverting, hostclaim.KindRelocating} {
460 o := hostclaim.Observed{Claim: &hostclaim.Claim{Op: hostclaim.Op{Kind: kind}}}
461 if err := w.RetireArtifacts(t.Context(), o, ClaimLaunchInputs{}, "G"); err != nil {
462 t.Fatal(err)
463 }
464 if err := w.RecoverArtifacts(t.Context(), o, ClaimLaunchInputs{}); err != nil {
465 t.Fatal(err)
466 }
467 }
468 if len(calls) != 8 {
469 t.Fatal(calls)
470 }
471 o := hostclaim.Observed{Claim: &hostclaim.Claim{Op: hostclaim.Op{Kind: hostclaim.KindServing}}}
472 if err := w.RetireArtifacts(t.Context(), o, ClaimLaunchInputs{}, "G"); err == nil {
473 t.Fatal("invalid serving authority accepted")
474 }
475 if len(calls) != 8 {
476 t.Fatal("serving fell through to operations")
477 }
478 if !w.RequireServingEnrollment || w.BeforeInstallLocked == nil || w.BeforeLegacyInstall == nil {
479 t.Fatal("missing serving hooks")
480 }
481}
482
483func TestServingArtifactDirectoryRefusesReplacement(t *testing.T) {
484 for _, kind := range []string{"missing", "replacement", "symlink", "mount", "device", "inode", "leaf-symlink"} {
485 t.Run(kind, func(t *testing.T) {
486 f, r, p, in := servingRetirementFixture(t)
487 admitted := servingAdmit(t, f, &in, "G")
488 if err := r.BeforeLegacyInstall(t.Context(), admitted, in, servingRetirementWork{p, r, admitted}); err != nil {
489 t.Fatal(err)
490 }
491 a := r.completed[admitted.Host]
492 switch kind {
493 case "missing", "replacement", "symlink":
494 dir := filepath.Dir(in.DBPath)
495 if err := os.Rename(dir, dir+".original"); err != nil {
496 t.Fatal(err)
497 }
498 if kind == "replacement" {
499 if err := os.Mkdir(dir, 0700); err != nil {
500 t.Fatal(err)
501 }
502 }
503 if kind == "symlink" {
504 if err := os.Symlink(dir+".original", dir); err != nil {
505 t.Fatal(err)
506 }
507 }
508 case "mount":
509 a.Directory.Mount++
510 case "device":
511 a.Directory.Device++
512 case "inode":
513 a.Directory.Inode++
514 case "leaf-symlink":
515 if err := os.Symlink(filepath.Join(t.TempDir(), "absent"), in.DBPath); err != nil {
516 t.Fatal(err)
517 }
518 }
519 if err := checkServingArtifactAbsence(in.DBPath, a.Directory); err == nil {
520 t.Fatal("unsafe directory accepted")
521 }
522 })
523 }
524}
525
526func TestServingInventoryPairValidationAndPreservation(t *testing.T) {
527 f, _, _, in := servingRetirementFixture(t)
528 o := servingAdmit(t, f, &in, "G")
529 current, err := servingWriterIdentity(o, "box-1", "ie")
530 if err != nil {
531 t.Fatal(err)
532 }
533 db, _ := filepath.Rel(f.guard.DataDir, in.DBPath)
534 for _, name := range []string{"absent", "current", "older", "missing-one", "mixed", "zero", "future", "ancestor", "descendant", "unrelated-overlap", "unnormalized", "flat"} {
535 t.Run(name, func(t *testing.T) {
536 c := *o.Claim
537 paths := map[string]uint64{"unrelated": 1}
538 entry := hostclaim.InventoryEntry{Layout: "nested", Paths: paths}
539 expected := false
540 prior := uint64(0)
541 switch name {
542 case "absent":
543 expected = true
544 case "current":
545 paths[db], paths[db+".gen"] = current.Stamp, current.Stamp
546 expected = true
547 case "older":
548 paths[db], paths[db+".gen"] = 1, 1
549 prior = 1
550 expected = true
551 case "missing-one":
552 paths[db] = 1
553 case "mixed":
554 paths[db], paths[db+".gen"] = 1, 2
555 case "zero":
556 paths[db], paths[db+".gen"] = 0, 0
557 case "future":
558 paths[db], paths[db+".gen"] = current.Stamp+1, current.Stamp+1
559 case "ancestor":
560 paths[filepath.Dir(db)] = 1
561 case "descendant":
562 paths[db+"/child"] = 1
563 case "unrelated-overlap":
564 paths["unrelated/child"] = 1
565 case "unnormalized":
566 paths["a/../b"] = 1
567 case "flat":
568 entry.Layout = "flat"
569 }
570 c.Inventory = map[string]hostclaim.InventoryEntry{"box-1": entry, "other": {Layout: "flat", Paths: map[string]uint64{"other-db": 99}}}
571 before, _ := json.Marshal(c)
572 err := servingInventoryTransition(current, db, prior)(&c)
573 if (err == nil) != expected {
574 t.Fatalf("accepted=%v: %v", expected, err)
575 }
576 if !expected {
577 after, _ := json.Marshal(c)
578 if string(after) != string(before) {
579 t.Fatal("refused transition mutated claim")
580 }
581 return
582 }
583 if c.Inventory["box-1"].Paths[db] != current.Stamp || c.Inventory["box-1"].Paths[db+".gen"] != current.Stamp || c.Inventory["box-1"].Paths["unrelated"] != 1 || c.Inventory["other"].Paths["other-db"] != 99 {
584 t.Fatal(c.Inventory)
585 }
586 })
587 }
588}
589
590func TestServingRetirementParkThenClaimRaceRefusesRetag(t *testing.T) {
591 f, r, p, in := servingRetirementFixture(t)
592 admitted := servingAdmit(t, f, &in, "G")
593 p.afterPark = func() {
594 p.afterPark = nil
595 newer := in
596 servingAdmit(t, f, &newer, "racer")
597 }
598 if err := r.BeforeLegacyInstall(t.Context(), admitted, in, servingRetirementWork{p, r, admitted}); err == nil {
599 t.Fatal("stale writer enrolled")
600 }
601 if len(f.read().Claim.Inventory) != 0 {
602 t.Fatal("stale writer tagged new installation")
603 }
604 if err := r.EnrollLocked(t.Context(), admitted, in); err == nil {
605 t.Fatal("stale writer permitted restore")
606 }
607}
608
609type servingBaselineInputs struct{ recoveryInputs }
610
611func (s servingBaselineInputs) Select(ctx context.Context, rt registry.Route, o hostclaim.Observed, policy activator.StartPolicy) (ClaimLaunchInputs, error) {
612 in, err := s.recoveryInputs.Select(ctx, rt, o, policy)
613 if err != nil {
614 return in, err
615 }
616 in.DBPath = provision.NestedDBPath(s.f.guard.DataDir, o.Host)
617 in.Empty = false
618 in.Baseline, err = genselect.Select(ctx, s.f.data, genselect.Request{Bucket: s.f.bucket, Region: "ie", Host: o.Host, Holder: *o.Claim.Holder})
619 if err == nil {
620 in.Admission.RemoteMaxTXID = in.Baseline.MaxTXID
621 }
622 return in, err
623}
624
625func TestServingRetirementOrdinaryPreparationThenSuccessor(t *testing.T) {
626 f, r, p, in := servingRetirementFixture(t)
627 bucket := leasetest.NewVersionedBucket()
628 bucket.AddBucket("data")
629 key := f.read().Claim.Holder.Location + "/0009/" + ltx.FormatFilename(1, 1)
630 payload := mwSnapshot(t, 1, "ordinary serving baseline")
631 version := bucket.Put("data", key, payload)
632 f.data, f.bucket = bucket, "data"
633 if err := os.WriteFile(in.DBPath, []byte("unmarked legacy"), 0600); err != nil {
634 t.Fatal(err)
635 }
636 controller := f.controller("boot-1", "controller-1", 4101, "start-1")
637 controller.wake.Inputs = servingBaselineInputs{recoveryInputs{f}}
638 if err := bindServingArtifactRetirement(controller.wake, r); err != nil {
639 t.Fatal(err)
640 }
641 var generations []string
642 for i := 0; i < 2; i++ {
643 admitted := controller.admit(t, f)
644 generations = append(generations, admitted.Claim.Holder.Generation)
645 start, err := controller.runtime.Prepare(t.Context(), registry.Route{Host: recoveryHost}, admitted, recoveryPolicy)
646 if err != nil {
647 t.Fatal(err)
648 }
649 if err = start.Restore(t.Context()); err != nil {
650 t.Fatal(err)
651 }
652 if got := mwReadDB(t, in.DBPath); got != "ordinary serving baseline" {
653 t.Fatal(got)
654 }
655 // Stop before Materialize: an unseeded ordinary C8 correctly refuses.
656 // FailedAdmission drives the real serving stop and retry settlement.
657 if err = controller.runtime.FailedAdmission(t.Context(), admitted, hostbox.ErrChanged); err != nil {
658 t.Fatal(err)
659 }
660 f.now = f.now.Add(time.Minute)
661 }
662 if generations[0] == generations[1] {
663 t.Fatal("successor reused generation")
664 }
665 if len(f.receipts()) != 2 {
666 t.Fatal("missing exact serving stops")
667 }
668 if f.retry().AttemptsReserved != 2 {
669 t.Fatal("retirement changed retry accounting")
670 }
671 remote, err := bucket.GetVersion(t.Context(), "data", key, version)
672 if err != nil || !bytes.Equal(remote, payload) {
673 t.Fatal("baseline changed")
674 }
675 if len(p.records) != 2 {
676 t.Fatal("retirement did not park both sources")
677 }
678}
679
680func TestServingRetirementContinuationKeepsOriginalIdentity(t *testing.T) {
681 f, r, p, in := servingRetirementFixture(t)
682 original := servingAdmit(t, f, &in, "G")
683 p.failComplete = true
684 if err := r.BeforeLegacyInstall(t.Context(), original, in, servingRetirementWork{p, r, original}); err == nil {
685 t.Fatal("incomplete park accepted")
686 }
687 var old servingParkAuthorization
688 for _, a := range p.records {
689 old = a
690 }
691 servingStop(t, f, f.read())
692 successor := servingAdmit(t, f, &in, "G2")
693 p.failComplete = false
694 p.continuePending = true
695 if err := r.Recover(t.Context(), successor, in); err != nil {
696 t.Fatal(err)
697 }
698 a := r.completed[successor.Host]
699 if a.ID != old.ID || a.OriginalSuccessor != old.OriginalSuccessor || a.Successor.Generation != "G2" || len(p.records) != 1 {
700 t.Fatal("continuation relabeled immutable retirement")
701 }
702 if err := r.EnrollLocked(t.Context(), successor, in); err != nil {
703 t.Fatal(err)
704 }
705 if err := r.EnrollLocked(t.Context(), original, in); err == nil {
706 t.Fatal("old admission reused continuation")
707 }
708}
709
710func TestServingRetirementAuthorizationBinding(t *testing.T) {
711 f, r, p, in := servingRetirementFixture(t)
712 admitted := servingAdmit(t, f, &in, "G")
713 if err := r.BeforeLegacyInstall(t.Context(), admitted, in, servingRetirementWork{p, r, admitted}); err != nil {
714 t.Fatal(err)
715 }
716 a := r.completed[admitted.Host]
717 fresh := f.read()
718 current, err := servingWriterIdentity(fresh, r.Box, r.Region)
719 if err != nil {
720 t.Fatal(err)
721 }
722 for _, field := range []string{"Host", "Box", "Region", "Incarnation", "Location", "Generation", "Stamp", "Session"} {
723 for _, target := range []string{"Successor", "OriginalSuccessor"} {
724 t.Run(target+"/"+field, func(t *testing.T) {
725 changed := a
726 v := reflect.ValueOf(&changed).Elem().FieldByName(target).FieldByName(field)
727 if v.Kind() == reflect.String {
728 v.SetString(v.String() + "wrong")
729 } else {
730 v.SetUint(v.Uint() + 1)
731 }
732 // Even a recomputed journal ID cannot bind a foreign original tuple.
733 changed.ID = servingRetirementID(changed.Predecessor, changed.OriginalSuccessor, changed.Legacy)
734 if err := r.validateAuthorization(t.Context(), fresh, current, changed); err == nil {
735 t.Fatal("foreign authorization accepted")
736 }
737 })
738 }
739 }
740 for _, name := range []string{"id", "mount", "inode", "device", "future-tags"} {
741 t.Run(name, func(t *testing.T) {
742 changed := a
743 switch name {
744 case "id":
745 changed.ID = "wrong"
746 case "mount":
747 changed.Directory.Mount = 0
748 case "inode":
749 changed.Directory.Inode = 0
750 case "device":
751 changed.Directory.Device = 0
752 case "future-tags":
753 changed.TagsFrom = current.Stamp + 1
754 }
755 if err := r.validateAuthorization(t.Context(), fresh, current, changed); err == nil {
756 t.Fatal("invalid authorization accepted")
757 }
758 })
759 }
760}
761
762func TestServingRetirementFailedReplayInvalidatesPermission(t *testing.T) {
763 f, r, p, in := servingRetirementFixture(t)
764 admitted := servingAdmit(t, f, &in, "G")
765 if err := r.BeforeLegacyInstall(t.Context(), admitted, in, servingRetirementWork{p, r, admitted}); err != nil {
766 t.Fatal(err)
767 }
768 if err := r.EnrollLocked(t.Context(), admitted, in); err != nil {
769 t.Fatal(err)
770 }
771 p.failComplete = true
772 if err := r.Recover(t.Context(), admitted, in); err == nil {
773 t.Fatal("refused completion accepted")
774 }
775 if err := r.EnrollLocked(t.Context(), admitted, in); err == nil {
776 t.Fatal("old completion survived failed replay")
777 }
778}
779
780// Root and the hardened edge see the same inode through different mounts.
781type servingNamespaceParker struct {
782 servingDatabaseParker
783 change func(*servingParkAuthorization)
784}
785
786func (p servingNamespaceParker) Park(ctx context.Context, current, prior servingArtifactIdentity) (servingParkAuthorization, error) {
787 a, err := p.servingDatabaseParker.Park(ctx, current, prior)
788 p.change(&a)
789 return a, err
790}
791
792func (p servingNamespaceParker) ParkLegacy(ctx context.Context, current servingArtifactIdentity, prior servingLegacyPredecessor) (servingParkAuthorization, error) {
793 a, err := p.servingDatabaseParker.ParkLegacy(ctx, current, prior)
794 p.change(&a)
795 return a, err
796}
797func TestServingEnrollmentMountNamespaces(t *testing.T) {
798 for _, kind := range []string{"namespace", "device", "inode", "replacement"} {
799 t.Run(kind, func(t *testing.T) {
800 f, r, p, in := servingRetirementFixture(t)
801 admitted := servingAdmit(t, f, &in, "G")
802 r.Root = servingNamespaceParker{p, func(a *servingParkAuthorization) {
803 a.Directory.Mount++
804 switch kind {
805 case "device":
806 a.Directory.Device++
807 case "inode":
808 a.Directory.Inode++
809 case "replacement":
810 dir := filepath.Dir(in.DBPath)
811 if err := os.Rename(dir, dir+".old"); err != nil {
812 t.Fatal(err)
813 }
814 if err := os.Mkdir(dir, 0700); err != nil {
815 t.Fatal(err)
816 }
817 }
818 }}
819 err := r.BeforeLegacyInstall(t.Context(), admitted, in, servingRetirementWork{p, r, admitted})
820 if kind == "namespace" {
821 if err != nil {
822 t.Fatal(err)
823 }
824 if err := r.EnrollLocked(t.Context(), admitted, in); err != nil {
825 t.Fatal(err)
826 }
827 if err := r.Recover(t.Context(), admitted, in); err != nil {
828 t.Fatal("namespace recovery", err)
829 }
830 } else if err == nil {
831 t.Fatal("changed source accepted")
832 }
833 })
834 }
835}
836