amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1//go:build linux
2
3package host
4
5import (
6 "context"
7 "errors"
8 "os"
9 "path/filepath"
10 "reflect"
11 "testing"
12
13 "github.com/carlosframework/platform/internal/hostbox"
14 "github.com/carlosframework/platform/internal/provision"
15 "github.com/carlosframework/platform/internal/rootstate"
16 "golang.org/x/sys/unix"
17)
18
19func quarantineFixture(t *testing.T) (*servingParkDiskFixture, servingArtifactDirectory, string) {
20 t.Helper()
21 f := newServingParkDiskFixture(t)
22 parent, id, err := f.fs.parent()
23 if err != nil {
24 t.Fatal(err)
25 }
26 parent.Close()
27 path := filepath.Join(f.data, "instances", ".serving-parked")
28 if err := os.Mkdir(path, 0700); err != nil {
29 t.Fatal(err)
30 }
31 if err := os.WriteFile(filepath.Join(path, "unknown"), []byte("keep these bytes"), 0600); err != nil {
32 t.Fatal(err)
33 }
34 return f, id, path
35}
36
37func quarantineRecover(t *testing.T, f *servingParkDiskFixture, expected servingArtifactDirectory) (servingParkQuarantine, error) {
38 t.Helper()
39 var q servingParkQuarantine
40 err := f.ledger.withNamespace(t.Context(), func() error {
41 var err error
42 q, err = f.fs.recoverNamespace(t.Context(), expected)
43 return err
44 })
45 return q, err
46}
47
48func quarantinePreserved(t *testing.T, f *servingParkDiskFixture, q servingParkQuarantine) {
49 t.Helper()
50 path := filepath.Join(f.data, "instances", q.name())
51 body, err := os.ReadFile(filepath.Join(path, "unknown"))
52 if err != nil || string(body) != "keep these bytes" {
53 t.Fatal("quarantine lost data", err)
54 }
55 d, err := os.Open(path)
56 if err != nil {
57 t.Fatal(err)
58 }
59 id, err := servingParkDirID(d)
60 d.Close()
61 if err != nil || id != q.Obstruction {
62 t.Fatal("quarantine changed identity", err)
63 }
64}
65
66func TestServingParkQuarantineUnknownRoot(t *testing.T) {
67 f, parent, path := quarantineFixture(t)
68 // Contents include a symlink to a victim and an unreadable subtree. The
69 // recovery only renames the outer directory and never inspects either.
70 victim := filepath.Join(t.TempDir(), "victim")
71 if err := os.WriteFile(victim, []byte("victim"), 0600); err != nil {
72 t.Fatal(err)
73 }
74 if err := os.Symlink(victim, filepath.Join(path, "link")); err != nil {
75 t.Fatal(err)
76 }
77 if err := os.Mkdir(filepath.Join(path, "unreadable"), 0000); err != nil {
78 t.Fatal(err)
79 }
80 q, err := quarantineRecover(t, f, parent)
81 if err != nil || !q.Complete || q.Previous != nil {
82 t.Fatalf("quarantine: %+v %v", q, err)
83 }
84 quarantinePreserved(t, f, q)
85 if body, err := os.ReadFile(victim); err != nil || string(body) != "victim" {
86 t.Fatal("quarantine followed victim", err)
87 }
88 before, err := f.disk.Read("namespace-quarantine.json")
89 if err != nil {
90 t.Fatal(err)
91 }
92 f.fs = newServingParkFS(f.data, f.disk)
93 f.fs.owner, f.fs.mount = uint32(os.Geteuid()), func(string) (bool, error) { return true, nil }
94 f.ledger.fs = f.fs
95 for i := 0; i < 3; i++ {
96 again, err := quarantineRecover(t, f, parent)
97 if err != nil || !reflect.DeepEqual(again, q) {
98 t.Fatal("quarantine replay", err)
99 }
100 }
101 after, err := f.disk.Read("namespace-quarantine.json")
102 if err != nil || string(after) != string(before) {
103 t.Fatal("replay changed quarantine record", err)
104 }
105 // A fresh host can use the empty namespace. Unknown bytes do not become
106 // retirement records and survive an ordinary expiry pass.
107 f.put("", "current database")
108 a, stops := f.evidence(10, true, "missing")
109 r, err := f.park(a, stops)
110 if err != nil {
111 t.Fatal("park after quarantine", err)
112 }
113 f.complete(a)
114 f.now = r.ReapAfter
115 if err := f.ledger.withNamespace(t.Context(), func() error { return f.ledger.reapLocked(t.Context(), a.ID) }); err != nil {
116 t.Fatal(err)
117 }
118 quarantinePreserved(t, f, q)
119 // Only one quarantine is allowed. A later replacement is retained in
120 // place for investigation, never adopted or moved to another quarantine.
121 if err := os.Rename(path, path+"-saved"); err != nil {
122 t.Fatal(err)
123 }
124 if err := os.Mkdir(path, 0700); err != nil {
125 t.Fatal(err)
126 }
127 if _, err := quarantineRecover(t, f, parent); err == nil {
128 t.Fatal("replaced root accepted")
129 }
130 names, err := os.ReadDir(filepath.Dir(path))
131 if err != nil || len(names) != 4 { // instance, fresh obstruction, saved root, quarantine
132 t.Fatalf("unexpected recovery directories: %v %v", names, err)
133 }
134}
135
136type quarantineWriteFault struct {
137 rootstate.Store
138 before, after func(string, []byte) error
139}
140
141func (s quarantineWriteFault) Write(name string, body []byte) error {
142 if s.before != nil {
143 if err := s.before(name, body); err != nil {
144 return err
145 }
146 }
147 if err := s.Store.Write(name, body); err != nil {
148 return err
149 }
150 if s.after != nil {
151 return s.after(name, body)
152 }
153 return nil
154}
155
156func TestServingParkQuarantineResumesDurableBoundaries(t *testing.T) {
157 for _, boundary := range []string{"intent", "rename", "sync", "moved", "fresh-sync", "fresh", "namespace", "complete"} {
158 t.Run(boundary, func(t *testing.T) {
159 f, parent, _ := quarantineFixture(t)
160 crash := errors.New("interrupted quarantine")
161 f.fs.state = quarantineWriteFault{Store: f.disk, after: func(name string, body []byte) error {
162 var q servingParkQuarantine
163 if name == "namespace-quarantine.json" {
164 if err := servingParkDecode(body, &q); err != nil {
165 return err
166 }
167 if boundary == "intent" && !q.Moved || boundary == "moved" && q.Moved && q.Fresh.Inode == 0 || boundary == "fresh" && q.Fresh.Inode != 0 && !q.Complete || boundary == "complete" && q.Complete {
168 return crash
169 }
170 }
171 if boundary == "namespace" && name == "namespace.json" {
172 return crash
173 }
174 return nil
175 }}
176 if boundary == "rename" {
177 f.fs.rename = func(a int, b string, c int, d string, flags uint) error {
178 if err := unix.Renameat2(a, b, c, d, flags); err != nil {
179 return err
180 }
181 return crash
182 }
183 }
184 if boundary == "sync" {
185 f.fs.sync = func(int) error { return crash }
186 }
187 if boundary == "fresh-sync" {
188 f.fs.sync = func(fd int) error {
189 if _, err := os.Stat(filepath.Join(f.data, "instances", ".serving-parked")); err == nil {
190 return crash
191 }
192 return unix.Fsync(fd)
193 }
194 f.fs.state = quarantineWriteFault{Store: f.disk, before: func(name string, body []byte) error {
195 var q servingParkQuarantine
196 if name == "namespace-quarantine.json" && servingParkDecode(body, &q) == nil && q.Fresh.Inode != 0 {
197 t.Fatal("fresh identity saved before successful parent sync")
198 }
199 return nil
200 }}
201 }
202 if _, err := quarantineRecover(t, f, parent); !errors.Is(err, crash) {
203 t.Fatal("missing interruption", err)
204 }
205 if boundary != "complete" {
206 if root, _, err := f.fs.namespace(); err == nil {
207 root.Close()
208 t.Fatal("unfinished quarantine allowed ordinary parking")
209 }
210 }
211 f.fs.state, f.fs.rename, f.fs.sync = f.disk, unix.Renameat2, unix.Fsync
212 q, err := quarantineRecover(t, f, parent)
213 if err != nil || !q.Complete {
214 t.Fatalf("recovery: %+v %v", q, err)
215 }
216 quarantinePreserved(t, f, q)
217 })
218 }
219}
220
221func TestServingParkQuarantineRefusesUnrecordedFreshRoot(t *testing.T) {
222 f, parent, _ := quarantineFixture(t)
223 f.fs.state = quarantineWriteFault{Store: f.disk, before: func(name string, body []byte) error {
224 var q servingParkQuarantine
225 if name == "namespace-quarantine.json" && servingParkDecode(body, &q) == nil && q.Fresh.Inode != 0 {
226 return unix.ENOSPC
227 }
228 return nil
229 }}
230 q, err := quarantineRecover(t, f, parent)
231 if !errors.Is(err, unix.ENOSPC) {
232 t.Fatal("expected full-disk fault", err)
233 }
234 // One unknown replacement can be retained. A second lost identity write
235 // exhausts that bounded recovery; neither directory becomes authority.
236 if _, err := quarantineRecover(t, f, parent); !errors.Is(err, unix.ENOSPC) {
237 t.Fatal("expected second full-disk fault", err)
238 }
239 f.fs.state = f.disk
240 if _, err := quarantineRecover(t, f, parent); !errors.Is(err, unix.EEXIST) {
241 t.Fatal("adopted fresh root without durable identity", err)
242 }
243 if _, err := f.disk.Read("namespace.json"); !errors.Is(err, os.ErrNotExist) {
244 t.Fatal("published an unrecorded root", err)
245 }
246 quarantinePreserved(t, f, q)
247}
248
249func quarantineUnrecordedFixture(t *testing.T) (*servingParkDiskFixture, servingArtifactDirectory, servingArtifactDirectory) {
250 t.Helper()
251 f, parent, path := quarantineFixture(t)
252 f.fs.state = quarantineWriteFault{Store: f.disk, before: func(name string, body []byte) error {
253 var q servingParkQuarantine
254 if name == "namespace-quarantine.json" && servingParkDecode(body, &q) == nil && q.Fresh.Inode != 0 {
255 return unix.ENOSPC
256 }
257 return nil
258 }}
259 if _, err := quarantineRecover(t, f, parent); !errors.Is(err, unix.ENOSPC) {
260 t.Fatal("missing creation interruption", err)
261 }
262 f.fs.state = f.disk
263 d, err := os.Open(path)
264 if err != nil {
265 t.Fatal(err)
266 }
267 id, err := servingParkDirID(d)
268 d.Close()
269 if err != nil {
270 t.Fatal(err)
271 }
272 if err := os.WriteFile(filepath.Join(path, "uncertain"), []byte("never adopt or delete"), 0600); err != nil {
273 t.Fatal(err)
274 }
275 if err := os.Symlink("/unavailable/victim", filepath.Join(path, "link")); err != nil {
276 t.Fatal(err)
277 }
278 if err := os.Mkdir(filepath.Join(path, "unreadable"), 0000); err != nil {
279 t.Fatal(err)
280 }
281 return f, parent, id
282}
283
284func TestServingParkQuarantineRetainsUnrecordedReplacement(t *testing.T) {
285 for _, boundary := range []string{"", "intent-before", "intent-after", "rename", "sync", "moved-before", "moved-after", "fresh-after", "namespace", "complete"} {
286 t.Run(boundary, func(t *testing.T) {
287 f, parent, unrecorded := quarantineUnrecordedFixture(t)
288 crash := errors.New("interrupted replacement retention")
289 writeFault := func(after bool) func(string, []byte) error {
290 return func(name string, body []byte) error {
291 var q servingParkQuarantine
292 if name == "namespace-quarantine.json" && servingParkDecode(body, &q) == nil && q.Unrecorded.Inode != 0 {
293 if boundary == "intent-before" && !after && !q.UnrecordedMoved || boundary == "intent-after" && after && !q.UnrecordedMoved || boundary == "moved-before" && !after && q.UnrecordedMoved || boundary == "moved-after" && after && q.UnrecordedMoved || boundary == "fresh-after" && after && q.Fresh.Inode != 0 || boundary == "complete" && after && q.Complete {
294 return crash
295 }
296 }
297 if boundary == "namespace" && after && name == "namespace.json" {
298 return crash
299 }
300 return nil
301 }
302 }
303 f.fs.state = quarantineWriteFault{Store: f.disk, before: writeFault(false), after: writeFault(true)}
304 if boundary == "rename" {
305 f.fs.rename = func(a int, b string, c int, d string, flags uint) error {
306 if err := unix.Renameat2(a, b, c, d, flags); err != nil {
307 return err
308 }
309 return crash
310 }
311 }
312 if boundary == "sync" {
313 f.fs.sync = func(int) error { return crash }
314 }
315 q, err := quarantineRecover(t, f, parent)
316 if boundary != "" && !errors.Is(err, crash) || boundary == "" && err != nil {
317 t.Fatal("unexpected interruption", err)
318 }
319 f.fs = newServingParkFS(f.data, f.disk)
320 f.fs.owner, f.fs.mount = uint32(os.Geteuid()), func(string) (bool, error) { return true, nil }
321 f.ledger.fs = f.fs
322 q, err = quarantineRecover(t, f, parent)
323 if err != nil || !q.Complete || !q.UnrecordedMoved || q.Unrecorded != unrecorded || q.Fresh == unrecorded {
324 t.Fatalf("replacement recovery: %+v %v", q, err)
325 }
326 quarantinePreserved(t, f, q)
327 body, err := os.ReadFile(filepath.Join(f.data, "instances", q.unrecordedName(), "uncertain"))
328 if err != nil || string(body) != "never adopt or delete" {
329 t.Fatal("lost unrecorded contents", err)
330 }
331 before, err := f.disk.Read("namespace-quarantine.json")
332 if err != nil {
333 t.Fatal(err)
334 }
335 for i := 0; i < 3; i++ {
336 if replay, err := quarantineRecover(t, f, parent); err != nil || !reflect.DeepEqual(replay, q) {
337 t.Fatal("completed replacement replay", err)
338 }
339 }
340 after, err := f.disk.Read("namespace-quarantine.json")
341 if err != nil || string(before) != string(after) {
342 t.Fatal("completed replay wrote authority", err)
343 }
344 names, err := os.ReadDir(filepath.Join(f.data, "instances"))
345 if err != nil || len(names) != 4 {
346 t.Fatalf("unbounded recovery directories: %v %v", names, err)
347 }
348 if boundary == "" {
349 f.put("", "new database")
350 a, stops := f.evidence(10, true, "missing")
351 r, err := f.park(a, stops)
352 if err != nil {
353 t.Fatal(err)
354 }
355 f.complete(a)
356 f.now = r.ReapAfter
357 if err := f.ledger.withNamespace(t.Context(), func() error { return f.ledger.reapLocked(t.Context(), a.ID) }); err != nil {
358 t.Fatal(err)
359 }
360 quarantinePreserved(t, f, q)
361 body, err := os.ReadFile(filepath.Join(f.data, "instances", q.unrecordedName(), "uncertain"))
362 if err != nil || string(body) != "never adopt or delete" {
363 t.Fatal("reaper touched retained replacement", err)
364 }
365 }
366 })
367 }
368}
369
370func TestServingParkQuarantineValidatesUnrecordedPhases(t *testing.T) {
371 f, parent, _ := quarantineUnrecordedFixture(t)
372 q, err := quarantineRecover(t, f, parent)
373 if err != nil {
374 t.Fatal(err)
375 }
376 for _, fault := range []string{"unmoved", "missing", "device", "mount", "inode", "obstruction", "fresh", "previous"} {
377 t.Run(fault, func(t *testing.T) {
378 broken := q
379 switch fault {
380 case "unmoved":
381 broken.UnrecordedMoved = false
382 case "missing":
383 broken.Unrecorded = servingArtifactDirectory{}
384 case "device":
385 broken.Unrecorded.Device++
386 case "mount":
387 broken.Unrecorded.Mount++
388 case "inode":
389 broken.Unrecorded.Inode = 0
390 case "obstruction":
391 broken.Unrecorded = q.Obstruction
392 case "fresh":
393 broken.Unrecorded = q.Fresh
394 case "previous":
395 broken.Previous = &servingParkNamespace{Version: 1, Parent: parent, Root: q.Unrecorded, BootID: q.BootID}
396 }
397 if !errors.Is(broken.validate(), hostbox.ErrChanged) {
398 t.Fatal("accepted invalid retained replacement", fault)
399 }
400 })
401 }
402}
403
404func TestServingParkQuarantineUnrecordedIdentityRefusals(t *testing.T) {
405 for _, fault := range []string{"parent", "symlink", "destination", "recorded-replacement", "cancel", "retained-replacement"} {
406 t.Run(fault, func(t *testing.T) {
407 f, parent, _ := quarantineUnrecordedFixture(t)
408 path := filepath.Join(f.data, "instances", ".serving-parked")
409 f.fs.state = quarantineWriteFault{Store: f.disk, after: func(name string, body []byte) error {
410 var q servingParkQuarantine
411 if name == "namespace-quarantine.json" && servingParkDecode(body, &q) == nil && q.Unrecorded.Inode != 0 && !q.UnrecordedMoved {
412 return unix.EIO
413 }
414 return nil
415 }}
416 q, err := quarantineRecover(t, f, parent)
417 if !errors.Is(err, unix.EIO) {
418 t.Fatal("missing retention intent", err)
419 }
420 f.fs.state = f.disk
421 switch fault {
422 case "parent":
423 parent.Inode++
424 case "symlink", "recorded-replacement":
425 if err := os.Rename(path, path+"-saved"); err != nil {
426 t.Fatal(err)
427 }
428 if fault == "symlink" {
429 err = os.Symlink(path+"-saved", path)
430 } else {
431 err = os.Mkdir(path, 0700)
432 }
433 if err != nil {
434 t.Fatal(err)
435 }
436 case "destination":
437 if err := os.Mkdir(filepath.Join(filepath.Dir(path), q.unrecordedName()), 0700); err != nil {
438 t.Fatal(err)
439 }
440 case "retained-replacement":
441 q, err = quarantineRecover(t, f, parent)
442 if err != nil {
443 t.Fatal(err)
444 }
445 retained := filepath.Join(filepath.Dir(path), q.unrecordedName())
446 if err := os.Rename(retained, retained+"-saved"); err != nil {
447 t.Fatal(err)
448 }
449 if err := os.Mkdir(retained, 0700); err != nil {
450 t.Fatal(err)
451 }
452 }
453 beforeNames, err := os.ReadDir(filepath.Dir(path))
454 if err != nil {
455 t.Fatal(err)
456 }
457 beforeRecord, err := f.disk.Read("namespace-quarantine.json")
458 if err != nil {
459 t.Fatal(err)
460 }
461 if fault == "cancel" {
462 ctx, cancel := context.WithCancel(t.Context())
463 cancel()
464 _, err = f.fs.recoverNamespace(ctx, parent)
465 } else {
466 _, err = quarantineRecover(t, f, parent)
467 }
468 if err == nil {
469 t.Fatal("unsafe replacement accepted")
470 }
471 afterNames, err := os.ReadDir(filepath.Dir(path))
472 if err != nil || len(afterNames) != len(beforeNames) {
473 t.Fatal("refusal changed retained directory names", err)
474 }
475 for i := range beforeNames {
476 if beforeNames[i].Name() != afterNames[i].Name() {
477 t.Fatal("refusal moved a replacement directory")
478 }
479 }
480 afterRecord, err := f.disk.Read("namespace-quarantine.json")
481 if err != nil || string(beforeRecord) != string(afterRecord) {
482 t.Fatal("refusal changed retention authority", err)
483 }
484 quarantinePreserved(t, f, q)
485 })
486 }
487}
488
489func TestServingParkQuarantineRefusesUnsafeAuthority(t *testing.T) {
490 for _, fault := range []string{"parent", "mode", "symlink", "mount", "cancel", "corrupt", "known-root"} {
491 t.Run(fault, func(t *testing.T) {
492 f, parent, path := quarantineFixture(t)
493 switch fault {
494 case "parent":
495 parent.Inode++
496 case "mode":
497 if err := os.Chmod(filepath.Dir(path), 0777); err != nil {
498 t.Fatal(err)
499 }
500 case "symlink":
501 if err := os.Rename(path, path+"-saved"); err != nil {
502 t.Fatal(err)
503 }
504 if err := os.Symlink(path+"-saved", path); err != nil {
505 t.Fatal(err)
506 }
507 case "mount":
508 f.fs.mount = func(string) (bool, error) { return false, nil }
509 case "corrupt":
510 if err := f.disk.Write("namespace.json", []byte("{}")); err != nil {
511 t.Fatal(err)
512 }
513 case "known-root":
514 d, err := os.Open(path)
515 if err != nil {
516 t.Fatal(err)
517 }
518 id, err := servingParkDirID(d)
519 d.Close()
520 if err != nil {
521 t.Fatal(err)
522 }
523 boot, err := f.fs.currentBoot()
524 if err != nil {
525 t.Fatal(err)
526 }
527 if err := f.ledger.write("namespace.json", servingParkNamespace{Version: 1, Parent: parent, Root: id, BootID: boot}); err != nil {
528 t.Fatal(err)
529 }
530 }
531 var err error
532 if fault == "cancel" {
533 ctx, cancel := context.WithCancel(t.Context())
534 cancel()
535 _, err = f.fs.recoverNamespace(ctx, parent)
536 } else {
537 _, err = quarantineRecover(t, f, parent)
538 }
539 if err == nil {
540 t.Fatal("unsafe quarantine accepted")
541 }
542 if _, err := f.disk.Read("namespace-quarantine.json"); !errors.Is(err, os.ErrNotExist) {
543 t.Fatal("unsafe authority persisted intent", err)
544 }
545 })
546 }
547}
548
549func TestServingParkQuarantineRetainedParkAllowsRetirement(t *testing.T) {
550 f := newServingParkDiskFixture(t)
551 f.put("", "old served database")
552 a, stops := f.evidence(10, true, "missing")
553 r, err := f.park(a, stops)
554 if err != nil {
555 t.Fatal(err)
556 }
557 f.complete(a)
558 path := filepath.Join(f.data, "instances", ".serving-parked")
559 if err := os.Rename(path, path+"-saved"); err != nil {
560 t.Fatal(err)
561 }
562 // Finding the genuine root does not authorize an unknown event child.
563 if err := os.WriteFile(filepath.Join(path+"-saved", a.ID, "unknown"), []byte("retain"), 0600); err != nil {
564 t.Fatal(err)
565 }
566 if err := os.Mkdir(path, 0700); err != nil {
567 t.Fatal(err)
568 }
569 if err := os.WriteFile(filepath.Join(path, "unknown"), []byte("keep these bytes"), 0600); err != nil {
570 t.Fatal(err)
571 }
572 q, err := quarantineRecover(t, f, r.Source.Parent)
573 if err != nil || q.Previous == nil {
574 t.Fatal("recorded-root recovery", err)
575 }
576 quarantinePreserved(t, f, q)
577 f.put("", "new served database")
578 next, receipts := f.evidence(11, false, "started")
579 parked, err := f.park(next, receipts)
580 if err != nil {
581 t.Fatal("retained quarantine blocked retirement", err)
582 }
583 f.complete(next)
584 f.retained(next, "new served database", parked.ReapAfter)
585 old, err := f.ledger.readHistorical(a.ID)
586 if err != nil || old.Closed || !old.ReapAfter.Equal(r.ReapAfter) {
587 t.Fatal("lost retained quarantine record", old, err)
588 }
589 body, err := os.ReadFile(filepath.Join(path+"-saved", a.ID, a.Successor.Host+".db"))
590 if err != nil || string(body) != "old served database" {
591 t.Fatal("old park changed", err)
592 }
593 body, err = os.ReadFile(filepath.Join(path+"-saved", a.ID, "unknown"))
594 if err != nil || string(body) != "retain" {
595 t.Fatal("unknown retained child changed", err)
596 }
597 if _, err = os.Stat(provision.NestedDBPath(f.data, f.base.Host)); !errors.Is(err, os.ErrNotExist) {
598 t.Fatal("new retirement left its source", err)
599 }
600 f.base.Host, f.target.Host = "other.example", "other.example"
601 if err := os.MkdirAll(filepath.Dir(provision.NestedDBPath(f.data, f.base.Host)), 0700); err != nil {
602 t.Fatal(err)
603 }
604 f.put("", "other host database")
605 other, otherStops := f.evidence(20, true, "missing")
606 if _, err := f.park(other, otherStops); err != nil {
607 t.Fatal("blocked host prevented unrelated park", err)
608 }
609 f.complete(other)
610 quarantinePreserved(t, f, q)
611 if _, err := f.disk.Read("retirement-" + q.ID + ".json"); !errors.Is(err, os.ErrNotExist) {
612 t.Fatal("quarantine became deletion authority", err)
613 }
614 if err := q.validate(); err != nil {
615 t.Fatal(err)
616 }
617 q.Fresh = q.Obstruction
618 if !errors.Is(q.validate(), hostbox.ErrChanged) {
619 t.Fatal("quarantine reused as fresh root")
620 }
621}
622