amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1//go:build linux
2
3package host
4
5import (
6 "context"
7 "errors"
8 "fmt"
9 "os"
10 "path/filepath"
11 "testing"
12 "time"
13
14 "github.com/carlosframework/platform/internal/hostbox"
15 "github.com/carlosframework/platform/internal/hostguard"
16 "github.com/carlosframework/platform/internal/instancectl"
17 "github.com/carlosframework/platform/internal/provision"
18 "github.com/carlosframework/platform/internal/rootstate"
19 "golang.org/x/sys/unix"
20)
21
22// This fixture exercises the durable driver, with explicit seams for root UID,
23// nosymfollow and the outer claim/host-guard authorization. It is not a
24// privileged mount or production transport witness.
25type servingParkDiskFixture struct {
26 t *testing.T
27 ledger *servingParkLedger
28 fs *servingParkFS
29 disk *rollbackDisk
30 data string
31 now time.Time
32 base servingArtifactIdentity
33 target hostguard.FenceTarget
34}
35
36func newServingParkDiskFixture(t *testing.T) *servingParkDiskFixture {
37 t.Helper()
38 rf, _, _, in := servingRetirementFixture(t)
39 admitted := servingAdmit(t, rf, &in, "G")
40 receipt := servingStop(t, rf, admitted)
41 base, err := servingWriterIdentity(admitted, "box-1", "ie")
42 if err != nil {
43 t.Fatal(err)
44 }
45 disk := &rollbackDisk{dir: t.TempDir()}
46 fs := newServingParkFS(rf.guard.DataDir, disk)
47 fs.volume = func(uint64) string { return "vol0123456789abcdef0" }
48 fs.owner = uint32(os.Geteuid())
49 fs.mount = func(string) (bool, error) { return true, nil }
50 f := &servingParkDiskFixture{t: t, fs: fs, disk: disk, data: rf.guard.DataDir, now: time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC), base: base, target: receipt.Target}
51 f.ledger = &servingParkLedger{state: disk, fs: fs, journal: rf.journal, now: func() time.Time { return f.now }}
52 return f
53}
54
55func (f *servingParkDiskFixture) evidence(n uint64, legacy bool, state string) (servingParkAuthorization, []hostguard.FenceReceipt) {
56 t := f.t
57 t.Helper()
58 prior, next := f.base, f.base
59 prior.Stamp, prior.Session, prior.Generation = n, n, fmt.Sprintf("G%d", n)
60 next.Stamp, next.Session, next.Generation = n+1, n+1, fmt.Sprintf("G%d", n+1)
61 target := f.target
62 target.Stamp, target.Session, target.FenceID = n, n, fmt.Sprintf("stop-%d", n)
63 s := StopReceipt{V: 1, ID: target.FenceID, Kind: StopReceiptStopped, Region: target.Region, ClaimETag: target.ClaimETag, ClaimGen: target.ClaimGen, FenceID: target.FenceID, Host: target.Host, Incarnation: target.Incarnation, InstallationStamp: target.Stamp, WriterSession: target.Session, Box: target.Box, Enrollment: target.Enrollment, BootID: target.BootID, ControllerID: target.ControllerID, DirectoryRevision: testReceiptNumeral(target), Binding: target.DirectoryRevision, EvidenceAt: f.now}
64 receipts := []hostguard.FenceReceipt{{Target: target, Bytes: mustMarshalStopReceipt(s), EvidenceAt: f.now}}
65 if state != "missing" && !legacy {
66 if err := f.ledger.journal.Prelaunch(t.Context(), hostbox.LaunchInventory{Target: target, ControllerPID: 77, ProcessStart: "controller-start", NoState: true, Groups: []hostbox.LaunchGroup{{RelativePath: "carlos/parent", Device: 1, Inode: 2}}}); err != nil {
67 t.Fatal(err)
68 }
69 if state != "prelaunch" {
70 res := hostbox.LaunchReservation{ID: fmt.Sprintf("%d:main", n), Role: "main", InvocationID: fmt.Sprintf("invocation-%d", n), Inputs: "inputs", Target: target, Group: hostbox.LaunchGroup{RelativePath: instancectl.ClaimUnitControlGroup(target.Host)}, Executable: hostbox.LaunchFile{Device: 1, Inode: 3}}
71 if err := f.ledger.journal.IntendUnit(t.Context(), res); err != nil {
72 t.Fatal(err)
73 }
74 if state == "started" {
75 group := hostbox.LaunchGroup{RelativePath: res.Group.RelativePath, Device: 1, Inode: 4, InvocationID: fmt.Sprintf("unit-%d", n)}
76 if err := f.ledger.journal.StartedUnit(t.Context(), res, group, 99, "tenant-start"); err != nil {
77 t.Fatal(err)
78 }
79 }
80 if state != "intended" {
81 if err := f.ledger.journal.SettleUnit(t.Context(), target, res.ID); err != nil {
82 t.Fatal(err)
83 }
84 }
85 }
86 }
87 source, err := os.Open(filepath.Dir(provision.NestedDBPath(f.data, next.Host)))
88 if err != nil {
89 t.Fatal(err)
90 }
91 id, err := servingParkDirID(source)
92 source.Close()
93 if err != nil {
94 t.Fatal(err)
95 }
96 a := servingParkAuthorization{TagsFrom: n, Predecessor: prior, OriginalSuccessor: next, Successor: next, Directory: id, Legacy: legacy}
97 if legacy {
98 a.Predecessor = servingArtifactIdentity{}
99 a.TagsFrom = 0
100 }
101 a.ID = servingRetirementID(a.Predecessor, next, legacy)
102 return a, receipts
103}
104func (f *servingParkDiskFixture) put(suffix, body string) {
105 f.t.Helper()
106 if err := os.WriteFile(provision.NestedDBPath(f.data, f.base.Host)+suffix, []byte(body), 0600); err != nil {
107 f.t.Fatal(err)
108 }
109}
110func (f *servingParkDiskFixture) park(a servingParkAuthorization, receipts []hostguard.FenceReceipt) (servingParkRecord, error) {
111 var r servingParkRecord
112 err := f.ledger.withNamespace(f.t.Context(), func() error { var err error; r, err = f.ledger.parkLocked(f.t.Context(), a, receipts); return err })
113 return r, err
114}
115func (f *servingParkDiskFixture) complete(a servingParkAuthorization) {
116 f.t.Helper()
117 if err := f.ledger.withNamespace(f.t.Context(), func() error { return f.ledger.completeLocked(f.t.Context(), a.ID, a.Successor) }); err != nil {
118 f.t.Fatal(err)
119 }
120}
121func (f *servingParkDiskFixture) path(a servingParkAuthorization, suffix string) string {
122 return filepath.Join(f.data, "instances", ".serving-parked", a.ID, a.Successor.Host+".db"+suffix)
123}
124func (f *servingParkDiskFixture) retained(a servingParkAuthorization, body string, deadline time.Time) {
125 f.t.Helper()
126 slot, err := f.ledger.slot(a)
127 if err != nil || slot.ID != a.ID {
128 f.t.Fatalf("park owner changed: %+v %v", slot, err)
129 }
130 b, err := os.ReadFile(f.path(a, ""))
131 if err != nil || string(b) != body {
132 f.t.Fatalf("retained database: %q %v", b, err)
133 }
134 r, err := f.ledger.read(a.ID)
135 if err != nil || !r.ReapAfter.Equal(deadline) {
136 f.t.Fatalf("park deadline changed: %v %v", r.ReapAfter, err)
137 }
138}
139
140func TestServingParkLedgerKeepsLastStartedDatabase(t *testing.T) {
141 f := newServingParkDiskFixture(t)
142 f.put("", "legacy writes")
143 f.put("-wal", "legacy WAL")
144 f.put("-shm", "legacy SHM")
145 legacy, stops := f.evidence(10, true, "missing")
146 parked, err := f.park(legacy, stops)
147 if err != nil {
148 t.Fatal(err)
149 }
150 f.complete(legacy)
151 for _, suffix := range []string{"-wal", "-shm"} {
152 if b, e := os.ReadFile(f.path(legacy, suffix)); e != nil || len(b) == 0 {
153 t.Fatalf("companion not preserved: %s %v", suffix, e)
154 }
155 }
156 for n := uint64(11); n < 179; n++ {
157 f.now = f.now.Add(time.Hour)
158 // Alternate a failed restore with no database and a successful restore
159 // whose tenant never started. Neither owns the retention slot.
160 if n%2 == 0 {
161 f.put("", "unlaunched baseline")
162 }
163 f.put(".gen", fmt.Sprintf("G%d", n))
164 f.put("-wal", "companion")
165 next, receipts := f.evidence(n, false, "prelaunch")
166 r, e := f.park(next, receipts)
167 if e != nil {
168 t.Fatalf("retirement %d: %v", n, e)
169 }
170 if !r.Closed || r.Disposition == servingParkRetain {
171 t.Fatalf("unlaunched source retained: %+v", r)
172 }
173 f.complete(next)
174 f.retained(legacy, "legacy writes", parked.ReapAfter)
175 }
176 f.put("", "tenant writes")
177 next, receipts := f.evidence(179, false, "started")
178 r, err := f.park(next, receipts)
179 if err != nil {
180 t.Fatal(err)
181 }
182 f.complete(next)
183 if _, err := os.Stat(f.path(legacy, "")); !errors.Is(err, os.ErrNotExist) {
184 t.Fatalf("legacy not replaced: %v", err)
185 }
186 f.retained(next, "tenant writes", r.ReapAfter)
187 // A tombstoned replay cannot replace the current slot or reset its clock.
188 if _, err := f.park(legacy, stops); err != nil {
189 t.Fatal(err)
190 }
191 f.complete(legacy)
192 f.retained(next, "tenant writes", r.ReapAfter)
193}
194
195func TestServingParkLedgerAmbiguousLaunchPreservesBothCopies(t *testing.T) {
196 for _, state := range []string{"missing", "intended", "settled-unobserved"} {
197 t.Run(state, func(t *testing.T) {
198 f := newServingParkDiskFixture(t)
199 f.put("", "legacy")
200 legacy, stops := f.evidence(10, true, "missing")
201 park, err := f.park(legacy, stops)
202 if err != nil {
203 t.Fatal(err)
204 }
205 f.complete(legacy)
206 f.put("", "uncertain tenant writes")
207 next, receipts := f.evidence(11, false, state)
208 if _, err := f.park(next, receipts); err == nil {
209 t.Fatal("ambiguous launch accepted")
210 }
211 f.retained(legacy, "legacy", park.ReapAfter)
212 b, err := os.ReadFile(provision.NestedDBPath(f.data, f.base.Host))
213 if err != nil || string(b) != "uncertain tenant writes" {
214 t.Fatalf("source changed: %q %v", b, err)
215 }
216 })
217 }
218}
219
220func TestServingParkLedgerExpiryAndIdentity(t *testing.T) {
221 for _, attack := range []string{"none", "inode", "hardlink", "symlink", "unknown child", "event", "root", "parent", "missing", "mount"} {
222 t.Run(attack, func(t *testing.T) {
223 f := newServingParkDiskFixture(t)
224 f.put("", "parked")
225 a, stops := f.evidence(10, true, "missing")
226 r, err := f.park(a, stops)
227 if err != nil {
228 t.Fatal(err)
229 }
230 f.complete(a)
231 f.now = r.ReapAfter.Add(-time.Nanosecond)
232 reap := func() error {
233 return f.ledger.withNamespace(t.Context(), func() error { return f.ledger.reapLocked(t.Context(), a.ID) })
234 }
235 if err := reap(); err == nil {
236 t.Fatal("reaped before deadline")
237 }
238 f.now = r.ParkedAt.Add(-time.Hour)
239 if err := reap(); err == nil {
240 t.Fatal("clock rollback reaped park")
241 }
242 f.now = r.ReapAfter
243 path := f.path(a, "")
244 victim := filepath.Join(t.TempDir(), "victim")
245 if err := os.WriteFile(victim, []byte("victim"), 0600); err != nil {
246 t.Fatal(err)
247 }
248 switch attack {
249 case "inode":
250 if err := os.Rename(path, path+".saved"); err != nil {
251 t.Fatal(err)
252 }
253 if err := os.WriteFile(path, []byte("victim"), 0600); err != nil {
254 t.Fatal(err)
255 }
256 if err := os.Rename(path+".saved", filepath.Join(t.TempDir(), "original")); err != nil {
257 t.Fatal(err)
258 }
259 case "hardlink":
260 if err := os.Link(path, path+".link"); err != nil {
261 t.Fatal(err)
262 }
263 case "symlink":
264 if err := os.Remove(path); err != nil {
265 t.Fatal(err)
266 }
267 if err := os.Symlink(victim, path); err != nil {
268 t.Fatal(err)
269 }
270 case "unknown child":
271 if err := os.WriteFile(path+".unknown", []byte("keep"), 0600); err != nil {
272 t.Fatal(err)
273 }
274 case "event", "root", "parent":
275 dir := filepath.Dir(path)
276 if attack == "root" {
277 dir = filepath.Dir(dir)
278 }
279 if attack == "parent" {
280 dir = filepath.Dir(filepath.Dir(dir))
281 }
282 if err := os.Rename(dir, dir+".saved"); err != nil {
283 t.Fatal(err)
284 }
285 if err := os.Mkdir(dir, 0700); err != nil {
286 t.Fatal(err)
287 }
288 case "missing":
289 if err := os.Remove(path); err != nil {
290 t.Fatal(err)
291 }
292 case "mount":
293 f.fs.mount = func(string) (bool, error) { return false, nil }
294 }
295 err = reap()
296 if attack == "none" {
297 if err != nil {
298 t.Fatal(err)
299 }
300 if _, e := os.Stat(path); !errors.Is(e, os.ErrNotExist) {
301 t.Fatalf("not reaped: %v", e)
302 }
303 } else if err == nil {
304 t.Fatal("unsafe reaping accepted")
305 }
306 if b, e := os.ReadFile(victim); e != nil || string(b) != "victim" {
307 t.Fatalf("victim changed: %q %v", b, e)
308 }
309 })
310 }
311}
312
313func TestServingParkLedgerNoReplaceAndCrossMountRefusal(t *testing.T) {
314 for _, failure := range []error{unix.EXDEV, unix.EEXIST} {
315 t.Run(failure.Error(), func(t *testing.T) {
316 f := newServingParkDiskFixture(t)
317 f.put("", "source")
318 a, stops := f.evidence(10, true, "missing")
319 f.fs.rename = func(int, string, int, string, uint) error { return failure }
320 if _, err := f.park(a, stops); !errors.Is(err, failure) {
321 t.Fatalf("rename result: %v", err)
322 }
323 b, err := os.ReadFile(provision.NestedDBPath(f.data, f.base.Host))
324 if err != nil || string(b) != "source" {
325 t.Fatalf("rename fallback touched source: %q %v", b, err)
326 }
327 f.fs.rename = unix.Renameat2
328 if _, err := f.park(a, stops); err != nil {
329 t.Fatal(err)
330 }
331 })
332 }
333}
334
335func TestServingParkLedgerLostRenameAndDeleteReplies(t *testing.T) {
336 for _, effect := range []string{"rename", "sync", "unlink"} {
337 for boundary := 1; boundary <= 4; boundary++ {
338 t.Run(fmt.Sprintf("%s-%d", effect, boundary), func(t *testing.T) {
339 f := newServingParkDiskFixture(t)
340 for _, suffix := range []string{"", "-wal", "-shm"} {
341 f.put(suffix, "source"+suffix)
342 }
343 a, stops := f.evidence(10, true, "missing")
344 calls := 0
345 fail := func(err error) error {
346 if err != nil {
347 return err
348 }
349 calls++
350 if calls == boundary {
351 return os.ErrPermission
352 }
353 return nil
354 }
355 if effect == "rename" {
356 f.fs.rename = func(a int, b string, c int, d string, e uint) error { return fail(unix.Renameat2(a, b, c, d, e)) }
357 }
358 if effect == "sync" {
359 f.fs.sync = func(fd int) error { return fail(unix.Fsync(fd)) }
360 }
361 _, _ = f.park(a, stops)
362 f.fs.rename, f.fs.sync = unix.Renameat2, unix.Fsync
363 // A failure before namespace.json exists is deliberately an obstruction;
364 // quarantine recovery (still unbound) must resolve it, never adoption.
365 if _, err := f.disk.Read("namespace.json"); errors.Is(err, os.ErrNotExist) {
366 if _, err := f.park(a, stops); err == nil {
367 t.Fatal("unrecorded namespace adopted")
368 }
369 return
370 }
371 r, err := f.park(a, stops)
372 if err != nil {
373 t.Fatal(err)
374 }
375 f.complete(a)
376 for _, suffix := range []string{"", "-wal", "-shm"} {
377 if b, e := os.ReadFile(f.path(a, suffix)); e != nil || string(b) != "source"+suffix {
378 t.Fatalf("parked bytes: %q %v", b, e)
379 }
380 }
381 f.now = r.ReapAfter
382 if effect == "unlink" {
383 f.fs.unlink = func(fd int, n string, flags int) error { return fail(unix.Unlinkat(fd, n, flags)) }
384 }
385 _ = f.ledger.withNamespace(t.Context(), func() error { return f.ledger.reapLocked(t.Context(), a.ID) })
386 f.fs.unlink = unix.Unlinkat
387 if err := f.ledger.withNamespace(t.Context(), func() error { return f.ledger.reapLocked(t.Context(), a.ID) }); err != nil {
388 t.Fatal(err)
389 }
390 })
391 }
392 }
393}
394
395func TestServingParkLedgerPendingCannotBeReplaced(t *testing.T) {
396 f := newServingParkDiskFixture(t)
397 f.put("", "legacy")
398 a, stops := f.evidence(10, true, "missing")
399 r, err := f.park(a, stops)
400 if err != nil {
401 t.Fatal(err)
402 }
403 f.put("", "new tenant writes")
404 next, receipts := f.evidence(11, false, "started")
405 if _, err := f.park(next, receipts); err == nil {
406 t.Fatal("pending park replaced")
407 }
408 f.retained(a, "legacy", r.ReapAfter)
409 if err := os.Remove(provision.NestedDBPath(f.data, f.base.Host)); err != nil {
410 t.Fatal(err)
411 }
412 f.complete(a)
413 f.put("", "new tenant writes")
414 if _, err := f.park(next, receipts); err != nil {
415 t.Fatal(err)
416 }
417}
418
419func TestServingParkLedgerCancelIsEffectFree(t *testing.T) {
420 f := newServingParkDiskFixture(t)
421 ctx, cancel := context.WithCancel(t.Context())
422 cancel()
423 called := false
424 if err := f.ledger.withNamespace(ctx, func() error { called = true; return nil }); !errors.Is(err, context.Canceled) || called {
425 t.Fatalf("cancel: %v, called=%v", err, called)
426 }
427}
428
429func TestServingParkLedgerDiscardResumesAfterUnlink(t *testing.T) {
430 f := newServingParkDiskFixture(t)
431 f.put("", "legacy")
432 old, stops := f.evidence(10, true, "missing")
433 r, err := f.park(old, stops)
434 if err != nil {
435 t.Fatal(err)
436 }
437 f.complete(old)
438 f.put("", "unlaunched")
439 next, receipts := f.evidence(11, false, "prelaunch")
440 f.fs.unlink = func(fd int, n string, flags int) error {
441 if err := unix.Unlinkat(fd, n, flags); err != nil {
442 return err
443 }
444 return os.ErrPermission
445 }
446 if _, err := f.park(next, receipts); err == nil {
447 t.Fatal("lost unlink reply succeeded")
448 }
449 f.retained(old, "legacy", r.ReapAfter)
450 f.fs.unlink = unix.Unlinkat
451 if _, err := f.park(next, receipts); err != nil {
452 t.Fatal(err)
453 }
454 f.complete(next)
455 f.retained(old, "legacy", r.ReapAfter)
456}
457
458func TestServingParkLedgerSourceReplacementDoesNotDeleteOldSlot(t *testing.T) {
459 f := newServingParkDiskFixture(t)
460 f.put("", "legacy")
461 old, stops := f.evidence(10, true, "missing")
462 r, err := f.park(old, stops)
463 if err != nil {
464 t.Fatal(err)
465 }
466 f.complete(old)
467 f.put("", "served")
468 next, receipts := f.evidence(11, false, "started")
469 // Interrupt just after the new source was captured and its intent saved.
470 f.disk.afterWrite = func(name string, _ []byte) {
471 if name == "retirement-"+next.ID+".json" {
472 panic("crash")
473 }
474 }
475 func() {
476 defer func() {
477 if recover() == nil {
478 t.Fatal("crash not reached")
479 }
480 }()
481 _, _ = f.park(next, receipts)
482 }()
483 f.disk.afterWrite = nil
484 source := provision.NestedDBPath(f.data, f.base.Host)
485 if err := os.Rename(source, source+".saved"); err != nil {
486 t.Fatal(err)
487 }
488 f.put("", "replacement")
489 if _, err := f.park(next, receipts); err == nil {
490 t.Fatal("replaced source accepted")
491 }
492 f.retained(old, "legacy", r.ReapAfter)
493}
494
495func TestServingParkLedgerChecksAllArtifactsBeforeDeletion(t *testing.T) {
496 f := newServingParkDiskFixture(t)
497 f.put("", "database")
498 f.put("-shm", "shared memory")
499 a, stops := f.evidence(10, true, "missing")
500 r, err := f.park(a, stops)
501 if err != nil {
502 t.Fatal(err)
503 }
504 f.complete(a)
505 if err := os.Remove(f.path(a, "-shm")); err != nil {
506 t.Fatal(err)
507 }
508 f.now = r.ReapAfter
509 if err := f.ledger.withNamespace(t.Context(), func() error { return f.ledger.reapLocked(t.Context(), a.ID) }); err == nil {
510 t.Fatal("unrecorded absence accepted")
511 }
512 if b, e := os.ReadFile(f.path(a, "")); e != nil || string(b) != "database" {
513 t.Fatalf("partial validation deleted database: %q %v", b, e)
514 }
515}
516
517func TestServingParkLedgerCompletionRechecksSource(t *testing.T) {
518 f := newServingParkDiskFixture(t)
519 f.put("", "legacy")
520 a, stops := f.evidence(10, true, "missing")
521 _, err := f.park(a, stops)
522 if err != nil {
523 t.Fatal(err)
524 }
525 f.put("", "replacement")
526 if err := f.ledger.withNamespace(t.Context(), func() error { return f.ledger.completeLocked(t.Context(), a.ID, a.Successor) }); err == nil {
527 t.Fatal("completed with replacement source")
528 }
529 r, err := f.ledger.read(a.ID)
530 if err != nil || r.Complete {
531 t.Fatalf("completion persisted: %v %v", r.Complete, err)
532 }
533}
534
535func TestServingParkLedgerLegacyConsumptionSurvivesReaping(t *testing.T) {
536 f := newServingParkDiskFixture(t)
537 f.put("", "legacy")
538 a, stops := f.evidence(10, true, "missing")
539 r, err := f.park(a, stops)
540 if err != nil {
541 t.Fatal(err)
542 }
543 f.complete(a)
544 f.now = r.ReapAfter
545 if err := f.ledger.withNamespace(t.Context(), func() error { return f.ledger.reapLocked(t.Context(), a.ID) }); err != nil {
546 t.Fatal(err)
547 }
548 // Even removal of the disposable retirement tombstone does not reauthorize
549 // a different legacy retirement for this installation/incarnation.
550 if err := f.disk.Remove("retirement-" + a.ID + ".json"); err != nil {
551 t.Fatal(err)
552 }
553 slot, err := f.ledger.slot(a)
554 if err == nil {
555 t.Fatal("slot accepted missing tombstone")
556 }
557 // Model future slot/tombstone GC; the durable consumed-source record stays.
558 slot = servingParkSlot{Version: 1, Box: a.Successor.Box, Host: a.Successor.Host}
559 if err := f.ledger.write("slot-"+servingParkKey(slot.Box, slot.Host)+".json", slot); err != nil {
560 t.Fatal(err)
561 }
562 f.put("", "restored successor")
563 next, receipts := f.evidence(11, true, "missing")
564 if _, err := f.park(next, receipts); err == nil {
565 t.Fatal("successor parked as legacy")
566 }
567}
568
569type servingParkFaultStore struct {
570 rootstate.Store
571 count, failAt int
572 before bool
573}
574
575func (s *servingParkFaultStore) Write(name string, body []byte) error {
576 s.count++
577 if s.count == s.failAt && s.before {
578 return os.ErrPermission
579 }
580 if err := s.Store.Write(name, body); err != nil {
581 return err
582 }
583 if s.count == s.failAt {
584 return os.ErrPermission
585 }
586 return nil
587}
588
589func TestServingParkLedgerDurableWriteFailures(t *testing.T) {
590 for _, before := range []bool{false, true} {
591 for boundary := 1; boundary <= 24; boundary++ {
592 t.Run(fmt.Sprintf("before-%t-boundary-%d", before, boundary), func(t *testing.T) {
593 f := newServingParkDiskFixture(t)
594 f.put("", "database")
595 f.put("-wal", "wal")
596 f.put("-shm", "shm")
597 a, stops := f.evidence(10, true, "missing")
598 faults := &servingParkFaultStore{Store: f.disk, failAt: boundary, before: before}
599 f.ledger.state, f.fs.state = faults, faults
600 r, err := f.park(a, stops)
601 if err == nil {
602 err = f.ledger.withNamespace(t.Context(), func() error { return f.ledger.completeLocked(t.Context(), a.ID, a.Successor) })
603 }
604 if err == nil {
605 f.now = r.ReapAfter
606 err = f.ledger.withNamespace(t.Context(), func() error { return f.ledger.reapLocked(t.Context(), a.ID) })
607 }
608 f.ledger.state, f.fs.state = f.disk, f.disk
609 if _, readErr := f.disk.Read("namespace.json"); os.IsNotExist(readErr) {
610 if _, statErr := os.Stat(filepath.Join(f.data, "instances", ".serving-parked")); statErr == nil {
611 if _, err := f.park(a, stops); err == nil {
612 t.Fatal("unknown root adopted after write failure")
613 }
614 return
615 }
616 }
617 r, err = f.park(a, stops)
618 if err != nil {
619 t.Fatalf("park replay: %v", err)
620 }
621 f.complete(a)
622 f.now = r.ReapAfter
623 if err := f.ledger.withNamespace(t.Context(), func() error { return f.ledger.reapLocked(t.Context(), a.ID) }); err != nil {
624 t.Fatalf("reap replay: %v", err)
625 }
626 got, err := f.ledger.read(a.ID)
627 if err != nil || !got.Closed || !got.Complete {
628 t.Fatalf("unfinished replay: %+v %v", got, err)
629 }
630 })
631 }
632 }
633}
634
635func TestServingParkLedgerSlotsAcrossIncarnationsAndHosts(t *testing.T) {
636 f := newServingParkDiskFixture(t)
637 f.put("", "first incarnation")
638 old, stops := f.evidence(10, true, "missing")
639 _, err := f.park(old, stops)
640 if err != nil {
641 t.Fatal(err)
642 }
643 f.complete(old)
644 f.base.Incarnation, f.target.Incarnation = "next-incarnation", "next-incarnation"
645 f.put("", "next incarnation")
646 next, receipts := f.evidence(11, false, "started")
647 r, err := f.park(next, receipts)
648 if err != nil {
649 t.Fatal(err)
650 }
651 f.complete(next)
652 if _, err := os.Stat(f.path(old, "")); !os.IsNotExist(err) {
653 t.Fatalf("old incarnation remains: %v", err)
654 }
655 f.base.Host, f.target.Host = "other.example", "other.example"
656 if err := os.MkdirAll(filepath.Dir(provision.NestedDBPath(f.data, f.base.Host)), 0700); err != nil {
657 t.Fatal(err)
658 }
659 f.put("", "other host")
660 other, receipts := f.evidence(12, true, "missing")
661 if _, err := f.park(other, receipts); err != nil {
662 t.Fatal(err)
663 }
664 f.complete(other)
665 f.retained(next, "next incarnation", r.ReapAfter)
666}
667
668func TestServingParkLedgerDiscardPendingBlocksNextRetirement(t *testing.T) {
669 f := newServingParkDiskFixture(t)
670 f.put("", "legacy")
671 old, stops := f.evidence(10, true, "missing")
672 r, err := f.park(old, stops)
673 if err != nil {
674 t.Fatal(err)
675 }
676 f.complete(old)
677 f.put("", "unlaunched")
678 pending, receipts := f.evidence(11, false, "prelaunch")
679 if _, err := f.park(pending, receipts); err != nil {
680 t.Fatal(err)
681 }
682 f.put("", "next tenant writes")
683 next, receipts := f.evidence(12, false, "started")
684 if _, err := f.park(next, receipts); err == nil {
685 t.Fatal("unfinished discard overtaken")
686 }
687 f.retained(old, "legacy", r.ReapAfter)
688}
689
690func TestServingParkLedgerRefusesPrematureTombstone(t *testing.T) {
691 f := newServingParkDiskFixture(t)
692 f.put("", "legacy")
693 a, stops := f.evidence(10, true, "missing")
694 r, err := f.park(a, stops)
695 if err != nil {
696 t.Fatal(err)
697 }
698 f.complete(a)
699 r.Complete, r.Deleting, r.Closed, r.Cause = true, true, true, "expiry"
700 if err := f.ledger.write("retirement-"+a.ID+".json", r); err != nil {
701 t.Fatal(err)
702 }
703 if _, err := f.ledger.read(a.ID); err == nil {
704 t.Fatal("incomplete deletion accepted as a tombstone")
705 }
706 if b, e := os.ReadFile(f.path(a, "")); e != nil || string(b) != "legacy" {
707 t.Fatalf("malformed record changed data: %q %v", b, e)
708 }
709}
710