| 1 | //go:build linux |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "bytes" |
| 7 | "fmt" |
| 8 | "log" |
| 9 | "os" |
| 10 | "testing" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/instancectl" |
| 13 | ) |
| 14 | |
| 15 | func TestServingParkRetainedDoesNotBlockRetirement(t *testing.T) { |
| 16 | for _, attack := range []string{"unreadable-volume", "changed-volume", "hardlink", "unreadable-binding"} { |
| 17 | t.Run(attack, func(t *testing.T) { |
| 18 | f, old := servingRebootRecordFixture(t, "complete") |
| 19 | servingRenumberHistory(t, f, &old) |
| 20 | if attack == "unreadable-volume" { |
| 21 | f.fs.volume = func(uint64) string { return "" } |
| 22 | } |
| 23 | if attack == "changed-volume" { |
| 24 | f.fs.volume = func(uint64) string { return "vol11111111111111111" } |
| 25 | } |
| 26 | root := &servingParkRoot{ledger: f.ledger} |
| 27 | if err := root.commitServingReboot(t.Context(), old.Authorization.ID); err != nil { |
| 28 | t.Fatal(err) |
| 29 | } |
| 30 | if attack == "hardlink" { |
| 31 | if err := os.Link(f.path(old.Authorization, ""), f.path(old.Authorization, "")+".extra"); err != nil { |
| 32 | t.Fatal(err) |
| 33 | } |
| 34 | } |
| 35 | if attack == "unreadable-binding" { |
| 36 | if err := f.disk.Write("current-retirement-"+old.Authorization.ID+".json", []byte("invalid")); err != nil { |
| 37 | t.Fatal(err) |
| 38 | } |
| 39 | } |
| 40 | var report bytes.Buffer |
| 41 | output := log.Writer() |
| 42 | log.SetOutput(&report) |
| 43 | defer log.SetOutput(output) |
| 44 | for n := uint64(2); n <= 3; n++ { |
| 45 | body := fmt.Sprintf("tenant writes %d", n) |
| 46 | f.put("", body) |
| 47 | next, stops := f.evidence(n, false, "started") |
| 48 | parked, err := f.park(next, stops) |
| 49 | if err != nil { |
| 50 | t.Fatalf("retained park blocked retirement %d: %v", n, err) |
| 51 | } |
| 52 | f.complete(next) |
| 53 | f.retained(next, body, parked.ReapAfter) |
| 54 | } |
| 55 | saved, err := f.ledger.readHistorical(old.Authorization.ID) |
| 56 | if err != nil || saved.Closed || !saved.ReapAfter.Equal(old.ReapAfter) { |
| 57 | t.Fatal("lost retained record", saved, err) |
| 58 | } |
| 59 | for suffix, want := range map[string]string{"": "last database", "-wal": "last WAL", "-shm": "last SHM"} { |
| 60 | got, err := os.ReadFile(f.path(old.Authorization, suffix)) |
| 61 | if err != nil || string(got) != want { |
| 62 | t.Fatalf("retained %s: %q %v", suffix, got, err) |
| 63 | } |
| 64 | } |
| 65 | ids, more, err := f.ledger.recoveryIDs(t.Context(), "", servingParkRecoveryLimit) |
| 66 | if err != nil || more || len(ids) != 3 { |
| 67 | t.Fatal("retained record missing from census", ids, more, err) |
| 68 | } |
| 69 | if !bytes.Contains(report.Bytes(), []byte(old.Authorization.ID)) { |
| 70 | t.Fatal("retained park not reported", report.String()) |
| 71 | } |
| 72 | if err := f.ledger.maintainParks(t.Context(), func() error { return instancectl.ErrDiskFull }); err == nil { |
| 73 | t.Fatal("pressure granted deletion of retained park") |
| 74 | } |
| 75 | if _, err := os.Stat(f.path(old.Authorization, "")); err != nil { |
| 76 | t.Fatal("pressure lost retained park", err) |
| 77 | } |
| 78 | }) |
| 79 | } |
| 80 | } |
| 81 | |
| 82 | func TestServingParkRetainedReplacementWriteFailures(t *testing.T) { |
| 83 | for _, before := range []bool{false, true} { |
| 84 | for boundary := 1; boundary <= 18; boundary++ { |
| 85 | t.Run(fmt.Sprintf("before-%t-write-%d", before, boundary), func(t *testing.T) { |
| 86 | f, old := servingRebootRecordFixture(t, "complete") |
| 87 | servingRenumberHistory(t, f, &old) |
| 88 | f.fs.volume = func(uint64) string { return "" } |
| 89 | root := &servingParkRoot{ledger: f.ledger} |
| 90 | if err := root.commitServingReboot(t.Context(), old.Authorization.ID); err != nil { |
| 91 | t.Fatal(err) |
| 92 | } |
| 93 | next, stops := f.evidence(2, false, "started") |
| 94 | faults := &servingParkFaultStore{Store: f.disk, failAt: boundary, before: before} |
| 95 | f.ledger.state, f.fs.state = faults, faults |
| 96 | if _, err := f.park(next, stops); err == nil { |
| 97 | _ = f.ledger.withNamespace(t.Context(), func() error { return f.ledger.completeLocked(t.Context(), next.ID, next.Successor) }) |
| 98 | } |
| 99 | f.ledger.state, f.fs.state = f.disk, f.disk |
| 100 | parked, err := f.park(next, stops) |
| 101 | if err != nil { |
| 102 | t.Fatal("retirement retry", err) |
| 103 | } |
| 104 | f.complete(next) |
| 105 | f.retained(next, "successor database", parked.ReapAfter) |
| 106 | got, err := os.ReadFile(f.path(old.Authorization, "")) |
| 107 | if err != nil || string(got) != "last database" { |
| 108 | t.Fatalf("retained bytes after retry: %q %v", got, err) |
| 109 | } |
| 110 | saved, err := f.ledger.readHistorical(old.Authorization.ID) |
| 111 | if err != nil || saved.Closed || !saved.ReapAfter.Equal(old.ReapAfter) { |
| 112 | t.Fatal("retained record after retry", saved, err) |
| 113 | } |
| 114 | }) |
| 115 | } |
| 116 | } |
| 117 | } |
| 118 | |
amadan