| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "encoding/json" |
| 5 | "errors" |
| 6 | "log" |
| 7 | "os" |
| 8 | "path/filepath" |
| 9 | "strings" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/hostclaim" |
| 14 | "github.com/carlosframework/platform/internal/provision" |
| 15 | "github.com/carlosframework/platform/internal/release" |
| 16 | ) |
| 17 | |
| 18 | // Ruling 61: a relocation journal lives while its op runs. Once the host's |
| 19 | // claim runs another op, or the claim is gone, the sweep removes the op's |
| 20 | // install and attempt journals, its release positions and its build |
| 21 | // scratch. The live op's files stay, and a journal whose host cannot be |
| 22 | // read stays and is reported: absence is never evidence. |
| 23 | func TestRelocationJournalsAreSweptWhenTheirOpEnds(t *testing.T) { |
| 24 | ctx := t.Context() |
| 25 | store := release.NewDirStore(t.TempDir()) |
| 26 | created := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) |
| 27 | record := provision.Record{Schema: 1, Host: "shop.example", Slug: "shop", App: "app", Account: "acct", Region: "ie", Channel: "stable", CreatedAt: created} |
| 28 | if err := provision.CreateRecord(ctx, store, record); err != nil { |
| 29 | t.Fatal(err) |
| 30 | } |
| 31 | obs, err := hostclaim.Read(ctx, store, record.Host) |
| 32 | if err != nil { |
| 33 | t.Fatal(err) |
| 34 | } |
| 35 | obs, err = hostclaim.Update(ctx, store, obs, hostclaim.Adopt(hostclaim.NewHolder{Family: "instance", Key: provision.RecordKey(record.Account, record.App, record.Host), |
| 36 | Account: record.Account, App: record.App, Host: record.Host, Region: record.Region, CreatedAt: created}, "src", nil, nil, "adopt", func() string { return "id" })) |
| 37 | if err != nil { |
| 38 | t.Fatal(err) |
| 39 | } |
| 40 | if _, err = hostclaim.Update(ctx, store, obs, func(c *hostclaim.Claim) error { |
| 41 | c.Op = hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "rel-live", Phase: hostclaim.PhaseInstalling, Exec: "dst", Participants: []string{"src", "dst"}, |
| 42 | Intent: map[string]string{"region": "ie", "source_box": "src", "target_box": "dst"}} |
| 43 | return nil |
| 44 | }); err != nil { |
| 45 | t.Fatal(err) |
| 46 | } |
| 47 | data := t.TempDir() |
| 48 | files := map[string]string{ |
| 49 | "relocate/ops/rel-live/install.json": `{"v":1,"host":"shop.example","op":"rel-live"}`, |
| 50 | "relocate/ops/rel-old/install.json": `{"v":1,"host":"shop.example","op":"rel-old"}`, |
| 51 | "relocate/ops/rel-old/attempts.json": `{"v":1,"host":"shop.example","op":"rel-old"}`, |
| 52 | "relocate/ops/rel-gone/attempts.json": `{"v":1,"host":"gone.example","op":"rel-gone"}`, |
| 53 | "relocate/ops/rel-torn/install.json": `{"v":1,"host":`, |
| 54 | "relocate/release/rel-live.json": `{"request":{"Host":"shop.example"}}`, |
| 55 | "relocate/release/rel-old.json": `{"request":{"Host":"shop.example"}}`, |
| 56 | "relocate/build/rel-live/shop.example.db": "scratch", |
| 57 | "relocate/build/rel-old/shop.example.db": "scratch", |
| 58 | } |
| 59 | for path, body := range files { |
| 60 | full := filepath.Join(data, filepath.FromSlash(path)) |
| 61 | if err := os.MkdirAll(filepath.Dir(full), 0o700); err != nil { |
| 62 | t.Fatal(err) |
| 63 | } |
| 64 | if err := os.WriteFile(full, []byte(body), 0o600); err != nil { |
| 65 | t.Fatal(err) |
| 66 | } |
| 67 | } |
| 68 | if err := sweepRelocationJournals(ctx, data, store); err == nil { |
| 69 | t.Fatal("a torn journal was not reported") |
| 70 | } |
| 71 | for path := range files { |
| 72 | _, err := os.Stat(filepath.Join(data, filepath.FromSlash(path))) |
| 73 | live := filepath.Base(filepath.Dir(path)) == "rel-live" || filepath.Base(path) == "rel-live.json" || filepath.Base(filepath.Dir(path)) == "rel-torn" |
| 74 | switch { |
| 75 | case live && err != nil: |
| 76 | t.Errorf("%s was swept while its op runs or its host is unread: %v", path, err) |
| 77 | case !live && !errors.Is(err, os.ErrNotExist): |
| 78 | t.Errorf("%s survived the end of its op: %v", path, err) |
| 79 | } |
| 80 | } |
| 81 | for _, dir := range []string{"relocate/ops/rel-old", "relocate/ops/rel-gone", "relocate/build/rel-old"} { |
| 82 | if _, err := os.Stat(filepath.Join(data, filepath.FromSlash(dir))); !errors.Is(err, os.ErrNotExist) { |
| 83 | t.Errorf("%s was left behind: %v", dir, err) |
| 84 | } |
| 85 | } |
| 86 | // A second pass over what is left removes nothing more and still |
| 87 | // reports the torn journal. |
| 88 | if err := sweepRelocationJournals(ctx, data, store); err == nil { |
| 89 | t.Fatal("the torn journal stopped being reported") |
| 90 | } |
| 91 | if _, err := os.Stat(filepath.Join(data, "relocate/ops/rel-live/install.json")); err != nil { |
| 92 | t.Fatalf("the second pass swept the live op: %v", err) |
| 93 | } |
| 94 | } |
| 95 | |
| 96 | // Ruling 76: the op a serving claim records as completed keeps its |
| 97 | // journals, the probe stop's fence owner among them, until the completed |
| 98 | // relocation is marked discharged, so the hold release always finds the |
| 99 | // owner it lifts under. Then the next sweep removes them. |
| 100 | func TestRelocationJournalsOfACompletedOpStayUntilItsDischarge(t *testing.T) { |
| 101 | ctx := t.Context() |
| 102 | store := release.NewDirStore(t.TempDir()) |
| 103 | created := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) |
| 104 | record := provision.Record{Schema: 1, Host: "shop.example", Slug: "shop", App: "app", Account: "acct", Region: "ie", Channel: "stable", CreatedAt: created} |
| 105 | if err := provision.CreateRecord(ctx, store, record); err != nil { |
| 106 | t.Fatal(err) |
| 107 | } |
| 108 | obs, err := hostclaim.Read(ctx, store, record.Host) |
| 109 | if err != nil { |
| 110 | t.Fatal(err) |
| 111 | } |
| 112 | obs, err = hostclaim.Update(ctx, store, obs, hostclaim.Adopt(hostclaim.NewHolder{Family: "instance", Key: provision.RecordKey(record.Account, record.App, record.Host), |
| 113 | Account: record.Account, App: record.App, Host: record.Host, Region: record.Region, CreatedAt: created}, "dst", nil, nil, "adopt", func() string { return "id" })) |
| 114 | if err != nil { |
| 115 | t.Fatal(err) |
| 116 | } |
| 117 | done, _ := json.Marshal(hostclaim.CompletedRelocation{Op: "rel-done", Stop: hostclaim.ProbeStop{Box: "dst", Session: 12, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)}, Phase: hostclaim.PhaseRelocatingProbing}) |
| 118 | if obs, err = hostclaim.Update(ctx, store, obs, func(c *hostclaim.Claim) error { |
| 119 | c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "dst", Participants: []string{"dst"}, Journal: map[string]string{hostclaim.JournalCompleted: string(done)}} |
| 120 | return nil |
| 121 | }); err != nil { |
| 122 | t.Fatal(err) |
| 123 | } |
| 124 | data := t.TempDir() |
| 125 | files := []string{"relocate/ops/rel-done/install.json", "relocate/ops/rel-done/probe-fence.json", "relocate/ops/rel-fence-only/probe-fence.json"} |
| 126 | bodies := []string{`{"v":1,"host":"shop.example","op":"rel-done"}`, `{"host":"shop.example","session":12,"owner":"0123"}`, `{"host":"shop.example","session":3,"owner":"4567"}`} |
| 127 | for i, path := range files { |
| 128 | full := filepath.Join(data, filepath.FromSlash(path)) |
| 129 | if err := os.MkdirAll(filepath.Dir(full), 0o700); err != nil { |
| 130 | t.Fatal(err) |
| 131 | } |
| 132 | if err := os.WriteFile(full, []byte(bodies[i]), 0o600); err != nil { |
| 133 | t.Fatal(err) |
| 134 | } |
| 135 | } |
| 136 | if err := sweepRelocationJournals(ctx, data, store); err != nil { |
| 137 | t.Fatal(err) |
| 138 | } |
| 139 | for _, path := range files[:2] { |
| 140 | if _, err := os.Stat(filepath.Join(data, filepath.FromSlash(path))); err != nil { |
| 141 | t.Fatalf("%s was swept before the discharge: %v", path, err) |
| 142 | } |
| 143 | } |
| 144 | if _, err := os.Stat(filepath.Join(data, "relocate/ops/rel-fence-only")); !errors.Is(err, os.ErrNotExist) { |
| 145 | t.Fatalf("an ended op known only by its fence owner survived: %v", err) |
| 146 | } |
| 147 | if _, err := hostclaim.Update(ctx, store, obs, hostclaim.MarkCompletedDischarged("rel-done")); err != nil { |
| 148 | t.Fatal(err) |
| 149 | } |
| 150 | if err := sweepRelocationJournals(ctx, data, store); err != nil { |
| 151 | t.Fatal(err) |
| 152 | } |
| 153 | if _, err := os.Stat(filepath.Join(data, "relocate/ops/rel-done")); !errors.Is(err, os.ErrNotExist) { |
| 154 | t.Fatalf("the discharged op's journals survived: %v", err) |
| 155 | } |
| 156 | } |
| 157 | |
| 158 | // sweepClaim is shop.example's claim, adopted on "dst", with its op |
| 159 | // replaced by op. |
| 160 | func sweepClaim(t *testing.T, op hostclaim.Op) *release.DirStore { |
| 161 | t.Helper() |
| 162 | ctx := t.Context() |
| 163 | store := release.NewDirStore(t.TempDir()) |
| 164 | created := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) |
| 165 | record := provision.Record{Schema: 1, Host: "shop.example", Slug: "shop", App: "app", Account: "acct", Region: "ie", Channel: "stable", CreatedAt: created} |
| 166 | if err := provision.CreateRecord(ctx, store, record); err != nil { |
| 167 | t.Fatal(err) |
| 168 | } |
| 169 | obs, err := hostclaim.Read(ctx, store, record.Host) |
| 170 | if err != nil { |
| 171 | t.Fatal(err) |
| 172 | } |
| 173 | obs, err = hostclaim.Update(ctx, store, obs, hostclaim.Adopt(hostclaim.NewHolder{Family: "instance", Key: provision.RecordKey(record.Account, record.App, record.Host), |
| 174 | Account: record.Account, App: record.App, Host: record.Host, Region: record.Region, CreatedAt: created}, "dst", nil, nil, "adopt", func() string { return "id" })) |
| 175 | if err != nil { |
| 176 | t.Fatal(err) |
| 177 | } |
| 178 | if _, err := hostclaim.Update(ctx, store, obs, func(c *hostclaim.Claim) error { c.Op = op; return nil }); err != nil { |
| 179 | t.Fatal(err) |
| 180 | } |
| 181 | return store |
| 182 | } |
| 183 | |
| 184 | func writeSweepFiles(t *testing.T, data string, files map[string]string) { |
| 185 | t.Helper() |
| 186 | for path, body := range files { |
| 187 | full := filepath.Join(data, filepath.FromSlash(path)) |
| 188 | if err := os.MkdirAll(filepath.Dir(full), 0o700); err != nil { |
| 189 | t.Fatal(err) |
| 190 | } |
| 191 | if err := os.WriteFile(full, []byte(body), 0o600); err != nil { |
| 192 | t.Fatal(err) |
| 193 | } |
| 194 | } |
| 195 | } |
| 196 | |
| 197 | // Fix round 1, I1: a serving claim whose completed relocation cannot be |
| 198 | // read keeps every journal of its host, the probe stop's fence owner |
| 199 | // included, and the sweep reports it: it may be that op's, and absence is |
| 200 | // never evidence. |
| 201 | func TestRelocationJournalsStayWhileTheCompletionCannotBeRead(t *testing.T) { |
| 202 | for name, raw := range map[string]string{"unparseable": `{"op":"rel-done",`, "no box": `{"op":"rel-done","stop":{"session":12}}`} { |
| 203 | t.Run(name, func(t *testing.T) { |
| 204 | store := sweepClaim(t, hostclaim.Op{Kind: hostclaim.KindServing, Exec: "dst", Participants: []string{"dst"}, Journal: map[string]string{hostclaim.JournalCompleted: raw}}) |
| 205 | data := t.TempDir() |
| 206 | files := map[string]string{ |
| 207 | "relocate/ops/rel-done/install.json": `{"v":1,"host":"shop.example","op":"rel-done"}`, |
| 208 | "relocate/ops/rel-done/probe-fence.json": `{"host":"shop.example","session":12,"owner":"0123"}`, |
| 209 | "relocate/release/rel-done.json": `{"request":{"Host":"shop.example"}}`, |
| 210 | "relocate/build/rel-done/shop.example.db": "scratch", |
| 211 | } |
| 212 | writeSweepFiles(t, data, files) |
| 213 | if err := sweepRelocationJournals(t.Context(), data, store); !errors.Is(err, hostclaim.ErrRelocation) { |
| 214 | t.Fatalf("an unreadable completion was not reported: %v", err) |
| 215 | } |
| 216 | for path := range files { |
| 217 | if _, err := os.Stat(filepath.Join(data, filepath.FromSlash(path))); err != nil { |
| 218 | t.Errorf("%s was swept over an unreadable completion: %v", path, err) |
| 219 | } |
| 220 | } |
| 221 | }) |
| 222 | } |
| 223 | } |
| 224 | |
| 225 | // Fix round 1, M5: what no committed journal can be is removed (an ops |
| 226 | // directory with no host-bearing journal, an atomic write's leftover |
| 227 | // temporaries under release/, a build directory holding only SQLite |
| 228 | // sidecars of an ended op, or nothing); the live op's sidecar stays; an |
| 229 | // entry the sweep does not know stays and is logged once, not every pass, |
| 230 | // and is no error. |
| 231 | func TestRelocationSweepRemovesLeftoversAndLogsAStrayOnce(t *testing.T) { |
| 232 | store := sweepClaim(t, hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "rel-live", Phase: hostclaim.PhaseInstalling, Exec: "dst", Participants: []string{"src", "dst"}, |
| 233 | Intent: map[string]string{"region": "ie", "source_box": "src", "target_box": "dst"}}) |
| 234 | data := t.TempDir() |
| 235 | writeSweepFiles(t, data, map[string]string{ |
| 236 | "relocate/ops/rel-tmp/.install.json.123456": `{"v":1,"host":`, |
| 237 | "relocate/release/rel-x.json.tmp": `{"request":`, |
| 238 | "relocate/release/.rel-y.json.987654": `{"req`, |
| 239 | "relocate/build/rel-wal/shop.example.db-wal": "wal", |
| 240 | "relocate/build/rel-jrn/shop.example.db-journal": "journal", |
| 241 | "relocate/build/rel-live/shop.example.db-wal": "wal", |
| 242 | "relocate/build/stray.txt": "?", |
| 243 | "relocate/release/notes.txt": "?", |
| 244 | }) |
| 245 | for _, dir := range []string{"relocate/ops/rel-empty", "relocate/build/rel-empty"} { |
| 246 | if err := os.MkdirAll(filepath.Join(data, filepath.FromSlash(dir)), 0o700); err != nil { |
| 247 | t.Fatal(err) |
| 248 | } |
| 249 | } |
| 250 | var logged strings.Builder |
| 251 | log.SetOutput(&logged) |
| 252 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 253 | for pass := 0; pass < 2; pass++ { |
| 254 | if err := sweepRelocationJournals(t.Context(), data, store); err != nil { |
| 255 | t.Fatalf("pass %d: %v", pass, err) |
| 256 | } |
| 257 | } |
| 258 | for _, gone := range []string{"relocate/ops/rel-tmp", "relocate/ops/rel-empty", "relocate/release/rel-x.json.tmp", "relocate/release/.rel-y.json.987654", |
| 259 | "relocate/build/rel-wal", "relocate/build/rel-jrn", "relocate/build/rel-empty"} { |
| 260 | if _, err := os.Stat(filepath.Join(data, filepath.FromSlash(gone))); !errors.Is(err, os.ErrNotExist) { |
| 261 | t.Errorf("%s survived the sweep: %v", gone, err) |
| 262 | } |
| 263 | } |
| 264 | for _, kept := range []string{"relocate/build/rel-live/shop.example.db-wal", "relocate/build/stray.txt", "relocate/release/notes.txt"} { |
| 265 | if _, err := os.Stat(filepath.Join(data, filepath.FromSlash(kept))); err != nil { |
| 266 | t.Errorf("%s was removed: %v", kept, err) |
| 267 | } |
| 268 | } |
| 269 | for _, stray := range []string{"stray.txt", "notes.txt"} { |
| 270 | if n := strings.Count(logged.String(), stray); n != 1 { |
| 271 | t.Errorf("%s logged %d times over two passes, want once: %q", stray, n, logged.String()) |
| 272 | } |
| 273 | } |
| 274 | } |
| 275 | |
amadan