| 1 | //go:build linux |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "errors" |
| 8 | "os" |
| 9 | "path/filepath" |
| 10 | "reflect" |
| 11 | "testing" |
| 12 | "time" |
| 13 | |
| 14 | "github.com/carlosframework/platform/internal/hostbox" |
| 15 | "github.com/carlosframework/platform/internal/provision" |
| 16 | "golang.org/x/sys/unix" |
| 17 | ) |
| 18 | |
| 19 | func servingRebootRecordFixture(t *testing.T, phase string) (*servingParkDiskFixture, servingParkRecord) { |
| 20 | t.Helper() |
| 21 | f := newServingParkDiskFixture(t) |
| 22 | a, receipts := f.evidence(1, true, "missing") |
| 23 | f.put("", "last database") |
| 24 | f.put("-wal", "last WAL") |
| 25 | f.put("-shm", "last SHM") |
| 26 | if err := f.fs.recordNamespaceParent(t.Context(), a.Successor, a.Directory); err != nil { |
| 27 | t.Fatal(err) |
| 28 | } |
| 29 | root, _, err := f.fs.namespace() |
| 30 | if err != nil { |
| 31 | t.Fatal(err) |
| 32 | } |
| 33 | root.Close() |
| 34 | var r servingParkRecord |
| 35 | if phase == "captured" || phase == "prepared" || phase == "rename-reply" { |
| 36 | r, err = f.ledger.capture(t.Context(), a, receipts, nil) |
| 37 | if err != nil { |
| 38 | t.Fatal(err) |
| 39 | } |
| 40 | if err = f.ledger.save(r); err != nil { |
| 41 | t.Fatal(err) |
| 42 | } |
| 43 | save := func() error { return f.ledger.save(r) } |
| 44 | if phase != "captured" { |
| 45 | if err = f.fs.Prepare(&r, save); err != nil { |
| 46 | t.Fatal(err) |
| 47 | } |
| 48 | } |
| 49 | if phase == "rename-reply" { |
| 50 | f.fs.rename = func(from int, old string, to int, name string, flags uint) error { |
| 51 | if err := unix.Renameat2(from, old, to, name, flags); err != nil { |
| 52 | return err |
| 53 | } |
| 54 | return unix.EIO |
| 55 | } |
| 56 | if err = f.fs.Move(&r, save); !errors.Is(err, unix.EIO) { |
| 57 | t.Fatal("missing lost rename reply", err) |
| 58 | } |
| 59 | f.fs.rename = unix.Renameat2 |
| 60 | } |
| 61 | } else { |
| 62 | r, err = f.park(a, receipts) |
| 63 | if err != nil { |
| 64 | t.Fatal(err) |
| 65 | } |
| 66 | if phase != "parked" { |
| 67 | f.complete(a) |
| 68 | } |
| 69 | if phase == "complete" { |
| 70 | f.put("", "successor database") |
| 71 | } |
| 72 | if phase == "unlink-reply" || phase == "partial-unlink" || phase == "rmdir-reply" || phase == "closed" { |
| 73 | f.now = f.now.Add(8 * 24 * time.Hour) |
| 74 | calls := 0 |
| 75 | f.fs.unlink = func(fd int, name string, flags int) error { |
| 76 | calls++ |
| 77 | if phase == "partial-unlink" && calls == 2 { |
| 78 | return unix.EIO |
| 79 | } |
| 80 | if err := unix.Unlinkat(fd, name, flags); err != nil { |
| 81 | return err |
| 82 | } |
| 83 | if phase == "unlink-reply" || phase == "rmdir-reply" && flags == unix.AT_REMOVEDIR { |
| 84 | return unix.EIO |
| 85 | } |
| 86 | return nil |
| 87 | } |
| 88 | err = f.ledger.withNamespace(t.Context(), func() error { return f.ledger.reapLocked(t.Context(), a.ID) }) |
| 89 | if phase == "closed" && err != nil || phase != "closed" && !errors.Is(err, unix.EIO) { |
| 90 | t.Fatal(phase, err) |
| 91 | } |
| 92 | f.fs.unlink = unix.Unlinkat |
| 93 | } |
| 94 | } |
| 95 | r, err = f.ledger.read(a.ID) |
| 96 | if err != nil { |
| 97 | t.Fatal(err) |
| 98 | } |
| 99 | f.fs.boot = func() (string, error) { return servingOtherBoot, nil } |
| 100 | return f, r |
| 101 | } |
| 102 | |
| 103 | func TestServingParkRebootRecordReadOnly(t *testing.T) { |
| 104 | for _, phase := range []string{"captured", "prepared", "rename-reply", "parked", "complete", "unlink-reply", "partial-unlink", "rmdir-reply", "closed"} { |
| 105 | t.Run(phase, func(t *testing.T) { |
| 106 | f, r := servingRebootRecordFixture(t, phase) |
| 107 | before := servingBootSnapshot(t, f.disk.dir) |
| 108 | var got servingParkRebootRecord |
| 109 | err := f.ledger.withNamespace(t.Context(), func() error { |
| 110 | var err error |
| 111 | got, err = f.fs.inspectRebootRecord(t.Context(), r.Authorization.ID) |
| 112 | return err |
| 113 | }) |
| 114 | if err != nil || !reflect.DeepEqual(got.Record, r) || got.Topology.BootID != servingOtherBoot { |
| 115 | t.Fatal("record inspection", got, err) |
| 116 | } |
| 117 | if r.Complete && got.Source != (servingArtifactDirectory{}) { |
| 118 | t.Fatal("completed park depended on live source") |
| 119 | } |
| 120 | if after := servingBootSnapshot(t, f.disk.dir); !reflect.DeepEqual(before, after) { |
| 121 | t.Fatal("inspection wrote authority") |
| 122 | } |
| 123 | if err := f.fs.CheckParked(&r); err == nil { |
| 124 | t.Fatal("observation granted old-boot effects") |
| 125 | } |
| 126 | }) |
| 127 | } |
| 128 | } |
| 129 | |
| 130 | func TestServingParkRebootRecordRefusesChangedEvidence(t *testing.T) { |
| 131 | for _, fault := range []string{"no-source-proof", "no-file-proof", "no-event-proof", "source-boot", "source-handle", "event-handle", "file-handle", "file-filesystem", "file-mode", "event-mode", "namespace", "source-replaced", "event-replaced", "file-replaced", "file-hardlink", "file-symlink", "unexpected-child", "missing-file", "duplicate-source", "old-format", "source-race", "event-race", "record-race", "cancel-race"} { |
| 132 | t.Run(fault, func(t *testing.T) { |
| 133 | f, r := servingRebootRecordFixture(t, "parked") |
| 134 | event := filepath.Dir(f.path(r.Authorization, "")) |
| 135 | source := filepath.Dir(provisionDBPath(f)) |
| 136 | ctx, cancel := context.WithCancel(t.Context()) |
| 137 | defer cancel() |
| 138 | switch fault { |
| 139 | case "no-source-proof": |
| 140 | r.Source.Persistent = servingParkPersistentDirectory{} |
| 141 | case "no-file-proof": |
| 142 | r.Source.Files[0].Persistent = servingParkPersistentFile{} |
| 143 | case "no-event-proof": |
| 144 | r.EventPersistent = servingParkPersistentDirectory{} |
| 145 | case "source-boot": |
| 146 | r.Source.BootID = servingOtherBoot |
| 147 | case "source-handle": |
| 148 | r.Source.Persistent.Handle = "aa" |
| 149 | case "event-handle": |
| 150 | r.EventPersistent.Handle = "aa" |
| 151 | case "file-handle": |
| 152 | r.Source.Files[0].Persistent.Handle = "aa" |
| 153 | case "file-filesystem": |
| 154 | r.Source.Files[0].Persistent.Filesystem[0]++ |
| 155 | case "file-mode": |
| 156 | r.Source.Files[0].Persistent.Mode ^= 0100 |
| 157 | case "event-mode": |
| 158 | r.EventPersistent.Mode ^= 0100 |
| 159 | case "namespace": |
| 160 | r.Namespace.Inode++ |
| 161 | case "source-replaced", "event-replaced": |
| 162 | path := source |
| 163 | if fault == "event-replaced" { |
| 164 | path = event |
| 165 | } |
| 166 | if err := os.Rename(path, path+"-old"); err != nil { |
| 167 | t.Fatal(err) |
| 168 | } |
| 169 | if err := os.Mkdir(path, 0700); err != nil { |
| 170 | t.Fatal(err) |
| 171 | } |
| 172 | case "file-replaced", "file-hardlink", "file-symlink", "missing-file": |
| 173 | path := f.path(r.Authorization, "") |
| 174 | if fault == "file-hardlink" { |
| 175 | if err := os.Link(path, path+"-link"); err != nil { |
| 176 | t.Fatal(err) |
| 177 | } |
| 178 | break |
| 179 | } |
| 180 | if err := os.Rename(path, path+"-old"); err != nil { |
| 181 | t.Fatal(err) |
| 182 | } |
| 183 | if fault == "file-symlink" { |
| 184 | if err := os.Symlink(path+"-old", path); err != nil { |
| 185 | t.Fatal(err) |
| 186 | } |
| 187 | } |
| 188 | if fault == "file-replaced" { |
| 189 | if err := os.WriteFile(path, []byte("replacement"), 0600); err != nil { |
| 190 | t.Fatal(err) |
| 191 | } |
| 192 | } |
| 193 | // Put the retained original outside the event so unknown-child checks |
| 194 | // cannot hide a missing/replaced-file validation bug. |
| 195 | if err := os.Rename(path+"-old", source+"/saved-original"); err != nil { |
| 196 | t.Fatal(err) |
| 197 | } |
| 198 | case "unexpected-child": |
| 199 | if err := os.WriteFile(filepath.Join(event, "unknown"), []byte("retain"), 0600); err != nil { |
| 200 | t.Fatal(err) |
| 201 | } |
| 202 | case "duplicate-source": |
| 203 | f.put("", "replacement") |
| 204 | case "old-format": |
| 205 | r.Source.Persistent = servingParkPersistentDirectory{} |
| 206 | r.Source.BootID = "" |
| 207 | r.EventPersistent = servingParkPersistentDirectory{} |
| 208 | for i := range r.Source.Files { |
| 209 | r.Source.Files[i].Persistent = servingParkPersistentFile{} |
| 210 | } |
| 211 | case "source-race", "event-race", "record-race", "cancel-race": |
| 212 | calls := 0 |
| 213 | f.fs.handle = func(d *os.File) (servingParkPersistentDirectory, error) { |
| 214 | p, err := servingParkPersistentDir(d) |
| 215 | calls++ |
| 216 | // The first topology inspection takes four handles; the pinned |
| 217 | // topology takes two, followed by source and event. |
| 218 | if calls == 8 { |
| 219 | switch fault { |
| 220 | case "cancel-race": |
| 221 | cancel() |
| 222 | case "record-race": |
| 223 | changed := r |
| 224 | changed.Receipts = nil |
| 225 | if err := f.ledger.write("retirement-"+r.Authorization.ID+".json", changed); err != nil { |
| 226 | t.Fatal(err) |
| 227 | } |
| 228 | default: |
| 229 | path := source |
| 230 | if fault == "event-race" { |
| 231 | path = event |
| 232 | } |
| 233 | if err := os.Rename(path, path+"-old"); err != nil { |
| 234 | t.Fatal(err) |
| 235 | } |
| 236 | if err := os.Mkdir(path, 0700); err != nil { |
| 237 | t.Fatal(err) |
| 238 | } |
| 239 | } |
| 240 | } |
| 241 | return p, err |
| 242 | } |
| 243 | } |
| 244 | if err := f.ledger.write("retirement-"+r.Authorization.ID+".json", r); err != nil { |
| 245 | t.Fatal(err) |
| 246 | } |
| 247 | before := servingBootSnapshot(t, f.disk.dir) |
| 248 | got, err := f.fs.inspectRebootRecord(ctx, r.Authorization.ID) |
| 249 | if err == nil || !reflect.DeepEqual(got, servingParkRebootRecord{}) { |
| 250 | t.Fatal("accepted changed record", got, err) |
| 251 | } |
| 252 | if fault != "record-race" { |
| 253 | if after := servingBootSnapshot(t, f.disk.dir); !reflect.DeepEqual(before, after) { |
| 254 | t.Fatal("refusal wrote authority") |
| 255 | } |
| 256 | } |
| 257 | }) |
| 258 | } |
| 259 | } |
| 260 | |
| 261 | func provisionDBPath(f *servingParkDiskFixture) string { |
| 262 | return filepath.Join(f.data, "instances", provision.InstanceID(f.base.Host), f.base.Host+".db") |
| 263 | } |
| 264 | |
| 265 | func TestServingParkPersistentEvidenceNeverUpgradesOldIntent(t *testing.T) { |
| 266 | for _, phase := range []string{"intent", "consumption"} { |
| 267 | t.Run(phase, func(t *testing.T) { |
| 268 | f := newServingParkDiskFixture(t) |
| 269 | a, receipts := f.evidence(1, true, "missing") |
| 270 | f.put("", "legacy") |
| 271 | r, err := f.ledger.capture(t.Context(), a, receipts, nil) |
| 272 | if err != nil { |
| 273 | t.Fatal(err) |
| 274 | } |
| 275 | r.Source.Persistent = servingParkPersistentDirectory{} |
| 276 | r.Source.BootID = "" |
| 277 | for i := range r.Source.Files { |
| 278 | r.Source.Files[i].Persistent = servingParkPersistentFile{} |
| 279 | } |
| 280 | if phase == "intent" { |
| 281 | err = f.ledger.save(r) |
| 282 | } else { |
| 283 | err = f.ledger.write(servingLegacyKey("legacy", a.Successor), servingLegacyConsumption{Authorization: a, Source: r.Source, Receipts: receipts}) |
| 284 | } |
| 285 | if err != nil { |
| 286 | t.Fatal(err) |
| 287 | } |
| 288 | parked, err := f.park(a, receipts) |
| 289 | if err != nil { |
| 290 | t.Fatal("old intent replay", err) |
| 291 | } |
| 292 | if parked.Source.Persistent != (servingParkPersistentDirectory{}) || parked.Source.BootID != "" || parked.EventPersistent != (servingParkPersistentDirectory{}) { |
| 293 | t.Fatal("upgraded old evidence") |
| 294 | } |
| 295 | f.complete(a) |
| 296 | }) |
| 297 | } |
| 298 | } |
| 299 | |
| 300 | func TestServingParkRebootFileRequiresDeleteIntent(t *testing.T) { |
| 301 | f, r := servingRebootRecordFixture(t, "unlink-reply") |
| 302 | r.Source.Files[0].DeleteIntent = false |
| 303 | if err := f.ledger.write("retirement-"+r.Authorization.ID+".json", r); err != nil { |
| 304 | t.Fatal(err) |
| 305 | } |
| 306 | if _, err := f.fs.inspectRebootRecord(t.Context(), r.Authorization.ID); !errors.Is(err, hostbox.ErrChanged) { |
| 307 | t.Fatal("absent artifact without delete intent accepted", err) |
| 308 | } |
| 309 | } |
| 310 | |
| 311 | func TestServingParkRebootRecordChangedMount(t *testing.T) { |
| 312 | f, r := servingRebootRecordFixture(t, "rename-reply") |
| 313 | oldMount := r.Source.Parent.Mount + 123 |
| 314 | r.Source.Parent.Mount = oldMount |
| 315 | r.Source.Directory.Mount = oldMount |
| 316 | r.Authorization.Directory.Mount = oldMount |
| 317 | r.Event.Mount = oldMount |
| 318 | r.Namespace.Mount = oldMount |
| 319 | for i := range r.Source.Files { |
| 320 | if r.Source.Files[i].Present { |
| 321 | r.Source.Files[i].Mount = oldMount |
| 322 | } |
| 323 | } |
| 324 | if err := f.ledger.save(r); err != nil { |
| 325 | t.Fatal(err) |
| 326 | } |
| 327 | parent, err := f.fs.namespaceParent() |
| 328 | if err != nil { |
| 329 | t.Fatal(err) |
| 330 | } |
| 331 | parent.Parent.Mount = oldMount |
| 332 | if err := f.ledger.write("namespace-parent.json", parent); err != nil { |
| 333 | t.Fatal(err) |
| 334 | } |
| 335 | var n servingParkNamespace |
| 336 | b, err := f.disk.Read("namespace.json") |
| 337 | if err != nil || servingParkDecode(b, &n) != nil { |
| 338 | t.Fatal(err) |
| 339 | } |
| 340 | n.Parent.Mount = oldMount |
| 341 | n.Root.Mount = oldMount |
| 342 | if err := f.ledger.write("namespace.json", n); err != nil { |
| 343 | t.Fatal(err) |
| 344 | } |
| 345 | got, err := f.fs.inspectRebootRecord(t.Context(), r.Authorization.ID) |
| 346 | if err != nil || got.Source.Mount == oldMount || got.Event.Mount == oldMount || got.Source.Mount != got.Event.Mount || !reflect.DeepEqual(got.Record, r) { |
| 347 | t.Fatal("changed-mount inspection", got, err) |
| 348 | } |
| 349 | } |
| 350 | |
| 351 | func TestServingParkRebootCompletedDoesNotReadSource(t *testing.T) { |
| 352 | f, r := servingRebootRecordFixture(t, "complete") |
| 353 | path := filepath.Dir(provisionDBPath(f)) |
| 354 | if err := os.Rename(path, path+"-removed"); err != nil { |
| 355 | t.Fatal(err) |
| 356 | } |
| 357 | if err := os.Symlink(path+"-removed", path); err != nil { |
| 358 | t.Fatal(err) |
| 359 | } |
| 360 | got, err := f.fs.inspectRebootRecord(t.Context(), r.Authorization.ID) |
| 361 | if err != nil || got.Source != (servingArtifactDirectory{}) { |
| 362 | t.Fatal("completed park consulted source", got, err) |
| 363 | } |
| 364 | } |
| 365 | |
| 366 | func TestServingParkPersistentLegacyConsumption(t *testing.T) { |
| 367 | f, r := servingRebootRecordFixture(t, "parked") |
| 368 | c, err := f.ledger.legacyConsumption(r.Authorization.Successor) |
| 369 | if err != nil || c == nil { |
| 370 | t.Fatal(err) |
| 371 | } |
| 372 | if !sameServingLegacyConsumption(*c, r) || !sameServingLegacySource(r.Source, c.Source) { |
| 373 | t.Fatal("fixture consumption mismatch") |
| 374 | } |
| 375 | old := r.Source |
| 376 | old.Persistent = servingParkPersistentDirectory{} |
| 377 | old.BootID = "" |
| 378 | for i := range old.Files { |
| 379 | old.Files[i].Persistent = servingParkPersistentFile{} |
| 380 | } |
| 381 | partial := old |
| 382 | partial.Files[0].Persistent = r.Source.Files[0].Persistent |
| 383 | if sameServingLegacySource(old, partial) { |
| 384 | t.Fatal("partial consumed proof accepted as old evidence") |
| 385 | } |
| 386 | for _, field := range []string{"boot", "directory", "file"} { |
| 387 | t.Run(field, func(t *testing.T) { |
| 388 | changed := *c |
| 389 | switch field { |
| 390 | case "boot": |
| 391 | changed.Source.BootID = servingOtherBoot |
| 392 | case "directory": |
| 393 | changed.Source.Persistent.Handle = "aa" |
| 394 | case "file": |
| 395 | changed.Source.Files[0].Persistent.Handle = "aa" |
| 396 | } |
| 397 | if sameServingLegacyConsumption(changed, r) || sameServingLegacySource(r.Source, changed.Source) { |
| 398 | t.Fatal("changed immutable consumption accepted") |
| 399 | } |
| 400 | }) |
| 401 | } |
| 402 | } |
| 403 | |
| 404 | func TestServingParkPersistentEventCaptureFailure(t *testing.T) { |
| 405 | f := newServingParkDiskFixture(t) |
| 406 | a, receipts := f.evidence(1, true, "missing") |
| 407 | f.put("", "preserve source") |
| 408 | f.fs.handle = func(d *os.File) (servingParkPersistentDirectory, error) { |
| 409 | if d.Name() == a.ID { |
| 410 | return servingParkPersistentDirectory{}, unix.EIO |
| 411 | } |
| 412 | return servingParkPersistentDir(d) |
| 413 | } |
| 414 | if _, err := f.park(a, receipts); !errors.Is(err, unix.EIO) { |
| 415 | t.Fatal("event capture failure", err) |
| 416 | } |
| 417 | r, err := f.ledger.read(a.ID) |
| 418 | if err != nil || r.Prepared || r.EventPersistent != (servingParkPersistentDirectory{}) { |
| 419 | t.Fatal("failed capture gained event authority", err) |
| 420 | } |
| 421 | body, err := os.ReadFile(provisionDBPath(f)) |
| 422 | if err != nil || string(body) != "preserve source" { |
| 423 | t.Fatal("capture failure changed source", err) |
| 424 | } |
| 425 | f.fs.handle = servingParkPersistentDir |
| 426 | if _, err := f.park(a, receipts); err != nil { |
| 427 | t.Fatal("same-boot event capture recovery", err) |
| 428 | } |
| 429 | } |
| 430 | |
amadan