| 1 | //go:build linux |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "errors" |
| 8 | "net" |
| 9 | "os" |
| 10 | "path/filepath" |
| 11 | "reflect" |
| 12 | "strings" |
| 13 | "testing" |
| 14 | |
| 15 | "github.com/carlosframework/platform/internal/hostbox" |
| 16 | "github.com/carlosframework/platform/internal/hostclaim" |
| 17 | "golang.org/x/sys/unix" |
| 18 | ) |
| 19 | |
| 20 | func servingRebootPreparationFixture(t *testing.T, phase string) (*servingRootFixture, servingParkRecord) { |
| 21 | t.Helper() |
| 22 | return servingRebootPreparationFixtureFor(t, phase, servingFixtureActor) |
| 23 | } |
| 24 | |
| 25 | func servingRebootPreparationFixtureFor(t *testing.T, phase string, actor hostbox.Actor) (*servingRootFixture, servingParkRecord) { |
| 26 | t.Helper() |
| 27 | f := newServingRootFixtureFor(t, true, actor) |
| 28 | fs := f.p.ledger.fs.(*servingParkFS) |
| 29 | fs.volume = func(uint64) string { return "vol0123456789abcdef0" } |
| 30 | if phase == "rename-reply" { |
| 31 | fs.rename = func(a int, b string, c int, d string, flags uint) error { |
| 32 | if err := unix.Renameat2(a, b, c, d, flags); err != nil { |
| 33 | return err |
| 34 | } |
| 35 | return unix.EIO |
| 36 | } |
| 37 | } |
| 38 | a, err := f.park() |
| 39 | if phase == "rename-reply" { |
| 40 | if !errors.Is(err, unix.EIO) { |
| 41 | t.Fatal(err) |
| 42 | } |
| 43 | a.ID = servingRetirementID(f.prior, f.current, false) |
| 44 | } else if err != nil { |
| 45 | t.Fatal(err) |
| 46 | } |
| 47 | fs.rename = unix.Renameat2 |
| 48 | if phase == "retagged" || phase == "complete" || phase == "source-gone" { |
| 49 | f.tags(f.current.Stamp) |
| 50 | } |
| 51 | if phase == "complete" || phase == "source-gone" { |
| 52 | if err := f.p.completeServingDatabasePark(t.Context(), f.conn, a.ID, f.current); err != nil { |
| 53 | t.Fatal(err) |
| 54 | } |
| 55 | } |
| 56 | if phase == "source-gone" { |
| 57 | if err := os.Rename(filepath.Dir(f.db), filepath.Dir(f.db)+"-gone"); err != nil { |
| 58 | t.Fatal(err) |
| 59 | } |
| 60 | } |
| 61 | r, err := f.p.ledger.read(a.ID) |
| 62 | if err != nil { |
| 63 | t.Fatal(err) |
| 64 | } |
| 65 | fs.boot = func() (string, error) { return servingOtherBoot, nil } |
| 66 | f.p.root.Stop.Authenticate = func(context.Context, net.Conn) (StopPeer, error) { panic("reboot preparation authenticated a peer") } |
| 67 | return f, r |
| 68 | } |
| 69 | |
| 70 | func TestServingParkRebootPreparationPreservesHistory(t *testing.T) { |
| 71 | for _, phase := range []string{"rename-reply", "parked", "retagged", "complete", "source-gone"} { |
| 72 | t.Run(phase, func(t *testing.T) { |
| 73 | f, r := servingRebootPreparationFixture(t, phase) |
| 74 | before := servingBootSnapshot(t, f.disk.dir) |
| 75 | claim := f.f.read() |
| 76 | root := servingBootSnapshot(t, f.rootDisk.dir) |
| 77 | name := servingRebootPreparationName(r.Authorization.ID, servingOtherBoot) |
| 78 | f.trace = nil |
| 79 | if err := f.p.prepareServingReboot(t.Context(), r.Authorization.ID); err != nil { |
| 80 | t.Fatal(err) |
| 81 | } |
| 82 | body, err := f.disk.Read(name) |
| 83 | var prep servingParkRebootPreparation |
| 84 | if err != nil || servingParkDecode(body, &prep) != nil || prep.Version != 1 || !reflect.DeepEqual(prep.Observation.Record, r) || (prep.Active == nil) != r.Complete { |
| 85 | t.Fatal("preparation", err) |
| 86 | } |
| 87 | after := servingBootSnapshot(t, f.disk.dir) |
| 88 | delete(after, name) |
| 89 | if !reflect.DeepEqual(before, after) || !reflect.DeepEqual(claim, f.f.read()) || !reflect.DeepEqual(root, servingBootSnapshot(t, f.rootDisk.dir)) { |
| 90 | t.Fatal("preparation changed authority or history") |
| 91 | } |
| 92 | if prep.Active != nil && (prep.Active.ClaimETag != claim.ETag || prep.Active.Directory != prep.Observation.Source) { |
| 93 | t.Fatal("active installation not bound") |
| 94 | } |
| 95 | trace := strings.Join(f.trace, "\n") |
| 96 | namespace := strings.Index(trace, "park:namespace.lock") |
| 97 | slot := strings.Index(trace, "park:slot-") |
| 98 | record := strings.Index(trace, "park:retirement-") |
| 99 | if namespace < 0 || slot < namespace || record < slot { |
| 100 | t.Fatal("lock order", trace) |
| 101 | } |
| 102 | if !r.Complete { |
| 103 | host := strings.Index(trace, "host:") |
| 104 | if host < namespace || host > slot { |
| 105 | t.Fatal("host lock order", trace) |
| 106 | } |
| 107 | } |
| 108 | f.disk.beforeWrite = func(string, []byte) { t.Fatal("unchanged preparation replay wrote") } |
| 109 | restarted := *f.p |
| 110 | if err := restarted.prepareServingReboot(t.Context(), r.Authorization.ID); err != nil { |
| 111 | t.Fatal("replay", err) |
| 112 | } |
| 113 | f.disk.beforeWrite = nil |
| 114 | fs := f.p.ledger.fs.(*servingParkFS) |
| 115 | if err := fs.recoverRecordedNamespace(t.Context()); err == nil { |
| 116 | t.Fatal("preparation granted namespace authority") |
| 117 | } |
| 118 | if err := fs.CheckParked(&r); err == nil { |
| 119 | t.Fatal("preparation granted artifact authority") |
| 120 | } |
| 121 | if err := fs.Delete(&r, func() error { t.Fatal("old authority wrote deletion progress"); return nil }); err == nil { |
| 122 | t.Fatal("preparation granted deletion") |
| 123 | } |
| 124 | }) |
| 125 | } |
| 126 | } |
| 127 | |
| 128 | func TestServingParkRebootPreparationRefusesAuthority(t *testing.T) { |
| 129 | for _, fault := range []string{"enrollment", "binding", "claim", "tags", "receipt", "launch-intent", "census", "cancel", "slot-id", "slot-pending", "slot-last", "slot-foreign", "slot-missing", "record-missing"} { |
| 130 | t.Run(fault, func(t *testing.T) { |
| 131 | f, r := servingRebootPreparationFixture(t, "parked") |
| 132 | ctx := t.Context() |
| 133 | var err error |
| 134 | switch fault { |
| 135 | case "enrollment": |
| 136 | f.actor.Enrollment = "changed" |
| 137 | case "binding": |
| 138 | f.actor.Binding = "8" |
| 139 | case "claim": |
| 140 | _, err = hostclaim.Update(ctx, f.f.claims, f.f.read(), func(c *hostclaim.Claim) error { c.Holder.WriterBox = "other"; return nil }) |
| 141 | case "tags": |
| 142 | f.tags(f.current.Stamp + 1) |
| 143 | case "receipt": |
| 144 | f.mutateRoot(func(m map[string]any) { m["State"].(map[string]any)["fence_journal"] = []any{} }) |
| 145 | case "census": |
| 146 | f.p.root.RequireNewWork = func(context.Context) error { return hostbox.ErrChanged } |
| 147 | case "cancel": |
| 148 | var cancel context.CancelFunc |
| 149 | ctx, cancel = context.WithCancel(ctx) |
| 150 | cancel() |
| 151 | case "launch-intent": |
| 152 | target := f.receipt.Target |
| 153 | target.Stamp, target.Session = f.current.Stamp, f.current.Session |
| 154 | err = f.f.journal.Prelaunch(ctx, hostbox.LaunchInventory{Target: target, ControllerPID: 77, ProcessStart: "edge-start", NoState: true, Groups: []hostbox.LaunchGroup{{RelativePath: "carlos/parent", Device: 1, Inode: 2}}}) |
| 155 | if err == nil { |
| 156 | err = f.f.journal.IntendUnit(ctx, hostbox.LaunchReservation{ID: "new:main", Role: "main", InvocationID: "new", Inputs: "digest", Target: target, Group: hostbox.LaunchGroup{RelativePath: "system.slice/carlos-app@shop.service"}, Executable: hostbox.LaunchFile{Device: 1, Inode: 3}}) |
| 157 | } |
| 158 | case "record-missing": |
| 159 | err = f.disk.Remove("retirement-" + r.Authorization.ID + ".json") |
| 160 | default: |
| 161 | slot, e := f.p.ledger.slot(r.Authorization) |
| 162 | if e != nil { |
| 163 | t.Fatal(e) |
| 164 | } |
| 165 | name := "slot-" + servingParkKey(slot.Box, slot.Host) + ".json" |
| 166 | switch fault { |
| 167 | case "slot-id": |
| 168 | slot.ID = "" |
| 169 | case "slot-pending": |
| 170 | slot.Pending = "" |
| 171 | case "slot-last": |
| 172 | slot.Last = "" |
| 173 | case "slot-foreign": |
| 174 | slot.Host = "other" |
| 175 | case "slot-missing": |
| 176 | err = f.disk.Remove(name) |
| 177 | } |
| 178 | if fault != "slot-missing" { |
| 179 | err = f.p.ledger.write(name, slot) |
| 180 | } |
| 181 | } |
| 182 | if err != nil { |
| 183 | t.Fatal(err) |
| 184 | } |
| 185 | before := servingBootSnapshot(t, f.disk.dir) |
| 186 | if err := f.p.prepareServingReboot(ctx, r.Authorization.ID); err == nil { |
| 187 | t.Fatal("accepted", fault) |
| 188 | } |
| 189 | if !reflect.DeepEqual(before, servingBootSnapshot(t, f.disk.dir)) { |
| 190 | t.Fatal("refusal wrote preparation") |
| 191 | } |
| 192 | }) |
| 193 | } |
| 194 | } |
| 195 | |
| 196 | func TestServingParkRebootPreparationLegacyConsumption(t *testing.T) { |
| 197 | for _, fault := range []string{"", "missing", "collected", "handle", "source-boot", "receipt"} { |
| 198 | t.Run(fault, func(t *testing.T) { |
| 199 | f, r := servingRebootRecordFixture(t, "complete") |
| 200 | key := servingLegacyKey("legacy", r.Authorization.Successor) |
| 201 | c, err := f.ledger.legacyConsumption(r.Authorization.Successor) |
| 202 | if err != nil || c == nil { |
| 203 | t.Fatal(err) |
| 204 | } |
| 205 | switch fault { |
| 206 | case "missing": |
| 207 | err = f.disk.Remove(key) |
| 208 | case "collected": |
| 209 | c.Collected = true |
| 210 | case "handle": |
| 211 | c.Source.Persistent.Handle = "aa" |
| 212 | case "source-boot": |
| 213 | c.Source.BootID = servingOtherBoot |
| 214 | case "receipt": |
| 215 | c.Receipts = nil |
| 216 | } |
| 217 | if fault != "missing" { |
| 218 | err = f.ledger.write(key, c) |
| 219 | } |
| 220 | if err != nil { |
| 221 | t.Fatal(err) |
| 222 | } |
| 223 | before := servingBootSnapshot(t, f.disk.dir) |
| 224 | p := servingParkRoot{ledger: f.ledger} |
| 225 | err = p.prepareServingReboot(t.Context(), r.Authorization.ID) |
| 226 | if (err != nil) != (fault != "") { |
| 227 | t.Fatal(fault, err) |
| 228 | } |
| 229 | after := servingBootSnapshot(t, f.disk.dir) |
| 230 | if fault == "" { |
| 231 | delete(after, servingRebootPreparationName(r.Authorization.ID, servingOtherBoot)) |
| 232 | } |
| 233 | if !reflect.DeepEqual(before, after) { |
| 234 | t.Fatal("changed permanent history") |
| 235 | } |
| 236 | }) |
| 237 | } |
| 238 | } |
| 239 | |
| 240 | func TestServingParkRebootPreparationLostWrite(t *testing.T) { |
| 241 | for _, before := range []bool{true, false} { |
| 242 | t.Run(map[bool]string{true: "before", false: "after"}[before], func(t *testing.T) { |
| 243 | f, r := servingRebootPreparationFixture(t, "parked") |
| 244 | original := servingBootSnapshot(t, f.disk.dir) |
| 245 | store := &servingParkFaultStore{Store: f.disk, failAt: 1, before: before} |
| 246 | f.p.ledger.state = store |
| 247 | if err := f.p.prepareServingReboot(t.Context(), r.Authorization.ID); !errors.Is(err, os.ErrPermission) { |
| 248 | t.Fatal("write failure", err) |
| 249 | } |
| 250 | f.p.ledger.state = f.disk |
| 251 | if err := f.p.prepareServingReboot(t.Context(), r.Authorization.ID); err != nil { |
| 252 | t.Fatal("retry", err) |
| 253 | } |
| 254 | after := servingBootSnapshot(t, f.disk.dir) |
| 255 | delete(after, servingRebootPreparationName(r.Authorization.ID, servingOtherBoot)) |
| 256 | if !reflect.DeepEqual(original, after) { |
| 257 | t.Fatal("write retry changed history") |
| 258 | } |
| 259 | }) |
| 260 | } |
| 261 | } |
| 262 | |
| 263 | func TestServingParkRebootPreparationNeverRefreshesStaleEvidence(t *testing.T) { |
| 264 | for _, fault := range []string{"slot", "record", "claim", "boot", "body", "unknown-field"} { |
| 265 | t.Run(fault, func(t *testing.T) { |
| 266 | f, r := servingRebootPreparationFixture(t, "parked") |
| 267 | id := r.Authorization.ID |
| 268 | name := servingRebootPreparationName(id, servingOtherBoot) |
| 269 | if err := f.p.prepareServingReboot(t.Context(), id); err != nil { |
| 270 | t.Fatal(err) |
| 271 | } |
| 272 | switch fault { |
| 273 | case "slot": |
| 274 | s, err := f.p.ledger.slot(r.Authorization) |
| 275 | if err != nil { |
| 276 | t.Fatal(err) |
| 277 | } |
| 278 | s.ID = "" |
| 279 | if err := f.p.ledger.write("slot-"+servingParkKey(s.Box, s.Host)+".json", s); err != nil { |
| 280 | t.Fatal(err) |
| 281 | } |
| 282 | case "record": |
| 283 | r.Cause = "fixture-change" |
| 284 | if err := f.p.ledger.save(r); err != nil { |
| 285 | t.Fatal(err) |
| 286 | } |
| 287 | case "claim": |
| 288 | f.tags(f.current.Stamp) |
| 289 | case "boot": |
| 290 | body, e := f.disk.Read(name) |
| 291 | if e != nil { |
| 292 | t.Fatal(e) |
| 293 | } |
| 294 | var prep servingParkRebootPreparation |
| 295 | if servingParkDecode(body, &prep) != nil { |
| 296 | t.Fatal("decode") |
| 297 | } |
| 298 | prep.Observation.Topology.BootID = "11234567-89ab-4cde-8fab-0123456789ab" |
| 299 | if e = f.p.ledger.write(name, prep); e != nil { |
| 300 | t.Fatal(e) |
| 301 | } |
| 302 | case "body": |
| 303 | if err := f.disk.Write(name, []byte("{}")); err != nil { |
| 304 | t.Fatal(err) |
| 305 | } |
| 306 | case "unknown-field": |
| 307 | body, err := f.disk.Read(name) |
| 308 | if err != nil { |
| 309 | t.Fatal(err) |
| 310 | } |
| 311 | body = append([]byte(`{"Unknown":true,`), body[1:]...) |
| 312 | if err := f.disk.Write(name, body); err != nil { |
| 313 | t.Fatal(err) |
| 314 | } |
| 315 | } |
| 316 | before := servingBootSnapshot(t, f.disk.dir) |
| 317 | if err := f.p.prepareServingReboot(t.Context(), id); err == nil { |
| 318 | t.Fatal("stale preparation refreshed") |
| 319 | } |
| 320 | if !reflect.DeepEqual(before, servingBootSnapshot(t, f.disk.dir)) { |
| 321 | t.Fatal("stale refusal wrote") |
| 322 | } |
| 323 | }) |
| 324 | } |
| 325 | } |
| 326 | |
| 327 | func TestServingParkRebootPreparationDetectsWriteRace(t *testing.T) { |
| 328 | for _, fault := range []string{"file", "slot", "cancel", "claim", "actor", "census"} { |
| 329 | t.Run(fault, func(t *testing.T) { |
| 330 | f, r := servingRebootPreparationFixture(t, "parked") |
| 331 | ctx, cancel := context.WithCancel(t.Context()) |
| 332 | defer cancel() |
| 333 | f.disk.afterWrite = func(name string, _ []byte) { |
| 334 | if !strings.HasPrefix(name, "reboot-") { |
| 335 | return |
| 336 | } |
| 337 | switch fault { |
| 338 | case "cancel": |
| 339 | cancel() |
| 340 | case "claim": |
| 341 | f.tags(f.current.Stamp) |
| 342 | case "actor": |
| 343 | f.actor.Binding = "8" |
| 344 | case "census": |
| 345 | f.p.root.RequireNewWork = func(context.Context) error { return hostbox.ErrChanged } |
| 346 | case "slot": |
| 347 | s, err := f.p.ledger.slot(r.Authorization) |
| 348 | if err != nil { |
| 349 | t.Fatal(err) |
| 350 | } |
| 351 | s.ID = "" |
| 352 | if err := f.p.ledger.write("slot-"+servingParkKey(s.Box, s.Host)+".json", s); err != nil { |
| 353 | t.Fatal(err) |
| 354 | } |
| 355 | case "file": |
| 356 | path := filepath.Join(f.f.guard.DataDir, "instances", ".serving-parked", r.Authorization.ID, r.Authorization.Successor.Host+".db") |
| 357 | if err := os.Rename(path, path+"-saved"); err != nil { |
| 358 | t.Fatal(err) |
| 359 | } |
| 360 | if err := os.WriteFile(path, []byte("replacement"), 0600); err != nil { |
| 361 | t.Fatal(err) |
| 362 | } |
| 363 | } |
| 364 | } |
| 365 | if err := f.p.prepareServingReboot(ctx, r.Authorization.ID); err == nil { |
| 366 | t.Fatal("accepted write race") |
| 367 | } |
| 368 | f.disk.afterWrite = nil |
| 369 | before := servingBootSnapshot(t, f.disk.dir) |
| 370 | if err := f.p.ledger.fs.(*servingParkFS).recoverRecordedNamespace(t.Context()); err == nil { |
| 371 | t.Fatal("failed preparation granted recovery") |
| 372 | } |
| 373 | if !reflect.DeepEqual(before, servingBootSnapshot(t, f.disk.dir)) { |
| 374 | t.Fatal("failed preparation changed authority") |
| 375 | } |
| 376 | }) |
| 377 | } |
| 378 | } |
| 379 | |
| 380 | func TestServingParkRebootPreparationLegacyContinuation(t *testing.T) { |
| 381 | for _, continued := range []bool{false, true} { |
| 382 | t.Run(map[bool]string{false: "original", true: "continued"}[continued], func(t *testing.T) { |
| 383 | f, _ := newServingLegacyRootFixture(t, false) |
| 384 | original := f.current |
| 385 | a, err := f.p.parkLegacyServingDatabase(t.Context(), f.conn, f.current, servingLegacyPredecessor{}) |
| 386 | if err != nil { |
| 387 | t.Fatal(err) |
| 388 | } |
| 389 | if continued { |
| 390 | servingStop(t, f.f, f.f.read()) |
| 391 | o := f.f.read() |
| 392 | later, err := hostclaim.Update(t.Context(), f.f.claims, o, hostclaim.AdmitWriter(hostclaim.Admission{ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: "box-1", Region: "ie", PredecessorRegion: "ie", Mode: "cold", NewGeneration: "G2"})) |
| 393 | if err != nil { |
| 394 | t.Fatal(err) |
| 395 | } |
| 396 | f.current, err = servingWriterIdentity(later, "box-1", "ie") |
| 397 | if err != nil { |
| 398 | t.Fatal(err) |
| 399 | } |
| 400 | f.admitted = later |
| 401 | f.install(later) |
| 402 | if _, err = f.p.parkLegacyServingDatabase(t.Context(), f.conn, f.current, servingLegacyPredecessor{}); err != nil { |
| 403 | t.Fatal(err) |
| 404 | } |
| 405 | } |
| 406 | fs := f.p.ledger.fs.(*servingParkFS) |
| 407 | fs.boot = func() (string, error) { return servingOtherBoot, nil } |
| 408 | before := servingBootSnapshot(t, f.disk.dir) |
| 409 | if err := f.p.prepareServingReboot(t.Context(), a.ID); err != nil { |
| 410 | t.Fatal(err) |
| 411 | } |
| 412 | name := servingRebootPreparationName(a.ID, servingOtherBoot) |
| 413 | body, err := f.disk.Read(name) |
| 414 | var prep servingParkRebootPreparation |
| 415 | if err != nil || servingParkDecode(body, &prep) != nil || prep.Active == nil || prep.Legacy == nil || prep.Observation.Record.Authorization.OriginalSuccessor != original || prep.Observation.Record.activeAuthorization().Successor != f.current { |
| 416 | t.Fatal("lost original or current authority", err) |
| 417 | } |
| 418 | after := servingBootSnapshot(t, f.disk.dir) |
| 419 | delete(after, name) |
| 420 | if !reflect.DeepEqual(before, after) { |
| 421 | t.Fatal("reboot preparation rewrote legacy or continuation") |
| 422 | } |
| 423 | }) |
| 424 | } |
| 425 | } |
| 426 | |
amadan