| 1 | //go:build linux |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "errors" |
| 8 | "fmt" |
| 9 | "os" |
| 10 | "os/exec" |
| 11 | "path/filepath" |
| 12 | "strings" |
| 13 | "testing" |
| 14 | "time" |
| 15 | |
| 16 | "github.com/carlosframework/platform/internal/activator" |
| 17 | "github.com/carlosframework/platform/internal/hostbox" |
| 18 | "github.com/carlosframework/platform/internal/hostclaim" |
| 19 | "github.com/carlosframework/platform/internal/hostguard" |
| 20 | "github.com/carlosframework/platform/internal/hostwrite" |
| 21 | "github.com/carlosframework/platform/internal/park" |
| 22 | "github.com/carlosframework/platform/internal/provision" |
| 23 | "github.com/carlosframework/platform/internal/registry" |
| 24 | "github.com/carlosframework/platform/internal/release" |
| 25 | ) |
| 26 | |
| 27 | // These tests drive claim Plan 9 D4 on a landing claim written only by |
| 28 | // main's own mutators: the create, the source's writer, BeginMove, the seal, |
| 29 | // the source's evidence, the flip, the landing phases, moves' local |
| 30 | // inventory under the flip's stamp F, and the probe's AdmitWriter. No |
| 31 | // park-local chore exists at any point. The wake's production Stop, Admit, |
| 32 | // Prepare and Restore closures run against the real guard, launch journal, |
| 33 | // root stop coordinator and park.Local; the kernel is recoveryKernel, a |
| 34 | // fixture that reads the real launch journal. |
| 35 | |
| 36 | var ( |
| 37 | plan9D4Probe = activator.StartPolicy{MaxStarts: 5, Window: 30 * time.Second, Probe: true} |
| 38 | plan9D4Wake = activator.StartPolicy{MaxStarts: 5, Window: 30 * time.Second} |
| 39 | ) |
| 40 | |
| 41 | type plan9D4 struct { |
| 42 | f *recoveryFixture |
| 43 | data, db string |
| 44 | inc string |
| 45 | flip uint64 |
| 46 | permissive bool |
| 47 | crossBox int |
| 48 | handovers []hostguard.FenceTarget |
| 49 | } |
| 50 | |
| 51 | func (d *plan9D4) local() park.Local { |
| 52 | return park.Local{Guard: d.f.guard, Claims: d.f.claims, DataDir: d.data, Box: "box-1"} |
| 53 | } |
| 54 | |
| 55 | func (d *plan9D4) step(t *testing.T, what string, fn func(*hostclaim.Claim) error) { |
| 56 | t.Helper() |
| 57 | if _, err := hostclaim.Update(t.Context(), d.f.claims, d.f.read(), fn); err != nil { |
| 58 | t.Fatalf("%s: %v", what, err) |
| 59 | } |
| 60 | } |
| 61 | |
| 62 | // plan9D4Landing moves the host from box-0 to box-1 with main's mutators and |
| 63 | // leaves the landing at probing, box-1's local install tagged F. |
| 64 | func plan9D4Landing(t *testing.T) *plan9D4 { |
| 65 | t.Helper() |
| 66 | ctx := t.Context() |
| 67 | f := newRecoveryFixture(t) |
| 68 | st := release.NewDirStore(t.TempDir()) |
| 69 | f.claims, f.guard.Claims = st, hostguard.StoreReader{Store: st} |
| 70 | d := &plan9D4{f: f, data: f.guard.DataDir, db: provision.DefaultDBPath(f.guard.DataDir, recoveryHost)} |
| 71 | if err := os.Mkdir(filepath.Join(d.data, ".parked"), 0770); err != nil { |
| 72 | t.Fatal(err) |
| 73 | } |
| 74 | in := hostclaim.NewHolder{Family: "instance", Key: "apps/bab/jam/instances/" + recoveryHost + ".json", Account: "bab", App: "jam", Host: recoveryHost, Region: "ie", RecordRegion: "ie", CreatedAt: f.now} |
| 75 | body := []byte(`{"schema":1,"host":"shop.bab.oncarlos.com","account":"bab","app":"jam","region":"ie","created_at":"2026-10-06T12:00:00Z"}`) |
| 76 | d.step(t, "create", hostclaim.BeginCreate(in, body, "create", "console", f.now)) |
| 77 | d.step(t, "complete the create", hostclaim.CompleteCreate("create", "box-0")) |
| 78 | h := f.read().Claim.Holder |
| 79 | d.step(t, "admit the source's writer", hostclaim.AdmitWriter(hostclaim.Admission{ObservedSession: h.Session, ObservedGeneration: h.Generation, Box: "box-0", Mode: "cold", NewGeneration: "src", KeepStamp: h.Stamp > h.Session, Region: "ie"})) |
| 80 | d.step(t, "begin the move", hostclaim.BeginMove("m1", "console", "box-0", "box-1", "ie", f.now)) |
| 81 | d.step(t, "touch", hostclaim.Touch("m1", []string{"ie"}, []string{"box-1"})) |
| 82 | d.step(t, "seal", hostclaim.SetMovePhase("m1", hostclaim.KindMoving, hostclaim.PhaseMoving, hostclaim.PhaseSealed)) |
| 83 | d.step(t, "source evidence", hostclaim.ExecToNone("m1", "box-0", "moves/bab/jam/"+recoveryHost+"/m1/stopped-source.json", hostclaim.JournalEvidenceSource)) |
| 84 | h = f.read().Claim.Holder |
| 85 | d.step(t, "flip", hostclaim.Flip("m1", "box-1", h.Stamp, h.Generation)) |
| 86 | if err := f.guard.Activate(ctx, recoveryHost, "box-1", nil); err != nil { |
| 87 | t.Fatal(err) |
| 88 | } |
| 89 | h = f.read().Claim.Holder |
| 90 | d.inc, d.flip = h.Incarnation, h.Stamp |
| 91 | order := provision.LandingOrder() |
| 92 | for i := 0; order[i] != provision.LandProbing; i++ { |
| 93 | if order[i] == provision.LandLocal { |
| 94 | d.installLocal(t) |
| 95 | } |
| 96 | d.step(t, "landing "+order[i+1], hostclaim.SetMovePhase("m1", hostclaim.KindLanding, order[i], order[i+1])) |
| 97 | } |
| 98 | return d |
| 99 | } |
| 100 | |
| 101 | // installLocal is what moves' local phase leaves: the database and the |
| 102 | // marker naming the prepared generation (the op id), the files directory and |
| 103 | // <host>.env, all four recorded under F through park.Local. |
| 104 | func (d *plan9D4) installLocal(t *testing.T) { |
| 105 | t.Helper() |
| 106 | files := filepath.Join(d.data, recoveryHost) |
| 107 | if err := os.Mkdir(files, 0700); err != nil { |
| 108 | t.Fatal(err) |
| 109 | } |
| 110 | for path, body := range map[string]string{d.db: "landing database", filepath.Join(files, "index.html"): "files", filepath.Join(d.data, recoveryHost+".env"): "A=1\n"} { |
| 111 | if err := os.WriteFile(path, []byte(body), 0600); err != nil { |
| 112 | t.Fatal(err) |
| 113 | } |
| 114 | } |
| 115 | if err := hostbox.WriteGenerationMarker(d.db+".gen", "m1"); err != nil { |
| 116 | t.Fatal(err) |
| 117 | } |
| 118 | check := hostclaim.RequireMoveOp("m1", hostclaim.KindLanding, provision.LandLocal) |
| 119 | paths := []string{recoveryHost + ".db", recoveryHost + ".db.gen", recoveryHost, recoveryHost + ".env"} |
| 120 | if err := d.local().RecordInventory(t.Context(), recoveryHost, hostguard.Captured{Incarnation: d.inc, Stamp: d.flip}, "flat", paths, check); err != nil { |
| 121 | t.Fatal(err) |
| 122 | } |
| 123 | } |
| 124 | |
| 125 | // plan9D4Inputs is the moves Inputs owner's landing admission: the first |
| 126 | // probe opens the prepared generation and keeps F; every later writer takes |
| 127 | // a fresh generation and its own stamp (claim §6.4a items 3 and 4). |
| 128 | type plan9D4Inputs struct { |
| 129 | recoveryInputs |
| 130 | d *plan9D4 |
| 131 | } |
| 132 | |
| 133 | func (in plan9D4Inputs) Select(_ context.Context, _ registry.Route, o hostclaim.Observed, _ activator.StartPolicy) (ClaimLaunchInputs, error) { |
| 134 | h, op := o.Claim.Holder, o.Claim.Op |
| 135 | generation, coverage := fmt.Sprintf("g%d", o.Claim.Gen+1), "" |
| 136 | if h.WriterBox != "box-1" { |
| 137 | generation, coverage = op.ID, "sealed" |
| 138 | } |
| 139 | return ClaimLaunchInputs{Roles: []string{"main"}, Identity: hostwrite.Identity{Config: "config", Release: "release"}, |
| 140 | Admission: hostclaim.Admission{ObservedOpID: op.ID, ObservedPhase: op.Phase, ObservedSession: h.Session, ObservedGeneration: h.Generation, Box: "box-1", Mode: "cold", NewGeneration: generation, KeepStamp: h.Stamp > h.Session, Coverage: coverage, Region: "ie", PredecessorRegion: "ie"}, |
| 141 | Executable: "/usr/bin/true", ExecutableSHA256: strings.Repeat("a", 64), DBPath: in.d.db, Empty: true, ReadyPath: "/ready"}, nil |
| 142 | } |
| 143 | |
| 144 | // handover stands in for production's AfterAdmission dispatch |
| 145 | // (claimAdmissionHandover). A first probe's predecessor is the source on |
| 146 | // box-0: D2's cross-box handover, tested in plan9_d2_test.go. Every later |
| 147 | // predecessor is box-1's own writer, and coldAdmissionHook first takes the |
| 148 | // predecessor session's completed receipt exactly as here, then asks root's |
| 149 | // ColdHandover to retire it, which this unprivileged fixture cannot run. |
| 150 | func (d *plan9D4) handover(ctx context.Context, prior, _ hostclaim.Observed) error { |
| 151 | old := prior.Claim.Holder |
| 152 | if old.WriterBox != "box-1" { |
| 153 | d.crossBox++ |
| 154 | return nil |
| 155 | } |
| 156 | if d.permissive { |
| 157 | return nil |
| 158 | } |
| 159 | receipt, err := d.f.guard.CompletedColdPredecessor(ctx, prior.Host, hostguard.Captured{Incarnation: old.Incarnation, Stamp: old.Stamp}, old.Session, prior.Claim.Op.ID, prior.Claim.Op.Phase) |
| 160 | if err != nil { |
| 161 | return err |
| 162 | } |
| 163 | if receipt.Target.ObligationID != hostguard.WriterStopObligation(old.Session) || receipt.Target.Box != "box-1" { |
| 164 | return hostbox.ErrChanged |
| 165 | } |
| 166 | d.handovers = append(d.handovers, receipt.Target) |
| 167 | return verifyRetirementReceipt(ctx, receipt) |
| 168 | } |
| 169 | |
| 170 | // controller starts one agent process on box-1. bound is the moves C1 |
| 171 | // integration, bound only with CARLOS_HOST_CLAIM and CARLOS_MOVES_PHASE1 on. |
| 172 | func (d *plan9D4) controller(boot, controller string, pid int, start string, bound bool) *recoveryController { |
| 173 | c := d.f.controller(boot, controller, pid, start) |
| 174 | c.wake.Inputs = plan9D4Inputs{recoveryInputs{d.f}, d} |
| 175 | c.wake.AfterAdmission = d.handover |
| 176 | if bound { |
| 177 | c.wake.Continue = hostclaim.SessionContinues |
| 178 | c.wake.Operations = &plan9D2Attempts{} |
| 179 | c.wake.ProbeReady = func(context.Context, *exec.Cmd, registry.Route, ClaimLaunchInputs, string) error { return nil } |
| 180 | c.wake.RetireArtifacts = claimArtifactRetirement{Local: d.local(), Continue: hostclaim.SessionContinues}.Retire |
| 181 | c.wake.RecoverArtifacts = claimArtifactRetirement{Local: d.local(), Continue: hostclaim.SessionContinues}.Recover |
| 182 | } |
| 183 | return c |
| 184 | } |
| 185 | |
| 186 | // start runs one landing writer start as the wake does: the production Stop |
| 187 | // of the captured predecessor (Prerequisites' call), a stand-in for the S3 |
| 188 | // lease, then Admit, Prepare and the production Restore closure, whose |
| 189 | // ColdRestore calls RetireArtifacts. The written database stands in for the |
| 190 | // started app's own writes. |
| 191 | func (d *plan9D4) start(t *testing.T, c *recoveryController, p activator.StartPolicy, stop bool, materialize ...func(hostclaim.Observed, activator.ClaimStart) error) (hostclaim.Observed, error) { |
| 192 | t.Helper() |
| 193 | ctx := t.Context() |
| 194 | o := d.f.read() |
| 195 | if stop { |
| 196 | if err := c.wake.Stop(ctx, o); err != nil { |
| 197 | t.Fatalf("stop the predecessor session %d: %v", o.Claim.Holder.Session, err) |
| 198 | } |
| 199 | } |
| 200 | live, cancel := context.WithCancel(context.Background()) |
| 201 | t.Cleanup(cancel) |
| 202 | done := make(chan struct{}) |
| 203 | go func() { <-live.Done(); close(done) }() |
| 204 | c.wake.mu.Lock() |
| 205 | c.wake.pending[recoveryHost] = &claimWakeAttempt{observed: o, held: recoveryHeld{}, ctx: live, cancel: cancel, done: done} |
| 206 | c.wake.mu.Unlock() |
| 207 | admitted, err := c.runtime.Admit(ctx, registry.Route{Host: recoveryHost}, o, p) |
| 208 | if err != nil { |
| 209 | t.Fatalf("admit: %v", err) |
| 210 | } |
| 211 | s, err := c.runtime.Prepare(ctx, registry.Route{Host: recoveryHost}, admitted, p) |
| 212 | if err != nil { |
| 213 | t.Fatalf("prepare: %v", err) |
| 214 | } |
| 215 | if err := s.Restore(ctx); err != nil { |
| 216 | return admitted, err |
| 217 | } |
| 218 | for _, run := range materialize { |
| 219 | if err := run(admitted, s); err != nil { |
| 220 | return admitted, err |
| 221 | } |
| 222 | } |
| 223 | session := admitted.Claim.Holder.Session |
| 224 | return admitted, os.WriteFile(d.db, []byte(fmt.Sprintf("written by session %d", session)), 0600) |
| 225 | } |
| 226 | |
| 227 | func (d *plan9D4) mustStart(t *testing.T, c *recoveryController, p activator.StartPolicy, stop bool) hostclaim.Observed { |
| 228 | t.Helper() |
| 229 | admitted, err := d.start(t, c, p, stop) |
| 230 | if err != nil { |
| 231 | t.Fatalf("restore: %v", err) |
| 232 | } |
| 233 | return admitted |
| 234 | } |
| 235 | |
| 236 | func (d *plan9D4) read(t *testing.T, path string) string { |
| 237 | t.Helper() |
| 238 | body, err := os.ReadFile(path) |
| 239 | if err != nil { |
| 240 | t.Fatal(err) |
| 241 | } |
| 242 | return string(body) |
| 243 | } |
| 244 | |
| 245 | func (d *plan9D4) parked(t *testing.T, retirement string) string { |
| 246 | return filepath.Join(d.data, ".parked", d.inc, recoveryHost, retirement, "artifacts") |
| 247 | } |
| 248 | |
| 249 | // requireRetired checks one restart or cold wake: the predecessor's writer |
| 250 | // stop, its database and marker parked under this admission's retirement and |
| 251 | // tagged with its stamp, the files and environment left at F, no chore. |
| 252 | func (d *plan9D4) requireRetired(t *testing.T, prior, admitted hostclaim.Observed, recovered bool) { |
| 253 | t.Helper() |
| 254 | old, h := prior.Claim.Holder, admitted.Claim.Holder |
| 255 | req := d.f.requests[len(d.f.requests)-1] |
| 256 | if req.ObligationID != hostguard.WriterStopObligation(old.Session) || req.OperationID != "m1" || req.Phase != prior.Claim.Op.Phase || req.WriterSession != old.Session || |
| 257 | req.InstallationStamp != old.Stamp || req.Box != "box-1" || req.Recover != recovered { |
| 258 | t.Fatalf("the predecessor's stop request: %+v", req) |
| 259 | } |
| 260 | if n := len(d.handovers); n == 0 || d.handovers[n-1].Session != old.Session || d.handovers[n-1].Stamp != old.Stamp { |
| 261 | t.Fatalf("the cold handover did not find session %d's receipt: %+v", old.Session, d.handovers) |
| 262 | } |
| 263 | retired := d.parked(t, fmt.Sprintf("superseded-%d-%d", h.Session, old.Stamp)) |
| 264 | if got := d.read(t, filepath.Join(retired, recoveryHost+".db.gen")); got != old.Generation+"\n" { |
| 265 | t.Fatalf("parked marker %q, want %s", got, old.Generation) |
| 266 | } |
| 267 | if got := d.read(t, filepath.Join(retired, recoveryHost+".db")); got != fmt.Sprintf("written by session %d", old.Session) { |
| 268 | t.Fatalf("parked database %q", got) |
| 269 | } |
| 270 | if got := d.read(t, d.db+".gen"); got != h.Generation+"\n" { |
| 271 | t.Fatalf("live marker %q, want %s", got, h.Generation) |
| 272 | } |
| 273 | c := d.f.read().Claim |
| 274 | paths := c.Inventory["box-1"].Paths |
| 275 | if paths[recoveryHost+".db"] != h.Stamp || paths[recoveryHost+".db.gen"] != h.Stamp || paths[recoveryHost] != d.flip || paths[recoveryHost+".env"] != d.flip || len(paths) != 4 { |
| 276 | t.Fatalf("inventory after retiring stamp %d: %+v", old.Stamp, paths) |
| 277 | } |
| 278 | if len(c.Obligations) != 0 { |
| 279 | t.Fatalf("a chore appeared: %+v", c.Obligations) |
| 280 | } |
| 281 | if d.read(t, filepath.Join(d.data, recoveryHost, "index.html")) != "files" || d.read(t, filepath.Join(d.data, recoveryHost+".env")) != "A=1\n" { |
| 282 | t.Fatal("the landing's files or environment moved") |
| 283 | } |
| 284 | } |
| 285 | |
| 286 | func TestPlan9D4RestartsAndColdWakeRetireOnlyTheSupersededGeneration(t *testing.T) { |
| 287 | d := plan9D4Landing(t) |
| 288 | ctx := t.Context() |
| 289 | c := d.controller("boot-1", "controller-1", 4101, "start-1", true) |
| 290 | |
| 291 | // The first probe opens the prepared generation and keeps F: nothing is |
| 292 | // superseded and nothing is retired. Its predecessor is the source. |
| 293 | first := d.mustStart(t, c, plan9D4Probe, false) |
| 294 | if h := first.Claim.Holder; h.Generation != "m1" || h.Stamp != d.flip || d.crossBox != 1 || len(d.f.requests) != 0 { |
| 295 | t.Fatalf("first probe: %+v crossBox=%d requests=%d", h, d.crossBox, len(d.f.requests)) |
| 296 | } |
| 297 | if entries, err := os.ReadDir(filepath.Join(d.data, ".parked")); err != nil || len(entries) != 0 { |
| 298 | t.Fatalf("the first probe parked something: %v %v", entries, err) |
| 299 | } |
| 300 | |
| 301 | // First restart: stop the probe as the landing's own writer, take a |
| 302 | // fresh generation and stamp, retire the prepared database tagged F. |
| 303 | second := d.mustStart(t, c, plan9D4Probe, true) |
| 304 | d.requireRetired(t, first, second, false) |
| 305 | |
| 306 | // Second restart: the same authority again, now for the restarted |
| 307 | // probe's session and its stamp. Nothing was consumed by the first. |
| 308 | third := d.mustStart(t, c, plan9D4Probe, true) |
| 309 | d.requireRetired(t, second, third, false) |
| 310 | |
| 311 | // The probe passes; the commit keeps the writer session (D1). |
| 312 | d.step(t, "probed", hostclaim.KeepWriterSession("m1", hostclaim.SetMovePhase("m1", hostclaim.KindLanding, hostclaim.PhaseLandingProbing, hostclaim.PhaseLandingProbed))) |
| 313 | d.step(t, "committed", hostclaim.KeepWriterSession("m1", hostclaim.SetMovePhase("m1", hostclaim.KindLanding, hostclaim.PhaseLandingProbed, hostclaim.PhaseLandingCommitted))) |
| 314 | committed := d.f.read() |
| 315 | |
| 316 | // The agent restarts. Its cold wake after the open stops the committed |
| 317 | // probe session in recovery, under the earlier controller's tuple. |
| 318 | after := d.controller("boot-1", "controller-2", 4102, "start-2", true) |
| 319 | wake := d.mustStart(t, after, plan9D4Wake, true) |
| 320 | d.requireRetired(t, committed, wake, true) |
| 321 | if wake.Claim.Op.Phase != hostclaim.PhaseLandingCommitted || wake.Claim.Holder.Session <= committed.Claim.Holder.Session { |
| 322 | t.Fatalf("cold wake: %+v", wake.Claim) |
| 323 | } |
| 324 | // Every predecessor holds root's completed full stop of its own session. |
| 325 | for _, prior := range []hostclaim.Observed{first, second, committed} { |
| 326 | h := prior.Claim.Holder |
| 327 | receipts, err := d.f.guard.CompletedWriterStops(ctx, recoveryHost, hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp}, "m1", "box-1") |
| 328 | if err != nil || len(receipts) != 1 || receipts[0].Target.Session != h.Session || verifyRetirementReceipt(ctx, receipts[0]) != nil { |
| 329 | t.Fatalf("session %d's receipt: %+v %v", h.Session, receipts, err) |
| 330 | } |
| 331 | } |
| 332 | } |
| 333 | |
| 334 | func TestPlan9D4WriterStopIsTheLandingsOwnWriterOnly(t *testing.T) { |
| 335 | d := plan9D4Landing(t) |
| 336 | ctx := t.Context() |
| 337 | c := d.controller("boot-1", "controller-1", 4101, "start-1", true) |
| 338 | |
| 339 | // Before the first probe the writer is the source's session on box-0. |
| 340 | // The writer stop does not apply; main's park-local refusal stands and |
| 341 | // root is never asked. |
| 342 | if err := c.wake.Stop(ctx, d.f.read()); !errors.Is(err, hostbox.ErrChanged) || len(d.f.requests) != 0 { |
| 343 | t.Fatalf("stop of the source's session from the target: %v requests=%d", err, len(d.f.requests)) |
| 344 | } |
| 345 | request := func() StopRequest { |
| 346 | o := d.f.read() |
| 347 | h := o.Claim.Holder |
| 348 | return StopRequest{Region: "ie", Host: recoveryHost, Incarnation: h.Incarnation, InstallationStamp: h.Stamp, WriterSession: h.Session, OperationID: "m1", Phase: o.Claim.Op.Phase, |
| 349 | ObligationID: hostguard.WriterStopObligation(h.Session), Box: "box-1", Enrollment: "enrollment-1", BootID: "boot-1", ControllerID: "controller-1", ExpectedClaimETag: o.ETag, ExpectedDirectoryRevision: "7", RosterRevision: 7} |
| 350 | } |
| 351 | if _, err := d.f.root(recoveryStopDrain{}).FenceAndStop(ctx, request()); !errors.Is(err, ErrClaimStopIdentity) { |
| 352 | t.Fatalf("root stopped the source's session as box-1's writer: %v", err) |
| 353 | } |
| 354 | |
| 355 | d.mustStart(t, c, plan9D4Probe, false) |
| 356 | // With either flag off (no Continue), the local landing writer keeps |
| 357 | // main's refusal: no root request, no fence. |
| 358 | c.wake.Continue = nil |
| 359 | if err := c.wake.Stop(ctx, d.f.read()); !errors.Is(err, hostbox.ErrChanged) || len(d.f.requests) != 0 { |
| 360 | t.Fatalf("flag-off stop: %v requests=%d", err, len(d.f.requests)) |
| 361 | } |
| 362 | c.wake.Continue = hostclaim.SessionContinues |
| 363 | |
| 364 | // Root refuses a writer-session stop for any other session or region. |
| 365 | for name, mutate := range map[string]func(*StopRequest){ |
| 366 | "another session's label": func(r *StopRequest) { r.ObligationID = hostguard.WriterStopObligation(r.WriterSession + 1) }, |
| 367 | "another region": func(r *StopRequest) { r.Region = "uk" }, |
| 368 | } { |
| 369 | r := request() |
| 370 | mutate(&r) |
| 371 | if _, err := d.f.root(recoveryStopDrain{}).FenceAndStop(ctx, r); !errors.Is(err, ErrClaimStopIdentity) { |
| 372 | t.Fatalf("%s: %v", name, err) |
| 373 | } |
| 374 | } |
| 375 | // A chore bearing the writer-session name is never taken for the writer: |
| 376 | // main falls back to the chore, and the shared authority check refuses |
| 377 | // the name before root is asked; root refuses it too. |
| 378 | h := d.f.read().Claim.Holder |
| 379 | label := hostguard.WriterStopObligation(h.Session) |
| 380 | d.step(t, "a chore with the writer's name", hostclaim.QueueLocalPark("m1", "box-1", func() string { return label })) |
| 381 | if err := c.wake.Stop(ctx, d.f.read()); !errors.Is(err, hostbox.ErrChanged) || len(d.f.requests) != 0 { |
| 382 | t.Fatalf("stop beside a chore named %s: %v requests=%d", label, err, len(d.f.requests)) |
| 383 | } |
| 384 | if _, err := d.f.root(recoveryStopDrain{}).FenceAndStop(ctx, request()); !errors.Is(err, ErrClaimStopIdentity) { |
| 385 | t.Fatalf("root stopped beside a chore named %s: %v", label, err) |
| 386 | } |
| 387 | receipts, err := d.f.guard.CompletedWriterStops(ctx, recoveryHost, hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp}, "m1", "box-1") |
| 388 | if err != nil || len(receipts) != 0 { |
| 389 | t.Fatalf("a refused stop left a receipt: %+v %v", receipts, err) |
| 390 | } |
| 391 | } |
| 392 | |
| 393 | func TestPlan9D4RetirementRequiresTheWriterStopReceipt(t *testing.T) { |
| 394 | for name, collect := range map[string]func(*testing.T, *plan9D4, *recoveryController){ |
| 395 | // The predecessor was never stopped. |
| 396 | "no receipt": func(*testing.T, *plan9D4, *recoveryController) {}, |
| 397 | // Root stopped the predecessor under another obligation name: that |
| 398 | // receipt is evidence for that name, not the writer's. |
| 399 | "another scope": func(t *testing.T, d *plan9D4, c *recoveryController) { |
| 400 | o := d.f.read() |
| 401 | h := o.Claim.Holder |
| 402 | drain, err := c.wake.Producers(t.Context(), o) |
| 403 | if err != nil { |
| 404 | t.Fatal(err) |
| 405 | } |
| 406 | r := StopRequest{Region: "ie", Host: recoveryHost, Incarnation: h.Incarnation, InstallationStamp: h.Stamp, WriterSession: h.Session, OperationID: "m1", Phase: o.Claim.Op.Phase, |
| 407 | ObligationID: "probe-chore", Box: "box-1", Enrollment: "enrollment-1", BootID: "boot-1", ControllerID: "controller-1", ExpectedClaimETag: o.ETag, ExpectedDirectoryRevision: "7", RosterRevision: 7} |
| 408 | if _, err := d.f.root(drain).FenceAndStop(t.Context(), r); err != nil { |
| 409 | t.Fatal(err) |
| 410 | } |
| 411 | }, |
| 412 | } { |
| 413 | t.Run(name, func(t *testing.T) { |
| 414 | d := plan9D4Landing(t) |
| 415 | c := d.controller("boot-1", "controller-1", 4101, "start-1", true) |
| 416 | d.mustStart(t, c, plan9D4Probe, false) |
| 417 | collect(t, d, c) |
| 418 | d.permissive = true |
| 419 | if _, err := d.start(t, c, plan9D4Probe, false); !errors.Is(err, hostguard.ErrNoFenceReceipt) { |
| 420 | t.Fatalf("retired without the writer's receipt: %v", err) |
| 421 | } |
| 422 | if got := d.read(t, d.db+".gen"); got != "m1\n" { |
| 423 | t.Fatalf("marker %q", got) |
| 424 | } |
| 425 | if entries, err := os.ReadDir(filepath.Join(d.data, ".parked")); err != nil || len(entries) != 0 { |
| 426 | t.Fatalf("parked without the writer's receipt: %v %v", entries, err) |
| 427 | } |
| 428 | if paths := d.f.read().Claim.Inventory["box-1"].Paths; paths[recoveryHost+".db.gen"] != d.flip || paths[recoveryHost+".db"] != d.flip { |
| 429 | t.Fatalf("inventory changed: %+v", paths) |
| 430 | } |
| 431 | }) |
| 432 | } |
| 433 | } |
| 434 | |
| 435 | func TestPlan9D4RetirementRefusesOutsideTheLandingContract(t *testing.T) { |
| 436 | // A restarted probe admitted after the production stop, with Restore not |
| 437 | // yet run: the hook is called directly with each refused shape. |
| 438 | setup := func(t *testing.T) (*plan9D4, *recoveryController, hostclaim.Observed, claimArtifactRetirement) { |
| 439 | d := plan9D4Landing(t) |
| 440 | c := d.controller("boot-1", "controller-1", 4101, "start-1", true) |
| 441 | d.mustStart(t, c, plan9D4Probe, false) |
| 442 | if err := c.wake.Stop(t.Context(), d.f.read()); err != nil { |
| 443 | t.Fatal(err) |
| 444 | } |
| 445 | o := d.f.read() |
| 446 | in, err := c.wake.Inputs.Select(t.Context(), registry.Route{Host: recoveryHost}, o, plan9D4Probe) |
| 447 | if err != nil { |
| 448 | t.Fatal(err) |
| 449 | } |
| 450 | admitted, err := hostclaim.Update(t.Context(), d.f.claims, o, hostclaim.AdmitWriter(in.Admission)) |
| 451 | if err != nil { |
| 452 | t.Fatal(err) |
| 453 | } |
| 454 | if err := d.f.guard.Activate(t.Context(), recoveryHost, "box-1", nil); err != nil { |
| 455 | t.Fatal(err) |
| 456 | } |
| 457 | return d, c, admitted, claimArtifactRetirement{Local: d.local(), Continue: hostclaim.SessionContinues} |
| 458 | } |
| 459 | untouched := func(t *testing.T, d *plan9D4, name string) { |
| 460 | t.Helper() |
| 461 | if entries, err := os.ReadDir(filepath.Join(d.data, ".parked")); err != nil || len(entries) != 0 { |
| 462 | t.Fatalf("%s parked: %v %v", name, entries, err) |
| 463 | } |
| 464 | if got := d.read(t, d.db+".gen"); got != "m1\n" { |
| 465 | t.Fatalf("%s moved the marker: %q", name, got) |
| 466 | } |
| 467 | } |
| 468 | for name, tc := range map[string]struct { |
| 469 | retire func(claimArtifactRetirement) claimArtifactRetirement |
| 470 | admitted func(hostclaim.Observed) hostclaim.Observed |
| 471 | generation string |
| 472 | mutate func(*testing.T, *plan9D4, *recoveryController) |
| 473 | }{ |
| 474 | "flag off": {retire: func(r claimArtifactRetirement) claimArtifactRetirement { r.Continue = nil; return r }}, |
| 475 | "serving": {admitted: func(o hostclaim.Observed) hostclaim.Observed { |
| 476 | c := *o.Claim |
| 477 | c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box-1"} |
| 478 | o.Claim = &c |
| 479 | return o |
| 480 | }}, |
| 481 | "foreign": {generation: "stranger"}, |
| 482 | "current": {generation: "admitted"}, |
| 483 | "untagged": {mutate: func(t *testing.T, d *plan9D4, _ *recoveryController) { d.retag(t, recoveryHost+".db.gen", 0) }}, |
| 484 | "split": {mutate: func(t *testing.T, d *plan9D4, _ *recoveryController) { d.retag(t, recoveryHost+".db", d.flip+1) }}, |
| 485 | "successor": {mutate: func(t *testing.T, d *plan9D4, c *recoveryController) { d.readmit(t, c) }}, |
| 486 | "other op": {admitted: func(o hostclaim.Observed) hostclaim.Observed { c := *o.Claim; c.Op.ID = "m0"; o.Claim = &c; return o }}, |
| 487 | "other exec": {admitted: func(o hostclaim.Observed) hostclaim.Observed { |
| 488 | c := *o.Claim |
| 489 | c.Op.Exec = "box-0" |
| 490 | o.Claim = &c |
| 491 | return o |
| 492 | }}, |
| 493 | } { |
| 494 | t.Run(name, func(t *testing.T) { |
| 495 | d, c, admitted, r := setup(t) |
| 496 | if tc.retire != nil { |
| 497 | r = tc.retire(r) |
| 498 | } |
| 499 | if tc.mutate != nil { |
| 500 | tc.mutate(t, d, c) |
| 501 | } |
| 502 | if tc.admitted != nil { |
| 503 | admitted = tc.admitted(admitted) |
| 504 | } |
| 505 | generation := "m1" |
| 506 | switch tc.generation { |
| 507 | case "admitted": |
| 508 | generation = admitted.Claim.Holder.Generation |
| 509 | case "": |
| 510 | default: |
| 511 | generation = tc.generation |
| 512 | } |
| 513 | if err := r.Retire(t.Context(), admitted, ClaimLaunchInputs{DBPath: d.db}, generation); !errors.Is(err, hostbox.ErrChanged) { |
| 514 | t.Fatalf("%s: %v", name, err) |
| 515 | } |
| 516 | untouched(t, d, name) |
| 517 | }) |
| 518 | } |
| 519 | // The same admission retires once its shape is right. |
| 520 | d, _, admitted, r := setup(t) |
| 521 | if err := r.Retire(t.Context(), admitted, ClaimLaunchInputs{DBPath: d.db}, "m1"); err != nil { |
| 522 | t.Fatal(err) |
| 523 | } |
| 524 | if _, err := os.Stat(filepath.Join(d.parked(t, fmt.Sprintf("superseded-%d-%d", admitted.Claim.Holder.Session, d.flip)), recoveryHost+".db.gen")); err != nil { |
| 525 | t.Fatal(err) |
| 526 | } |
| 527 | } |
| 528 | |
| 529 | // retag rewrites one inventory tag of box-1 (0 removes it). |
| 530 | func (d *plan9D4) retag(t *testing.T, path string, stamp uint64) { |
| 531 | t.Helper() |
| 532 | d.step(t, "retag "+path, func(c *hostclaim.Claim) error { |
| 533 | entry := c.Inventory["box-1"] |
| 534 | if stamp == 0 { |
| 535 | delete(entry.Paths, path) |
| 536 | } else { |
| 537 | entry.Paths[path] = stamp |
| 538 | } |
| 539 | c.Inventory["box-1"] = entry |
| 540 | return nil |
| 541 | }) |
| 542 | } |
| 543 | |
| 544 | // readmit admits yet another writer, so the admission the hook was called |
| 545 | // for is no longer the claim's. |
| 546 | func (d *plan9D4) readmit(t *testing.T, c *recoveryController) { |
| 547 | t.Helper() |
| 548 | o := d.f.read() |
| 549 | in, err := c.wake.Inputs.Select(t.Context(), registry.Route{Host: recoveryHost}, o, plan9D4Probe) |
| 550 | if err != nil { |
| 551 | t.Fatal(err) |
| 552 | } |
| 553 | d.step(t, "readmit", hostclaim.AdmitWriter(in.Admission)) |
| 554 | } |
| 555 | |
amadan