| 1 | package hostrelocate |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "fmt" |
| 7 | "strconv" |
| 8 | "strings" |
| 9 | "testing" |
| 10 | "time" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/hostclaim" |
| 13 | ) |
| 14 | |
| 15 | // fakeBox is one box's local effects. Its writes to the claim stand in for |
| 16 | // what the real effects leave in storage (P built, a probe admitted). |
| 17 | type fakeBox struct { |
| 18 | f *fixture |
| 19 | box string |
| 20 | certified []bool |
| 21 | // opened counts controllers opened; restarted makes Ready report none |
| 22 | // until the next Acknowledge, as a restarted agent would. |
| 23 | opened int |
| 24 | restarted bool |
| 25 | effects []string |
| 26 | op string |
| 27 | finished bool |
| 28 | // failProbes makes every probe admission fail readiness. |
| 29 | failProbes bool |
| 30 | // retainedErr is a damaged journal entry Retained reports. |
| 31 | retainedErr error |
| 32 | // retired: a scoped cleanup removed the local install; restores, |
| 33 | // reinstalls and probedMissing count what EnsureLocal and Probe saw. |
| 34 | retired bool |
| 35 | restores, reinstalls, probedMissing int |
| 36 | // receiptLost: the box's durable local install receipt is gone. |
| 37 | receiptLost bool |
| 38 | // preflightErr is root refusing the relocating kind, or |
| 39 | // ErrLaterAttemptUnlaunchable (rulings 48, 58). |
| 40 | preflightErr error |
| 41 | // endProbeErr is a stop root keeps refusing (ruling 67). |
| 42 | endProbeErr error |
| 43 | // writesOnInstall: the install writes the claim (the typed inventory), |
| 44 | // as the production install's RecordInventory does (ruling 62). |
| 45 | writesOnInstall bool |
| 46 | // healthy stands for the box's durable attempt journal: the sessions |
| 47 | // whose probe passed and settled (final review A2). passedErr is a |
| 48 | // journal that cannot be read. flaky, when set, is armed by Probe so |
| 49 | // the claim access after it fails once. |
| 50 | healthy map[uint64]bool |
| 51 | passedErr error |
| 52 | flaky *flakyStore |
| 53 | } |
| 54 | |
| 55 | func (b *fakeBox) Acknowledge(_ context.Context, o hostclaim.Observed) (hostclaim.ExecutionAcknowledgement, error) { |
| 56 | b.opened++ |
| 57 | b.restarted, b.finished, b.op = false, false, o.Claim.Op.ID |
| 58 | return hostclaim.ExecutionAcknowledgement{Op: o.Claim.Op.ID, Enrollment: enrollment(b.box), BootID: "boot", ControllerID: "ctl-" + b.box, LeaseETag: "lease", Expires: b.f.now.Add(15 * 60e9)}, nil |
| 59 | } |
| 60 | func (b *fakeBox) Retained() ([]Participation, error) { |
| 61 | if b.opened == 0 || b.finished { |
| 62 | return nil, b.retainedErr |
| 63 | } |
| 64 | return []Participation{{Host: host, Op: b.op}}, b.retainedErr |
| 65 | } |
| 66 | func (b *fakeBox) Finish(ctx context.Context, h, op string) error { |
| 67 | b.finished = true |
| 68 | b.effects = append(b.effects, "finish "+op) |
| 69 | // Production's Finish is the target's hold release, which ends by |
| 70 | // marking the completed relocation discharged (region HA ruling 76). |
| 71 | obs, err := hostclaim.Read(ctx, b.f.s, h) |
| 72 | if err != nil { |
| 73 | return err |
| 74 | } |
| 75 | done, ok, err := hostclaim.ReadCompletedRelocation(obs.Claim) |
| 76 | if err == nil && ok && done.Op == op && done.Stop.Box == b.box && !done.Discharged { |
| 77 | _, err = hostclaim.Update(ctx, b.f.s, obs, hostclaim.MarkCompletedDischarged(op)) |
| 78 | } |
| 79 | return err |
| 80 | } |
| 81 | func (b *fakeBox) Ready(context.Context, hostclaim.Observed) error { |
| 82 | b.effects = append(b.effects, "ready") |
| 83 | if b.restarted || b.opened == 0 { |
| 84 | return ErrNoController |
| 85 | } |
| 86 | return nil |
| 87 | } |
| 88 | func (b *fakeBox) Release(_ context.Context, o hostclaim.Observed) (hostclaim.Release, error) { |
| 89 | h := o.Claim.Holder |
| 90 | return hostclaim.Release{ReleasedBy: b.box, Enrollment: enrollment(b.box), Generation: h.Generation, Session: h.Session, FinalTXID: 40, ManifestSeq: 7, Receipt: "r-" + b.box, ReceiptSHA256: strings.Repeat("d", 64)}, nil |
| 91 | } |
| 92 | func (b *fakeBox) Build(context.Context, hostclaim.Observed) error { |
| 93 | b.effects = append(b.effects, "build") |
| 94 | b.f.participant(func(c *hostclaim.Claim) error { c.Op.Journal["test.built"] = c.Op.Journal["prepared"]; return nil }) |
| 95 | return nil |
| 96 | } |
| 97 | func (b *fakeBox) InstallLocal(_ context.Context, o hostclaim.Observed) (hostclaim.LocalInstall, error) { |
| 98 | b.effects = append(b.effects, "install") |
| 99 | if b.writesOnInstall { |
| 100 | b.f.participant(func(c *hostclaim.Claim) error { c.Op.Journal["test.inventory"] = b.box; return nil }) |
| 101 | } |
| 102 | n, _ := strconv.ParseUint(o.Claim.Op.Journal["install_stamp"], 10, 64) |
| 103 | return hostclaim.LocalInstall{Box: b.box, Enrollment: enrollment(b.box), InstallStamp: n, FilesManifestVersion: 1, JournalSHA256: strings.Repeat("e", 64)}, nil |
| 104 | } |
| 105 | func (b *fakeBox) EnsureLocal(_ context.Context, o hostclaim.Observed) error { |
| 106 | b.effects = append(b.effects, "ensure-local") |
| 107 | if hostclaim.PreparedOpened(o.Claim) { |
| 108 | b.reinstalls++ // never: an opened P is not reinstalled |
| 109 | } |
| 110 | if b.retired { |
| 111 | b.retired, b.restores = false, b.restores+1 |
| 112 | } |
| 113 | return nil |
| 114 | } |
| 115 | func (b *fakeBox) VerifyReceipt(context.Context, hostclaim.Observed) (bool, error) { |
| 116 | b.effects = append(b.effects, "verify-receipt") |
| 117 | return !b.receiptLost, nil |
| 118 | } |
| 119 | func (b *fakeBox) Probe(ctx context.Context, o hostclaim.Observed, certified bool) (uint64, bool, error) { |
| 120 | if b.retired { |
| 121 | b.probedMissing++ // a probe admission over a missing local install |
| 122 | } |
| 123 | b.certified = append(b.certified, certified) |
| 124 | c := o.Claim |
| 125 | next, err := hostclaim.Update(ctx, b.f.s, o, hostclaim.AdmitWriter(hostclaim.Admission{ObservedOpID: c.Op.ID, ObservedPhase: c.Op.Phase, ObservedSession: c.Holder.Session, ObservedGeneration: c.Holder.Generation, |
| 126 | Box: b.box, Mode: "cold", NewGeneration: c.Op.Journal["prepared"], KeepStamp: true, PredecessorCertified: certified, Region: "ie", PredecessorRegion: "ie"})) |
| 127 | if err != nil { |
| 128 | return 0, false, err |
| 129 | } |
| 130 | if b.failProbes { |
| 131 | return next.Claim.Holder.Session, false, nil |
| 132 | } |
| 133 | if b.healthy == nil { |
| 134 | b.healthy = map[uint64]bool{} |
| 135 | } |
| 136 | b.healthy[next.Claim.Holder.Session] = true |
| 137 | if b.flaky != nil { |
| 138 | b.flaky.armed = true |
| 139 | } |
| 140 | return next.Claim.Holder.Session, true, nil |
| 141 | } |
| 142 | func (b *fakeBox) PassedProbe(_ context.Context, o hostclaim.Observed) (uint64, bool, error) { |
| 143 | b.effects = append(b.effects, "passed-probe") |
| 144 | if b.passedErr != nil { |
| 145 | return 0, false, b.passedErr |
| 146 | } |
| 147 | s := o.Claim.Holder.Session |
| 148 | return s, b.healthy[s], nil |
| 149 | } |
| 150 | |
| 151 | func (f *fixture) clock() time.Time { return f.now } |
| 152 | |
| 153 | // run steps the worker and both participants until the host serves again. |
| 154 | func run(t *testing.T, f *fixture, parts ...Participant) hostclaim.Observed { |
| 155 | t.Helper() |
| 156 | ctx := context.Background() |
| 157 | for i := 0; i < 60; i++ { |
| 158 | if o := f.claim(); o.Claim.Op.Kind == hostclaim.KindServing { |
| 159 | return o |
| 160 | } |
| 161 | if _, err := f.w.Step(ctx, host); err != nil && !errors.Is(err, ErrNotYet) { |
| 162 | t.Fatalf("worker: %v", err) |
| 163 | } |
| 164 | for _, p := range parts { |
| 165 | if _, err := p.Step(ctx, host); err != nil && !errors.Is(err, ErrNotYet) { |
| 166 | t.Fatalf("%s: %v", p.Box, err) |
| 167 | } |
| 168 | } |
| 169 | } |
| 170 | t.Fatalf("no progress: %+v", f.claim().Claim.Op) |
| 171 | return hostclaim.Observed{} |
| 172 | } |
| 173 | |
| 174 | func TestADrainCompletesWithBothParticipants(t *testing.T) { |
| 175 | f := newFixture(t) |
| 176 | src, dst := &fakeBox{f: f, box: "src"}, &fakeBox{f: f, box: "dst"} |
| 177 | if _, err := f.w.Start(context.Background(), host, Request{Reason: hostclaim.ReasonDrain, Source: "src", Target: "dst"}); err != nil { |
| 178 | t.Fatal(err) |
| 179 | } |
| 180 | o := run(t, f, Participant{Claims: f.s, Box: "src", Enrollment: enrollment("src"), Effects: src, Now: f.clock}, Participant{Claims: f.s, Box: "dst", Enrollment: enrollment("dst"), Effects: dst, Now: f.clock}) |
| 181 | if o.Claim.Op.Exec != "dst" || len(dst.certified) != 1 || !dst.certified[0] { |
| 182 | t.Fatalf("exec %s certified %v", o.Claim.Op.Exec, dst.certified) |
| 183 | } |
| 184 | last := o.Claim.Holder.Admissions[len(o.Claim.Holder.Admissions)-1] |
| 185 | if last.WritesLost != nil { |
| 186 | t.Fatalf("a drain declared a loss: %+v", last) |
| 187 | } |
| 188 | } |
| 189 | |
| 190 | func TestAFailoverCompletesAndDeclaresTheSource(t *testing.T) { |
| 191 | f := newFixture(t) |
| 192 | f.terminate("src") |
| 193 | dst := &fakeBox{f: f, box: "dst"} |
| 194 | if _, err := f.w.Start(context.Background(), host, Request{Reason: hostclaim.ReasonFailover, Source: "src", Target: "dst"}); err != nil { |
| 195 | t.Fatal(err) |
| 196 | } |
| 197 | o := run(t, f, Participant{Claims: f.s, Box: "dst", Enrollment: enrollment("dst"), Effects: dst, Now: f.clock}) |
| 198 | last := o.Claim.Holder.Admissions[len(o.Claim.Holder.Admissions)-1] |
| 199 | if dst.certified[0] || last.WritesLost == nil || last.WritesLost.Box != "src" { |
| 200 | t.Fatalf("certified %v last %+v", dst.certified, last) |
| 201 | } |
| 202 | } |
| 203 | |
| 204 | func TestALostParticipantDoesNothing(t *testing.T) { |
| 205 | f := newFixture(t) |
| 206 | f.terminate("src") |
| 207 | if _, err := f.w.Start(context.Background(), host, Request{Reason: hostclaim.ReasonFailover, Source: "src", Target: "dst"}); err != nil { |
| 208 | t.Fatal(err) |
| 209 | } |
| 210 | src := &fakeBox{f: f, box: "src"} |
| 211 | before := f.claim().ETag |
| 212 | if _, err := (Participant{Claims: f.s, Box: "src", Enrollment: enrollment("src"), Effects: src, Now: f.clock}).Step(context.Background(), host); !errors.Is(err, ErrNotYet) || f.claim().ETag != before { |
| 213 | t.Fatalf("a fenced source acted: %v", err) |
| 214 | } |
| 215 | } |
| 216 | |
| 217 | func TestAParticipantChecksItsControllerBeforeEveryEffectAndReopensAfterARestart(t *testing.T) { |
| 218 | f := newFixture(t) |
| 219 | ctx := context.Background() |
| 220 | src, dst := &fakeBox{f: f, box: "src"}, &fakeBox{f: f, box: "dst"} |
| 221 | if _, err := f.w.Start(ctx, host, Request{Reason: hostclaim.ReasonDrain, Source: "src", Target: "dst"}); err != nil { |
| 222 | t.Fatal(err) |
| 223 | } |
| 224 | ps := Participant{Claims: f.s, Box: "src", Enrollment: enrollment("src"), Effects: src, Now: f.clock} |
| 225 | pd := Participant{Claims: f.s, Box: "dst", Enrollment: enrollment("dst"), Effects: dst, Now: f.clock} |
| 226 | for _, p := range []Participant{ps, pd} { |
| 227 | if _, err := p.Step(ctx, host); err != nil { |
| 228 | t.Fatal(err) |
| 229 | } |
| 230 | } |
| 231 | f.worker(nil) // releasing |
| 232 | // The target's agent restarts: its next step opens a controller and |
| 233 | // journals nothing; the one after acts. |
| 234 | dst.restarted = true |
| 235 | before := f.claim().ETag |
| 236 | if _, err := pd.Step(ctx, host); !errors.Is(err, ErrNotYet) || dst.opened != 2 || f.claim().ETag != before { |
| 237 | t.Fatalf("restart: %v opened=%d wrote=%v", err, dst.opened, f.claim().ETag != before) |
| 238 | } |
| 239 | if n := len(f.claim().Claim.Op.Journal["exec_ack.dst"]); n == 0 { |
| 240 | t.Fatal("the first acknowledgement was dropped") |
| 241 | } |
| 242 | // The source releases: Ready ran before the effect. |
| 243 | src.effects = nil |
| 244 | if _, err := ps.Step(ctx, host); err != nil { |
| 245 | t.Fatal(err) |
| 246 | } |
| 247 | if len(src.effects) == 0 || src.effects[0] != "ready" { |
| 248 | t.Fatalf("release ran without checking the controller: %v", src.effects) |
| 249 | } |
| 250 | // A lost controller refuses the effect and writes nothing. |
| 251 | o := run(t, f, ps, pd) |
| 252 | if o.Claim.Op.Exec != "dst" { |
| 253 | t.Fatalf("exec %s", o.Claim.Op.Exec) |
| 254 | } |
| 255 | for i, e := range dst.effects { |
| 256 | if e == "build" && (i == 0 || dst.effects[i-1] != "ready") { |
| 257 | t.Fatalf("build without a readiness check first: %v", dst.effects) |
| 258 | } |
| 259 | } |
| 260 | } |
| 261 | |
| 262 | func TestAParticipantGivesUpAfterThreeProbeAttempts(t *testing.T) { |
| 263 | f := newFixture(t) |
| 264 | ctx := context.Background() |
| 265 | src, dst := &fakeBox{f: f, box: "src"}, &fakeBox{f: f, box: "dst", failProbes: true} |
| 266 | if _, err := f.w.Start(ctx, host, Request{Reason: hostclaim.ReasonDrain, Source: "src", Target: "dst"}); err != nil { |
| 267 | t.Fatal(err) |
| 268 | } |
| 269 | ps := Participant{Claims: f.s, Box: "src", Enrollment: enrollment("src"), Effects: src, Now: f.clock} |
| 270 | pd := Participant{Claims: f.s, Box: "dst", Enrollment: enrollment("dst"), Effects: dst, Now: f.clock} |
| 271 | for i := 0; i < 40; i++ { |
| 272 | o := f.claim() |
| 273 | if o.Claim.Op.Phase == hostclaim.PhaseProbeFailed { |
| 274 | if n := len(dst.certified); n != hostclaim.MaxProbeAttempts { |
| 275 | t.Fatalf("%d probe admissions, want %d", n, hostclaim.MaxProbeAttempts) |
| 276 | } |
| 277 | if journal := o.Claim.Op.Journal; journal["probe_attempts.dst"] != "3" { |
| 278 | t.Fatalf("attempts journaled: %v", journal["probe_attempts.dst"]) |
| 279 | } |
| 280 | return |
| 281 | } |
| 282 | f.w.Step(ctx, host) |
| 283 | ps.Step(ctx, host) |
| 284 | pd.Step(ctx, host) |
| 285 | } |
| 286 | t.Fatalf("never probe-failed: %+v", f.claim().Claim.Op) |
| 287 | } |
| 288 | |
| 289 | func TestAParticipantFinishesItsControllerOnceTheOperationIsOver(t *testing.T) { |
| 290 | f := newFixture(t) |
| 291 | ctx := context.Background() |
| 292 | src, dst := &fakeBox{f: f, box: "src"}, &fakeBox{f: f, box: "dst"} |
| 293 | if _, err := f.w.Start(ctx, host, Request{Reason: hostclaim.ReasonDrain, Source: "src", Target: "dst"}); err != nil { |
| 294 | t.Fatal(err) |
| 295 | } |
| 296 | ps := Participant{Claims: f.s, Box: "src", Enrollment: enrollment("src"), Effects: src, Now: f.clock} |
| 297 | pd := Participant{Claims: f.s, Box: "dst", Enrollment: enrollment("dst"), Effects: dst, Now: f.clock} |
| 298 | run(t, f, ps, pd) |
| 299 | if err := pd.Sweep(ctx); err != nil || !dst.finished { |
| 300 | t.Fatalf("the serving target kept its relocation controller: %v finished=%v", err, dst.finished) |
| 301 | } |
| 302 | // A damaged journal entry beside it is reported, and the readable |
| 303 | // participation is still finished. |
| 304 | src.retainedErr = errors.New("a damaged journal entry") |
| 305 | if err := ps.Sweep(ctx); !errors.Is(err, src.retainedErr) || !src.finished { |
| 306 | t.Fatalf("the former source kept its controller, or the damaged entry was hidden: %v finished=%v", err, src.finished) |
| 307 | } |
| 308 | src.retainedErr = nil |
| 309 | // A second relocation opens a fresh controller. |
| 310 | spa := &fakeBox{f: f, box: "spa"} |
| 311 | if _, err := f.w.Start(ctx, host, Request{Reason: hostclaim.ReasonDrain, Source: "dst", Target: "spa"}); err != nil { |
| 312 | t.Fatal(err) |
| 313 | } |
| 314 | before := dst.opened |
| 315 | run(t, f, pd, Participant{Claims: f.s, Box: "spa", Enrollment: enrollment("spa"), Effects: spa, Now: f.clock}) |
| 316 | if dst.opened != before+1 { |
| 317 | t.Fatalf("the second relocation did not open its own controller: %d -> %d", before, dst.opened) |
| 318 | } |
| 319 | // No third relocation starts from spa until spa has finished the |
| 320 | // second one (region HA ruling 77); then, mid-operation, Sweep keeps |
| 321 | // the controller. |
| 322 | pspa := Participant{Claims: f.s, Box: "spa", Enrollment: enrollment("spa"), Effects: spa, Now: f.clock} |
| 323 | if _, err := f.w.Start(ctx, host, Request{Reason: hostclaim.ReasonDrain, Source: "spa", Target: "sp2"}); !errors.Is(err, hostclaim.ErrPendingChore) { |
| 324 | t.Fatalf("a successor started before spa finished: %v", err) |
| 325 | } |
| 326 | if err := pspa.Sweep(ctx); err != nil || !spa.finished { |
| 327 | t.Fatalf("spa kept the second relocation's controller: %v", err) |
| 328 | } |
| 329 | if _, err := f.w.Start(ctx, host, Request{Reason: hostclaim.ReasonDrain, Source: "spa", Target: "sp2"}); err != nil { |
| 330 | t.Fatal(err) |
| 331 | } |
| 332 | spa.finished = false |
| 333 | if _, err := pspa.Step(ctx, host); err != nil { |
| 334 | t.Fatal(err) |
| 335 | } |
| 336 | if err := pspa.Sweep(ctx); err != nil || spa.finished { |
| 337 | t.Fatalf("a running operation's controller was finished: %v", err) |
| 338 | } |
| 339 | } |
| 340 | |
| 341 | // Round 14 finding 2: the local install is journaled, the lease is lost, |
| 342 | // and the scoped cleanup retires the unopened P's local install. Before |
| 343 | // probing, and before P's first admission, the participant has the |
| 344 | // install verified and restored; once P is opened it is never touched. |
| 345 | func TestAParticipantRestoresARetiredLocalInstallBeforeTheFirstAdmission(t *testing.T) { |
| 346 | f := newFixture(t) |
| 347 | ctx := context.Background() |
| 348 | src, dst := &fakeBox{f: f, box: "src"}, &fakeBox{f: f, box: "dst"} |
| 349 | if _, err := f.w.Start(ctx, host, Request{Reason: hostclaim.ReasonDrain, Source: "src", Target: "dst"}); err != nil { |
| 350 | t.Fatal(err) |
| 351 | } |
| 352 | ps := Participant{Claims: f.s, Box: "src", Enrollment: enrollment("src"), Effects: src, Now: f.clock} |
| 353 | pd := Participant{Claims: f.s, Box: "dst", Enrollment: enrollment("dst"), Effects: dst, Now: f.clock} |
| 354 | for i := 0; i < 40 && f.claim().Claim.Op.Journal["local_install"] == ""; i++ { |
| 355 | _, _ = f.w.Step(ctx, host) |
| 356 | ps.Step(ctx, host) |
| 357 | pd.Step(ctx, host) |
| 358 | } |
| 359 | if f.claim().Claim.Op.Journal["local_install"] == "" { |
| 360 | t.Fatalf("never installed locally: %+v", f.claim().Claim.Op) |
| 361 | } |
| 362 | dst.retired = true // the lease was lost; the scoped cleanup retired it |
| 363 | run(t, f, ps, pd) |
| 364 | if dst.restores != 1 || dst.probedMissing != 0 || dst.reinstalls != 0 { |
| 365 | t.Fatalf("restores %d, probes over a missing install %d, reinstalls after P %d", dst.restores, dst.probedMissing, dst.reinstalls) |
| 366 | } |
| 367 | } |
| 368 | |
| 369 | // Round 15 finding 1: after P's first admission the local install is |
| 370 | // never remade, and a box that has lost its receipt cannot rebuild it from |
| 371 | // P: the next admission is refused and the operation is stuck (§5.8, |
| 372 | // B2-5), across a restart. With the receipt intact, the later attempt goes |
| 373 | // ahead and nothing is reinstalled or reverted: the files, env and timers |
| 374 | // the admitted writers changed are theirs. |
| 375 | func TestAParticipantIsStuckWhenItsReceiptIsLostAfterPOpens(t *testing.T) { |
| 376 | for _, lost := range []bool{true, false} { |
| 377 | t.Run(fmt.Sprintf("receipt lost %v", lost), func(t *testing.T) { |
| 378 | f := newFixture(t) |
| 379 | ctx := context.Background() |
| 380 | src, dst := &fakeBox{f: f, box: "src"}, &fakeBox{f: f, box: "dst", failProbes: true} |
| 381 | if _, err := f.w.Start(ctx, host, Request{Reason: hostclaim.ReasonDrain, Source: "src", Target: "dst"}); err != nil { |
| 382 | t.Fatal(err) |
| 383 | } |
| 384 | ps := Participant{Claims: f.s, Box: "src", Enrollment: enrollment("src"), Effects: src, Now: f.clock} |
| 385 | pd := Participant{Claims: f.s, Box: "dst", Enrollment: enrollment("dst"), Effects: dst, Now: f.clock} |
| 386 | for i := 0; i < 60 && len(dst.certified) == 0; i++ { |
| 387 | _, _ = f.w.Step(ctx, host) |
| 388 | ps.Step(ctx, host) |
| 389 | pd.Step(ctx, host) |
| 390 | } |
| 391 | if len(dst.certified) != 1 || !hostclaim.PreparedOpened(f.claim().Claim) { |
| 392 | t.Fatalf("P was not opened: %d admissions", len(dst.certified)) |
| 393 | } |
| 394 | dst.receiptLost, dst.restarted = lost, true // and the agent restarts |
| 395 | dst.effects = nil |
| 396 | for i := 0; i < 6; i++ { |
| 397 | pd.Step(ctx, host) |
| 398 | } |
| 399 | c := f.claim().Claim |
| 400 | for _, e := range dst.effects { |
| 401 | if e == "ensure-local" || e == "install" { |
| 402 | t.Fatalf("the local install was touched after P opened: %v", dst.effects) |
| 403 | } |
| 404 | } |
| 405 | if lost { |
| 406 | if c.Op.Journal["stuck"] == "" || len(dst.certified) != 1 { |
| 407 | t.Fatalf("a lost receipt after P opened: stuck %q, admissions %d", c.Op.Journal["stuck"], len(dst.certified)) |
| 408 | } |
| 409 | return |
| 410 | } |
| 411 | if c.Op.Journal["stuck"] != "" || len(dst.certified) < 2 { |
| 412 | t.Fatalf("an intact receipt: stuck %q, admissions %d", c.Op.Journal["stuck"], len(dst.certified)) |
| 413 | } |
| 414 | }) |
| 415 | } |
| 416 | } |
| 417 | |
| 418 | // The scoped cleanup can also retire the unopened P's local install after |
| 419 | // the target has entered probing and before P's first admission: the |
| 420 | // probing step restores it before it counts or makes an attempt. |
| 421 | func TestAParticipantRestoresALocalInstallRetiredAfterItEntersProbing(t *testing.T) { |
| 422 | f := newFixture(t) |
| 423 | ctx := context.Background() |
| 424 | src, dst := &fakeBox{f: f, box: "src"}, &fakeBox{f: f, box: "dst"} |
| 425 | if _, err := f.w.Start(ctx, host, Request{Reason: hostclaim.ReasonDrain, Source: "src", Target: "dst"}); err != nil { |
| 426 | t.Fatal(err) |
| 427 | } |
| 428 | ps := Participant{Claims: f.s, Box: "src", Enrollment: enrollment("src"), Effects: src, Now: f.clock} |
| 429 | pd := Participant{Claims: f.s, Box: "dst", Enrollment: enrollment("dst"), Effects: dst, Now: f.clock} |
| 430 | for i := 0; i < 40 && f.claim().Claim.Op.Phase != hostclaim.PhaseRelocatingProbing; i++ { |
| 431 | _, _ = f.w.Step(ctx, host) |
| 432 | ps.Step(ctx, host) |
| 433 | pd.Step(ctx, host) |
| 434 | } |
| 435 | if c := f.claim().Claim; c.Op.Phase != hostclaim.PhaseRelocatingProbing || hostclaim.PreparedOpened(c) || len(dst.certified) != 0 { |
| 436 | t.Fatalf("not probing before P's first admission: %s, %d admissions", c.Op.Phase, len(dst.certified)) |
| 437 | } |
| 438 | dst.retired = true // the lease was lost after probing began |
| 439 | run(t, f, ps, pd) |
| 440 | if dst.restores != 1 || dst.probedMissing != 0 || dst.reinstalls != 0 { |
| 441 | t.Fatalf("restores %d, probes over a missing install %d, reinstalls after P %d", dst.restores, dst.probedMissing, dst.reinstalls) |
| 442 | } |
| 443 | } |
| 444 | |
| 445 | func (b *fakeBox) Preflight(context.Context, hostclaim.Observed) error { |
| 446 | b.effects = append(b.effects, "preflight") |
| 447 | return b.preflightErr |
| 448 | } |
| 449 | |
| 450 | func (b *fakeBox) EndProbe(_ context.Context, o hostclaim.Observed) (hostclaim.ProbeStop, error) { |
| 451 | b.effects = append(b.effects, "end-probe") |
| 452 | if b.endProbeErr != nil { |
| 453 | return hostclaim.ProbeStop{}, b.endProbeErr |
| 454 | } |
| 455 | return hostclaim.ProbeStop{Box: b.box, Session: o.Claim.Holder.Session, Receipt: "fence-" + b.box, ReceiptSHA256: strings.Repeat("a", 64)}, nil |
| 456 | } |
| 457 | |
amadan