| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "os" |
| 8 | "path/filepath" |
| 9 | "reflect" |
| 10 | "strconv" |
| 11 | "strings" |
| 12 | "testing" |
| 13 | "time" |
| 14 | |
| 15 | "github.com/carlosframework/platform/internal/activator" |
| 16 | "github.com/carlosframework/platform/internal/hostclaim" |
| 17 | "github.com/carlosframework/platform/internal/moves" |
| 18 | "github.com/carlosframework/platform/internal/moves/archive" |
| 19 | "github.com/carlosframework/platform/internal/provision" |
| 20 | ) |
| 21 | |
| 22 | // probeLog is a fake probe wake. Each call admits a writer on the claim as |
| 23 | // the claim's runtime would (a cold admission into the move id, or a fresh |
| 24 | // generation once the move id is admitted), journals the session's binding |
| 25 | // to the record's attempt as Task 5's attempts do (and, when openWindow is |
| 26 | // set, the attempt's opened time), then answers from its script. A passing |
| 27 | // call leaves a live entry for the admitted session. |
| 28 | type probeLog struct { |
| 29 | t *testing.T |
| 30 | h *landingHarness |
| 31 | configs []activator.ProbeConfig |
| 32 | answers []error |
| 33 | before func(n int) |
| 34 | admissions []probeAdmission |
| 35 | openWindow bool |
| 36 | // noBind leaves the admission without Task 5's binding: a crash |
| 37 | // between the admission CAS and Reserve(0). |
| 38 | noBind bool |
| 39 | last activator.ProbeResult |
| 40 | live bool |
| 41 | } |
| 42 | |
| 43 | // probeAdmission is one admission the fake made: the record attempt it was |
| 44 | // made under and the session it admitted. |
| 45 | type probeAdmission struct { |
| 46 | attempt int |
| 47 | session uint64 |
| 48 | } |
| 49 | |
| 50 | func (p *probeLog) wake(ctx context.Context, cfg activator.ProbeConfig) (activator.ProbeResult, error) { |
| 51 | p.configs = append(p.configs, cfg) |
| 52 | n := len(p.configs) |
| 53 | if p.before != nil { |
| 54 | p.before(n) |
| 55 | } |
| 56 | m := p.h.read(p.t, p1Host).Move |
| 57 | k := m.Landing.Attempts |
| 58 | obs, err := hostclaim.Read(ctx, p.h.s, p1Host) |
| 59 | if err != nil { |
| 60 | return activator.ProbeResult{}, err |
| 61 | } |
| 62 | h := obs.Claim.Holder |
| 63 | gen := p1ID |
| 64 | if h.WriterBox == p.h.dstBox { |
| 65 | gen = "fresh" + strconv.Itoa(n) |
| 66 | } |
| 67 | // The predecessor's region is the probe owners' own rule: the last |
| 68 | // admission's region, or the move's source's engine region. |
| 69 | from, err := predecessorRegion(h, m.From) |
| 70 | if err != nil { |
| 71 | return activator.ProbeResult{}, err |
| 72 | } |
| 73 | admitted, err := hostclaim.Update(ctx, p.h.s, obs, hostclaim.AdmitWriter(hostclaim.Admission{ObservedOpID: p1ID, ObservedPhase: provision.LandProbing, |
| 74 | ObservedSession: h.Session, ObservedGeneration: h.Generation, Box: p.h.dstBox, Mode: "cold", NewGeneration: gen, KeepStamp: h.Stamp > h.Session, Region: p.h.dstEngine, PredecessorRegion: from})) |
| 75 | if err != nil { |
| 76 | return activator.ProbeResult{}, err |
| 77 | } |
| 78 | ah := admitted.Claim.Holder |
| 79 | p.admissions = append(p.admissions, probeAdmission{attempt: k, session: ah.Session}) |
| 80 | j, err := moves.ReadJournal(p.h.tdir, p1Host, p1ID) |
| 81 | if err != nil { |
| 82 | return activator.ProbeResult{}, err |
| 83 | } |
| 84 | if !p.noBind { |
| 85 | j.Mark(probeStepSession+strconv.FormatUint(ah.Session, 10), strconv.Itoa(k)) |
| 86 | } |
| 87 | if p.openWindow { |
| 88 | j.Mark(probeStepOpened+strconv.Itoa(k), p.h.clock.Now().UTC().Format(time.RFC3339Nano)) |
| 89 | } |
| 90 | if err := j.Commit(p.h.tdir); err != nil { |
| 91 | return activator.ProbeResult{}, err |
| 92 | } |
| 93 | p.last = activator.ProbeResult{Session: ah.Session, Stamp: ah.Stamp, Generation: ah.Generation} |
| 94 | if n <= len(p.answers) && p.answers[n-1] != nil { |
| 95 | p.live = false |
| 96 | return activator.ProbeResult{}, p.answers[n-1] |
| 97 | } |
| 98 | p.live = true |
| 99 | return p.last, nil |
| 100 | } |
| 101 | |
| 102 | // liveProbe is the activator's LiveProbe over the fake's last passing call. |
| 103 | func (p *probeLog) liveProbe(host string) (string, activator.ProbeResult, bool) { |
| 104 | if host != p1Host || !p.live { |
| 105 | return "", activator.ProbeResult{}, false |
| 106 | } |
| 107 | return p1ID, p.last, true |
| 108 | } |
| 109 | |
| 110 | func failedAttempt(why string) error { |
| 111 | return errors.Join(activator.ErrProbeFailed, errors.New(why)) |
| 112 | } |
| 113 | |
| 114 | // installedLanding is a landing at restored, with the cut's admitted probe |
| 115 | // attempts set to attempts (0 clears them) through the record's writer. |
| 116 | func installedLanding(t *testing.T, attempts int) (*landingHarness, *probeLog) { |
| 117 | t.Helper() |
| 118 | h := newLandingHarness(t) |
| 119 | h.flip(t) |
| 120 | h.land(t) |
| 121 | h.installed(t) |
| 122 | if _, err := provision.SetRecordMove(context.Background(), h.s, "bab", "app", p1Host, func(rec *provision.Record) error { |
| 123 | rec.Move.CutAttempts = attempts |
| 124 | return nil |
| 125 | }); err != nil { |
| 126 | t.Fatal(err) |
| 127 | } |
| 128 | p := &probeLog{t: t, h: h} |
| 129 | h.probe, h.probeLive = p.wake, p.liveProbe |
| 130 | h.holdProbed = true |
| 131 | return h, p |
| 132 | } |
| 133 | |
| 134 | func (h *landingHarness) atPhase(t *testing.T, phase string) *provision.MoveState { |
| 135 | t.Helper() |
| 136 | m := h.read(t, p1Host).Move |
| 137 | if m.State != provision.MoveLanding || m.LandingPhase != phase || m.Reason != "" { |
| 138 | t.Fatalf("the record is not landing/%s: %+v", phase, m) |
| 139 | } |
| 140 | if c := h.claim(t); c.Op.Kind != hostclaim.KindLanding || c.Op.Phase != phase { |
| 141 | t.Fatalf("the claim is not landing/%s: %+v", phase, c.Op) |
| 142 | } |
| 143 | return m |
| 144 | } |
| 145 | |
| 146 | // TestLandingProbesWithinTheBudget (brief): two admitted attempts, the |
| 147 | // first fails and the second passes, so the landing is probed, record and |
| 148 | // claim; a third is never attempted. Each attempt is counted on the record |
| 149 | // and journalled before its call, and the probe is given the landing's |
| 150 | // deadline (H8), the build's manifest and the pins. |
| 151 | func TestLandingProbesWithinTheBudget(t *testing.T) { |
| 152 | t.Parallel() |
| 153 | h, p := installedLanding(t, 2) |
| 154 | p.answers = []error{failedAttempt("not ready")} |
| 155 | h.land(t) |
| 156 | m := h.atPhase(t, provision.LandProbed) |
| 157 | if len(p.configs) != 2 || m.Landing == nil || m.Landing.Attempts != 2 || h.landJournal(t).Attempt != 2 { |
| 158 | t.Fatalf("attempts: calls %d record %+v journal %d", len(p.configs), m.Landing, h.landJournal(t).Attempt) |
| 159 | } |
| 160 | deadline, _ := landingDeadlineOf(m) |
| 161 | want := activator.ProbeConfig{Host: p1Host, Op: p1ID, HoldOp: p1ID, Deadline: *deadline, Prepared: h.preparedManifest(t), Inputs: m.Inputs} |
| 162 | for i, got := range p.configs { |
| 163 | if !reflect.DeepEqual(got, want) { |
| 164 | t.Fatalf("probe %d was given %+v, want %+v", i+1, got, want) |
| 165 | } |
| 166 | } |
| 167 | // The passing attempt's admitted writer (fix round 1, Minor): its |
| 168 | // session, stamp and generation survive the probed mirroring. |
| 169 | if c := h.claim(t); len(p.admissions) != 2 || c.Holder.Session != p.last.Session || c.Holder.Stamp != p.last.Stamp || c.Holder.Generation != p.last.Generation || c.Holder.WriterBox != "uk-box" { |
| 170 | t.Fatalf("the probed step did not keep the admitted writer %+v: holder %+v", p.last, c.Holder) |
| 171 | } |
| 172 | // probed is the probe's end: another pass probes nothing. |
| 173 | h.land(t) |
| 174 | if len(p.configs) != 2 { |
| 175 | t.Fatalf("a pass at probed probed again: %d", len(p.configs)) |
| 176 | } |
| 177 | h.atPhase(t, provision.LandProbed) |
| 178 | } |
| 179 | |
| 180 | // preparedManifest is the build's install manifest from the op journal. |
| 181 | func (h *landingHarness) preparedManifest(t *testing.T) *moves.InstallManifest { |
| 182 | t.Helper() |
| 183 | m := h.installedManifest(t) |
| 184 | return &m |
| 185 | } |
| 186 | |
| 187 | func (h *landingHarness) installedManifest(t *testing.T) moves.InstallManifest { |
| 188 | t.Helper() |
| 189 | j := h.landJournal(t) |
| 190 | var m moves.InstallManifest |
| 191 | if err := json.Unmarshal([]byte(j.Steps[landStepInstall]), &m); err != nil { |
| 192 | t.Fatal(err) |
| 193 | } |
| 194 | return m |
| 195 | } |
| 196 | |
| 197 | // TestLandingExhaustedAttemptsMarkRollback (brief): every admitted attempt |
| 198 | // fails: the landing records the attempts-exhausted rollback sentence, |
| 199 | // which the next pass's rollback decision turns into the rollback. |
| 200 | func TestLandingExhaustedAttemptsMarkRollback(t *testing.T) { |
| 201 | t.Parallel() |
| 202 | h, p := installedLanding(t, 2) |
| 203 | p.answers = []error{failedAttempt("one"), errors.Join(activator.ErrProbeFailed, activator.ErrAttemptOver)} |
| 204 | h.land(t) |
| 205 | if m := h.read(t, p1Host).Move; m.Reason != provision.RollbackProbeAttempts || len(p.configs) != 2 { |
| 206 | t.Fatalf("exhausted: %d calls, %+v", len(p.configs), m) |
| 207 | } |
| 208 | h.land(t) |
| 209 | rolledBack(t, h, provision.RollbackProbeAttempts, provision.LandProbing) |
| 210 | if len(p.configs) != 2 { |
| 211 | t.Fatalf("a rolled-back landing probed again: %d", len(p.configs)) |
| 212 | } |
| 213 | } |
| 214 | |
| 215 | // A changed configuration found by the probe (Task 5's ProbeInputsChanged, |
| 216 | // before or after the admission) is the configuration rollback, at once. |
| 217 | func TestLandingConfigChangedDuringTheProbeRollsBack(t *testing.T) { |
| 218 | t.Parallel() |
| 219 | for name, answer := range map[string]error{ |
| 220 | "before the admission": &ProbeInputsChanged{Reason: provision.RollbackConfigChanged}, |
| 221 | "after the admission": errors.Join(activator.ErrProbeFailed, &ProbeInputsChanged{Reason: provision.RollbackConfigChanged}), |
| 222 | } { |
| 223 | t.Run(name, func(t *testing.T) { |
| 224 | h, p := installedLanding(t, 3) |
| 225 | p.answers = []error{answer} |
| 226 | h.land(t) |
| 227 | if m := h.read(t, p1Host).Move; m.Reason != provision.RollbackConfigChanged || len(p.configs) != 1 { |
| 228 | t.Fatalf("%d calls, %+v", len(p.configs), m) |
| 229 | } |
| 230 | h.land(t) |
| 231 | rolledBack(t, h, provision.RollbackConfigChanged, provision.LandProbing) |
| 232 | }) |
| 233 | } |
| 234 | } |
| 235 | |
| 236 | // H10: a cut that recorded no probe attempts refuses the probe with its |
| 237 | // recorded sentence; nothing is probed. |
| 238 | func TestLandingProbeNeedsTheCutsAttempts(t *testing.T) { |
| 239 | t.Parallel() |
| 240 | h, p := installedLanding(t, 0) |
| 241 | h.land(t) |
| 242 | if m := h.read(t, p1Host).Move; m.Reason != provision.RollbackProbeUnpriced || len(p.configs) != 0 { |
| 243 | t.Fatalf("%d calls, %+v", len(p.configs), m) |
| 244 | } |
| 245 | h.land(t) |
| 246 | rolledBack(t, h, provision.RollbackProbeUnpriced, provision.LandProbing) |
| 247 | } |
| 248 | |
| 249 | // H8: probing and probed check the landing's deadline at entry. A deadline |
| 250 | // that passes during an attempt ends the probing with the deadline's |
| 251 | // sentence, and no further attempt is made; a probed landing past its |
| 252 | // deadline rolls back. |
| 253 | func TestLandingProbingChecksTheDeadline(t *testing.T) { |
| 254 | t.Parallel() |
| 255 | h, p := installedLanding(t, 3) |
| 256 | p.before = func(int) { h.clock.Advance(cutLandingWindow + time.Minute) } |
| 257 | p.answers = []error{errors.Join(activator.ErrProbeFailed, activator.ErrAttemptOver)} |
| 258 | h.land(t) |
| 259 | if len(p.configs) != 1 { |
| 260 | t.Fatalf("probed past the deadline: %d calls", len(p.configs)) |
| 261 | } |
| 262 | h.land(t) |
| 263 | rolledBack(t, h, provision.RollbackLandingDeadline, provision.LandProbing) |
| 264 | |
| 265 | h2, _ := installedLanding(t, 3) |
| 266 | h2.land(t) |
| 267 | h2.atPhase(t, provision.LandProbed) |
| 268 | h2.clock.Advance(cutLandingWindow + time.Minute) |
| 269 | h2.land(t) |
| 270 | rolledBack(t, h2, provision.RollbackLandingDeadline, provision.LandProbed) |
| 271 | } |
| 272 | |
| 273 | // TestLandingProbeCrashRestartsOnlyTheProbe (brief, §7.13): a crash after |
| 274 | // an attempt is counted and before its probe returns restarts only the |
| 275 | // probe: a fresh worker probes the same attempt again (its window never |
| 276 | // opened), and local and restored are not rerun, their journal steps and |
| 277 | // .files-gen untouched. An attempt whose window had opened and passed |
| 278 | // counts as failed, and the next one is made. |
| 279 | func TestLandingProbeCrashRestartsOnlyTheProbe(t *testing.T) { |
| 280 | t.Parallel() |
| 281 | h, p := installedLanding(t, 2) |
| 282 | steps := map[string]string{} |
| 283 | for k, v := range h.landJournal(t).Steps { |
| 284 | steps[k] = v |
| 285 | } |
| 286 | marker := filepath.Join(h.tdir, p1Host, archive.MarkerName) |
| 287 | before, err := os.Stat(marker) |
| 288 | if err != nil { |
| 289 | t.Fatal(err) |
| 290 | } |
| 291 | renders, timers := h.renders, len(h.replaced) |
| 292 | h.landAt = "before-probe-1" |
| 293 | h.crashLand(t) |
| 294 | if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandProbing || m.Landing.Attempts != 1 || h.landJournal(t).Attempt != 1 || len(p.configs) != 0 { |
| 295 | t.Fatalf("the crash point: %+v journal %d calls %d", m, h.landJournal(t).Attempt, len(p.configs)) |
| 296 | } |
| 297 | h.freshTarget() |
| 298 | h.land(t) |
| 299 | m := h.atPhase(t, provision.LandProbed) |
| 300 | if len(p.configs) != 1 || m.Landing.Attempts != 1 { |
| 301 | t.Fatalf("after the restart: %d calls, record %+v", len(p.configs), m.Landing) |
| 302 | } |
| 303 | after, err := os.Stat(marker) |
| 304 | if err != nil || !after.ModTime().Equal(before.ModTime()) { |
| 305 | t.Fatalf(".files-gen was written again: %v", err) |
| 306 | } |
| 307 | j := h.landJournal(t) |
| 308 | for k, v := range steps { |
| 309 | if j.Steps[k] != v { |
| 310 | t.Fatalf("the journal step %s changed", k) |
| 311 | } |
| 312 | } |
| 313 | for k := range j.Steps { |
| 314 | if _, ok := steps[k]; !ok && !strings.HasPrefix(k, "probe_") { |
| 315 | t.Fatalf("a non-probe step %s was added", k) |
| 316 | } |
| 317 | } |
| 318 | if h.renders != renders || len(h.replaced) != timers { |
| 319 | t.Fatalf("configured or local ran again: renders %d->%d timers %d->%d", renders, h.renders, timers, len(h.replaced)) |
| 320 | } |
| 321 | |
| 322 | // An attempt whose window opened and passed before the crash counts as |
| 323 | // failed: the restarted worker makes the next one. |
| 324 | h2, p2 := installedLanding(t, 2) |
| 325 | h2.landAt = "before-probe-1" |
| 326 | h2.crashLand(t) |
| 327 | j2 := h2.landJournal(t) |
| 328 | j2.Mark("probe_opened_1", h2.clock.Now().Add(-31*time.Second).UTC().Format(time.RFC3339Nano)) |
| 329 | if err := j2.Commit(h2.tdir); err != nil { |
| 330 | t.Fatal(err) |
| 331 | } |
| 332 | h2.freshTarget() |
| 333 | h2.land(t) |
| 334 | if m := h2.atPhase(t, provision.LandProbed); len(p2.configs) != 1 || m.Landing.Attempts != 2 { |
| 335 | t.Fatalf("the spent attempt was retried: %d calls, record %+v", len(p2.configs), m.Landing) |
| 336 | } |
| 337 | } |
| 338 | |
| 339 | // Unarmed (production until claim Plan 11's launcher and Plan 5 Task 9): |
| 340 | // with no probe wake the landing waits at restored and writes nothing. |
| 341 | func TestLandingUnarmedProbeWaitsAtRestored(t *testing.T) { |
| 342 | t.Parallel() |
| 343 | h, _ := installedLanding(t, 3) |
| 344 | h.probe = nil |
| 345 | before := h.read(t, p1Host).Move.UpdatedAt |
| 346 | h.land(t) |
| 347 | if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandRestored || !m.UpdatedAt.Equal(before) { |
| 348 | t.Fatalf("an unarmed probe moved the landing: %+v", m) |
| 349 | } |
| 350 | } |
| 351 | |
| 352 | // H10: Task 5's Admitted hook reads the admission's recorded count, and |
| 353 | // refuses a record that has none. |
| 354 | func TestMovesRecordAdmittedAttempts(t *testing.T) { |
| 355 | ctx := context.Background() |
| 356 | if n, err := MovesRecordAdmittedAttempts(ctx, &provision.Record{Move: &provision.MoveState{CutAttempts: 2}}); err != nil || n != 2 { |
| 357 | t.Fatalf("recorded: %d %v", n, err) |
| 358 | } |
| 359 | for _, rec := range []*provision.Record{nil, {}, {Move: &provision.MoveState{}}} { |
| 360 | if _, err := MovesRecordAdmittedAttempts(ctx, rec); err == nil { |
| 361 | t.Fatalf("no recorded count was admitted: %+v", rec) |
| 362 | } |
| 363 | } |
| 364 | } |
| 365 | |
| 366 | // Fix round 1, ruling 1: a passing probe whose probing to probed record |
| 367 | // write fails is not probed again. probe_passed_<k> names the admitted |
| 368 | // writer; the next pass validates it against the claim and the activator's |
| 369 | // live entry and retries only the phase writes, record then claim. |
| 370 | func TestLandingProbePassReplaysTheRecordWrite(t *testing.T) { |
| 371 | t.Parallel() |
| 372 | h, p := installedLanding(t, 2) |
| 373 | h.landAt = "before-record-probed" |
| 374 | h.crashLand(t) |
| 375 | if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandProbing || len(p.configs) != 1 { |
| 376 | t.Fatalf("the crash point: %d calls, %+v", len(p.configs), m) |
| 377 | } |
| 378 | if !h.landJournal(t).Done("probe_passed_1") { |
| 379 | t.Fatalf("the pass was not journalled before the record write: %v", h.landJournal(t).Steps) |
| 380 | } |
| 381 | h.land(t) |
| 382 | h.atPhase(t, provision.LandProbed) |
| 383 | if len(p.configs) != 1 || len(p.admissions) != 1 { |
| 384 | t.Fatalf("the replay admitted again: %d calls, %d admissions", len(p.configs), len(p.admissions)) |
| 385 | } |
| 386 | if c := h.claim(t); c.Holder.Session != p.last.Session || c.Holder.Generation != p1ID { |
| 387 | t.Fatalf("the probed writer: %+v", c.Holder) |
| 388 | } |
| 389 | |
| 390 | // The session is gone (the entry did not survive): the attempt is |
| 391 | // consumed and failed, and the next one is made. |
| 392 | h2, p2 := installedLanding(t, 2) |
| 393 | h2.landAt = "before-record-probed" |
| 394 | h2.crashLand(t) |
| 395 | p2.live = false |
| 396 | h2.land(t) |
| 397 | if m := h2.atPhase(t, provision.LandProbed); len(p2.configs) != 2 || m.Landing.Attempts != 2 || p2.admissions[1].attempt != 2 { |
| 398 | t.Fatalf("a lost passing session: %d calls, record %+v, admissions %+v", len(p2.configs), m.Landing, p2.admissions) |
| 399 | } |
| 400 | } |
| 401 | |
| 402 | // Fix round 1, ruling 2 (§7.7): one cold admission per budgeted attempt. A |
| 403 | // crash after the attempt's admission, with its window still open and no |
| 404 | // failure journalled, consumes the attempt: the restart never admits again |
| 405 | // under the same index. |
| 406 | func TestLandingProbeAdmittedAttemptIsConsumedOnRestart(t *testing.T) { |
| 407 | t.Parallel() |
| 408 | h, p := installedLanding(t, 2) |
| 409 | p.openWindow = true |
| 410 | p.answers = []error{errors.New("the controller died during the attempt")} |
| 411 | h.land(t) |
| 412 | if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandProbing || len(p.admissions) != 1 || p.admissions[0].attempt != 1 || h.landJournal(t).Done("probe_failed_1") { |
| 413 | t.Fatalf("the crash: %+v admissions %+v", m, p.admissions) |
| 414 | } |
| 415 | h.freshTarget() |
| 416 | h.land(t) |
| 417 | h.atPhase(t, provision.LandProbed) |
| 418 | if len(p.admissions) != 2 || p.admissions[1].attempt != 2 { |
| 419 | t.Fatalf("an index was admitted twice: %+v", p.admissions) |
| 420 | } |
| 421 | |
| 422 | // With one attempt admitted, the restart has none left. |
| 423 | h1, p1 := installedLanding(t, 1) |
| 424 | p1.openWindow = true |
| 425 | p1.answers = []error{errors.New("the controller died during the attempt")} |
| 426 | h1.land(t) |
| 427 | h1.freshTarget() |
| 428 | h1.land(t) |
| 429 | if m := h1.read(t, p1Host).Move; m.Reason != provision.RollbackProbeAttempts || len(p1.admissions) != 1 { |
| 430 | t.Fatalf("the only attempt was admitted again: %+v admissions %+v", m, p1.admissions) |
| 431 | } |
| 432 | } |
| 433 | |
| 434 | // Fix round 2, ruling 1: the claim, not the binding, says whether attempt k |
| 435 | // was admitted. A crash after the admission CAS and before Reserve's |
| 436 | // binding leaves probe_admitting_<k> and a claim whose writer session moved |
| 437 | // on: the restart consumes k and never admits under it again. |
| 438 | func TestLandingProbeAdmissionWithoutBindingIsConsumed(t *testing.T) { |
| 439 | t.Parallel() |
| 440 | h, p := installedLanding(t, 2) |
| 441 | p.noBind = true |
| 442 | p.answers = []error{errors.New("the controller died after the admission")} |
| 443 | h.land(t) |
| 444 | if len(p.admissions) != 1 || !h.landJournal(t).Done("probe_admitting_1") || attemptAdmitted(h.landJournal(t), 1) { |
| 445 | t.Fatalf("the crash: admissions %+v journal %v", p.admissions, h.landJournal(t).Steps) |
| 446 | } |
| 447 | h.freshTarget() |
| 448 | p.noBind = false |
| 449 | h.land(t) |
| 450 | h.atPhase(t, provision.LandProbed) |
| 451 | if len(p.admissions) != 2 || p.admissions[1].attempt != 2 { |
| 452 | t.Fatalf("index 1 was admitted twice: %+v", p.admissions) |
| 453 | } |
| 454 | |
| 455 | // probe_admitting_<k> with the claim unchanged (no admission happened): |
| 456 | // the same index is used. |
| 457 | h2, p2 := installedLanding(t, 2) |
| 458 | h2.landAt = "before-probe-1" |
| 459 | h2.crashLand(t) |
| 460 | if !h2.landJournal(t).Done("probe_admitting_1") { |
| 461 | t.Fatalf("no admitting step before the call: %v", h2.landJournal(t).Steps) |
| 462 | } |
| 463 | h2.freshTarget() |
| 464 | h2.land(t) |
| 465 | if m := h2.atPhase(t, provision.LandProbed); m.Landing.Attempts != 1 || len(p2.admissions) != 1 || p2.admissions[0].attempt != 1 { |
| 466 | t.Fatalf("an unadmitted attempt was not reused: %+v %+v", m.Landing, p2.admissions) |
| 467 | } |
| 468 | } |
| 469 | |
| 470 | // Fix round 2, ruling 2: a probe that passed but whose probe_passed_<k> |
| 471 | // write failed is found through the activator's live entry: its writer is |
| 472 | // the claim's, so the attempt passed, and the next pass reaches probed |
| 473 | // without a second ProbeWake. |
| 474 | func TestLandingProbePassFoundThroughTheLiveEntry(t *testing.T) { |
| 475 | t.Parallel() |
| 476 | h, p := installedLanding(t, 2) |
| 477 | h.landAt = "before-journal-passed" |
| 478 | h.crashLand(t) |
| 479 | if j := h.landJournal(t); j.Done("probe_passed_1") || len(p.configs) != 1 || !p.live { |
| 480 | t.Fatalf("the crash point: %v calls %d live %v", j.Steps, len(p.configs), p.live) |
| 481 | } |
| 482 | h.land(t) |
| 483 | h.atPhase(t, provision.LandProbed) |
| 484 | if len(p.configs) != 1 || len(p.admissions) != 1 || !h.landJournal(t).Done("probe_passed_1") { |
| 485 | t.Fatalf("probed again: %d calls %d admissions", len(p.configs), len(p.admissions)) |
| 486 | } |
| 487 | if c := h.claim(t); c.Holder.Session != p.last.Session { |
| 488 | t.Fatalf("the probed writer: %+v", c.Holder) |
| 489 | } |
| 490 | } |
| 491 | |
amadan