| 1 | package rollout |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "strings" |
| 7 | "sync/atomic" |
| 8 | "testing" |
| 9 | "time" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/activator" |
| 12 | "github.com/carlosframework/platform/internal/appconfig" |
| 13 | "github.com/carlosframework/platform/internal/registry" |
| 14 | "github.com/carlosframework/platform/internal/release" |
| 15 | "github.com/carlosframework/platform/internal/status" |
| 16 | ) |
| 17 | |
| 18 | func wakeRoute(installed, good string) registry.Route { |
| 19 | return registry.Route{Host: "a", Kind: "instance", App: "blog", Account: "bab", Status: "ready", Hibernate: true, |
| 20 | Version: installed, GoodVersion: good, GoodDigest: "sha-" + good} |
| 21 | } |
| 22 | |
| 23 | func inputs(rt registry.Route, b *FailureBook, p AppPolicy) wakeInputs { |
| 24 | return wakeInputs{Route: rt, OptDir: "/opt/carlos", Policy: p, Book: b, |
| 25 | Demand: func(ver, _ string) (string, string, error) { return "", "sha-" + ver, nil }} |
| 26 | } |
| 27 | |
| 28 | func TestDecideWake(t *testing.T) { |
| 29 | overlap := DefaultPolicy() |
| 30 | held := &FailureBook{} |
| 31 | held.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "x", At: t0}) |
| 32 | down := &FailureBook{} |
| 33 | down.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "a", HeldDown: []string{"a"}, At: t0}) |
| 34 | |
| 35 | if p, _ := decideWake(inputs(wakeRoute("", ""), &FailureBook{}, overlap)); p.Boot.StagedPath != "" || p.Refuse != "no build is installed for this instance yet" { |
| 36 | t.Fatal("nothing stamped: today's wake, unplanned") |
| 37 | } |
| 38 | p, _ := decideWake(inputs(wakeRoute("v1", "v1"), &FailureBook{}, overlap)) |
| 39 | if p.Boot.Version != "v1" || p.FirstWake || p.Boot.Digest != "sha-v1" { |
| 40 | t.Fatalf("installed is the good build: recovery by the retained identity, got %+v", p) |
| 41 | } |
| 42 | p, _ = decideWake(inputs(wakeRoute("v2", "v1"), &FailureBook{}, overlap)) |
| 43 | if p.Boot.Version != "v2" || !p.FirstWake || p.Fallback == nil || p.Fallback.Version != "v1" || p.Mode != activator.ModeOverlap { |
| 44 | t.Fatalf("a new build: a first wake with the good build behind it, got %+v", p) |
| 45 | } |
| 46 | p, _ = decideWake(inputs(wakeRoute("v2", "v1"), &FailureBook{}, AppPolicy{Swap: activator.ModeHold, Concurrency: 1})) |
| 47 | if p.Mode != activator.ModeHold || p.Fallback != nil { |
| 48 | t.Fatalf("hold: no fallback, got %+v", p) |
| 49 | } |
| 50 | in := inputs(wakeRoute("v2", "v1"), &FailureBook{}, overlap) |
| 51 | in.Demand = func(ver, _ string) (string, string, error) { return "hold", "sha-" + ver, nil } |
| 52 | if p, _ := decideWake(in); p.Mode != activator.ModeHold || p.Fallback != nil { |
| 53 | t.Fatalf("the release demands hold: no fallback, got %+v", p) |
| 54 | } |
| 55 | if p, _ := decideWake(inputs(wakeRoute("v2", "v1"), held, overlap)); p.Boot.Version != "v1" || p.FirstWake { |
| 56 | t.Fatalf("installed build held by a record: boot the good build, got %+v", p) |
| 57 | } |
| 58 | // Spec §8.5: an instance with no good build has no fallback, which is |
| 59 | // today's behaviour. A sibling's record must not refuse it outright. |
| 60 | if p, _ := decideWake(inputs(wakeRoute("v2", ""), held, overlap)); p.Refuse != "" || p.Boot.Version != "v2" || !p.FirstWake || |
| 61 | p.Fallback != nil || p.Mode != activator.ModeOverlap { |
| 62 | t.Fatalf("held, no good build, not held down: boot the installed build as a first wake with no fallback, got %+v", p) |
| 63 | } |
| 64 | if p, _ := decideWake(inputs(wakeRoute("v2", ""), down, overlap)); p.Refuse == "" { |
| 65 | t.Fatal("held down with no good build: it stays down (spec §6.4)") |
| 66 | } |
| 67 | if p, _ := decideWake(inputs(wakeRoute("v2", "v1"), down, overlap)); p.Refuse == "" { |
| 68 | t.Fatal("held down after a hold failure: no automatic fallback (spec §6.4)") |
| 69 | } |
| 70 | in = inputs(wakeRoute("v2", "v1"), &FailureBook{}, overlap) |
| 71 | in.Demand = func(string, string) (string, string, error) { return "", "", errors.New("digest mismatch") } |
| 72 | p, err := decideWake(in) |
| 73 | if err == nil || p.Boot.Version != "v1" || p.FirstWake { |
| 74 | t.Fatalf("a staging refusal with a good build boots the good build, got %+v, %v", p, err) |
| 75 | } |
| 76 | } |
| 77 | |
| 78 | // Review focus 3. |
| 79 | func TestPlanBootsTheGoodBuildForAHeldInstallation(t *testing.T) { |
| 80 | h := newHarness(t, "a") |
| 81 | h.stage("v2") |
| 82 | h.install("v2", "a") |
| 83 | h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "b", At: t0}) |
| 84 | p := h.m.Plan(context.Background(), h.rows.get("a")) |
| 85 | if p.Boot.Version != "v1" || p.FirstWake { |
| 86 | t.Fatalf("plan = %+v, want the good v1 without a repoint", p) |
| 87 | } |
| 88 | } |
| 89 | |
| 90 | // registerThenInstall makes the scheduler know host (the pass) while every |
| 91 | // host still wants v1, and only then stamps v2 on host's row. The pass sees |
| 92 | // nothing to adopt, so no swap onto v2 runs beside the wake outcome a test |
| 93 | // hands OnGen: these tests are about the wake, and a swap racing it would |
| 94 | // change what the activator serves under them. |
| 95 | func (h *harness) registerThenInstall(ver, host string) { |
| 96 | h.pass() |
| 97 | h.install(ver, host) |
| 98 | } |
| 99 | |
| 100 | func TestAFellBackWakeRecordsTheFailureAndKeepsTheGoodBuild(t *testing.T) { |
| 101 | h := newHarness(t, "a") |
| 102 | h.stage("v2") |
| 103 | h.registerThenInstall("v2", "a") |
| 104 | rt := h.rows.get("a") |
| 105 | h.act.serve("a", "v1") // the good build the fallback left serving |
| 106 | served, _ := h.act.Current("a") |
| 107 | h.m.OnGen(activator.GenEvent{Kind: activator.GenFellBack, Route: rt, FirstWake: true, Mode: activator.ModeOverlap, |
| 108 | Gen: served, Failed: release.StagedIdentity{Version: "v2"}, Reason: "health check failed"}) |
| 109 | if _, ok := h.book.Lookup("bab", "blog", "v2"); !ok { |
| 110 | t.Fatal("no failure record") |
| 111 | } |
| 112 | if r := h.rows.get("a"); r.GoodVersion != "v1" || r.ServingVersion != "v1" { |
| 113 | t.Fatalf("row = %+v", r) |
| 114 | } |
| 115 | if !h.events.has(textFallback("v2", "v1")) { |
| 116 | t.Fatal("no fallback event") |
| 117 | } |
| 118 | } |
| 119 | |
| 120 | // Spec §8.1, §9.4 (Astra R3 flag triage T11): a first wake that fails on a |
| 121 | // host whose app has no scheduler yet still halts the cohort and says so, |
| 122 | // once, when the record is new. The same failure delivered again (the record |
| 123 | // stands) is not announced twice, and a failure of an execution that began |
| 124 | // before a retry of the build (a pre-retry record) is never announced as a |
| 125 | // halt. |
| 126 | func TestAFirstWakeFailureWithNoSchedulerAnnouncesTheHalt(t *testing.T) { |
| 127 | h := newHarness(t, "a") |
| 128 | h.stage("v2") |
| 129 | h.install("v2", "a") // no pass: the manager has no scheduler for the host |
| 130 | if h.m.schedulerOf("a") != nil { |
| 131 | t.Fatal("the harness already has a scheduler for the host") |
| 132 | } |
| 133 | halted := textCohortHalted("v2", h.m.d.Box, status.ReasonHealth) |
| 134 | ev := activator.GenEvent{Kind: activator.GenFailed, Route: h.rows.get("a"), FirstWake: true, Mode: activator.ModeOverlap, |
| 135 | Failed: release.StagedIdentity{Version: "v2"}, Reason: "GET / answered 500"} |
| 136 | h.m.OnGen(ev) |
| 137 | if _, ok := h.book.Lookup("bab", "blog", "v2"); !ok { |
| 138 | t.Fatal("no failure record") |
| 139 | } |
| 140 | if n := h.events.count(halted); n != 1 { |
| 141 | t.Fatalf("%d cohort-halted event(s) after a new record, want 1", n) |
| 142 | } |
| 143 | h.m.OnGen(ev) |
| 144 | if n := h.events.count(halted); n != 1 { |
| 145 | t.Fatalf("%d cohort-halted event(s) after the same failure again, want still 1", n) |
| 146 | } |
| 147 | |
| 148 | // A wake planned before a retry of its build fails stale: PreRetry. |
| 149 | p := newHarness(t, "a") |
| 150 | p.stage("v2") |
| 151 | p.install("v2", "a") |
| 152 | p.wakeHost("a") |
| 153 | plan := p.m.Plan(context.Background(), p.rows.get("a")) |
| 154 | if !plan.FirstWake || plan.Boot.Version != "v2" { |
| 155 | t.Fatalf("plan %+v: want a first wake of v2", plan) |
| 156 | } |
| 157 | if p.m.schedulerOf("a") != nil { |
| 158 | t.Fatal("the harness has a scheduler for the host") |
| 159 | } |
| 160 | p.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "s", PromotedAt: t0, At: t0}) |
| 161 | if ok, err := p.book.RetryHost("bab", "blog", "v2", "a"); !ok || err != nil { |
| 162 | t.Fatalf("RetryHost = %v, %v: cleared nothing", ok, err) |
| 163 | } |
| 164 | p.m.OnGen(activator.GenEvent{Kind: activator.GenFailed, Route: p.rows.get("a"), Failed: plan.Boot, Mode: plan.Mode, |
| 165 | FirstWake: true, Reason: "health check failed", Token: plan.Token, ConfigRev: plan.ConfigRev}) |
| 166 | if n := p.events.count(halted); n != 0 { |
| 167 | t.Fatalf("%d cohort-halted event(s) for a failure that began before a retry", n) |
| 168 | } |
| 169 | if p.book.Blocks("bab", "blog", "v2", "a", false) { |
| 170 | t.Fatal("the stale failure halted the retried host") |
| 171 | } |
| 172 | } |
| 173 | |
| 174 | // Spec §8.1: a fallback publishes the failure hold before the serving build |
| 175 | // and the active config. Writing the serving build first showed the good |
| 176 | // build serving with no hold on the failed one, a window in which a wake or |
| 177 | // an admission could select the failed build again. |
| 178 | func TestAFallbackHoldsTheFailedBuildBeforeItPublishesServing(t *testing.T) { |
| 179 | h := newHarness(t, "a") |
| 180 | h.stage("v2") |
| 181 | h.registerThenInstall("v2", "a") |
| 182 | h.act.serve("a", "v1") |
| 183 | served, _ := h.act.Current("a") |
| 184 | var early atomic.Int32 |
| 185 | h.rows.mu.Lock() |
| 186 | h.rows.setServingHook = func(string) { |
| 187 | if _, ok := h.book.Lookup("bab", "blog", "v2"); !ok { |
| 188 | early.Add(1) |
| 189 | } |
| 190 | } |
| 191 | h.rows.mu.Unlock() |
| 192 | h.m.OnGen(activator.GenEvent{Kind: activator.GenFellBack, Route: h.rows.get("a"), FirstWake: true, Mode: activator.ModeOverlap, |
| 193 | Gen: served, Failed: release.StagedIdentity{Version: "v2"}, Reason: "health check failed"}) |
| 194 | if n := early.Load(); n != 0 { |
| 195 | t.Fatal("the serving build was published before the failed build was held") |
| 196 | } |
| 197 | if r := h.rows.get("a"); r.ServingVersion != "v1" { |
| 198 | t.Fatalf("serving = %q, want the good v1", r.ServingVersion) |
| 199 | } |
| 200 | } |
| 201 | |
| 202 | func TestOnlyASettledFirstWakeBecomesTheGoodBuild(t *testing.T) { |
| 203 | h := newHarness(t, "a") |
| 204 | h.stage("v2") |
| 205 | h.registerThenInstall("v2", "a") |
| 206 | rt := h.rows.get("a") |
| 207 | h.act.serve("a", "v2") // the first wake's generation, which the activator serves |
| 208 | g, _ := h.act.Current("a") |
| 209 | h.m.OnGen(activator.GenEvent{Kind: activator.GenHealthy, Route: rt, Gen: g, FirstWake: true}) |
| 210 | if h.rows.get("a").GoodVersion != "v1" { |
| 211 | t.Fatal("a first wake became the good build before its 10 s (spec §8.5)") |
| 212 | } |
| 213 | h.m.OnGen(activator.GenEvent{Kind: activator.GenSettled, Route: rt, Gen: g, FirstWake: true}) |
| 214 | if r := h.rows.get("a"); r.GoodVersion != "v2" || r.ServingVersion != "v2" { |
| 215 | t.Fatalf("row = %+v: the settled first wake must be the good and the serving build", r) |
| 216 | } |
| 217 | } |
| 218 | |
| 219 | // Task 9's verdict floor: a first wake left too little health window boots |
| 220 | // the good build untried and reports an ordinary healthy wake of it |
| 221 | // (FirstWake false). The scheduler records the good build as serving and |
| 222 | // records nothing against the untried build, which stays installed for a |
| 223 | // later wake. |
| 224 | func TestAnUntriedFirstWakeIsAnOrdinaryWakeOfTheGoodBuild(t *testing.T) { |
| 225 | h := newHarness(t, "a") |
| 226 | h.stage("v2") |
| 227 | h.registerThenInstall("v2", "a") |
| 228 | h.act.serve("a", "v1") |
| 229 | g, _ := h.act.Current("a") |
| 230 | h.m.OnGen(activator.GenEvent{Kind: activator.GenHealthy, Route: h.rows.get("a"), Gen: g, Mode: activator.ModeOverlap}) |
| 231 | if _, ok := h.book.Lookup("bab", "blog", "v2"); ok { |
| 232 | t.Fatal("the untried build was recorded as failed") |
| 233 | } |
| 234 | if r := h.rows.get("a"); r.ServingVersion != "v1" || r.GoodVersion != "v1" || r.Version != "v2" { |
| 235 | t.Fatalf("row = %+v: want v1 serving and good, v2 still installed", r) |
| 236 | } |
| 237 | if p := h.m.Plan(context.Background(), h.rows.get("a")); !p.FirstWake || p.Boot.Version != "v2" { |
| 238 | t.Fatalf("plan %+v: the untried build must stay a candidate for the next wake", p) |
| 239 | } |
| 240 | } |
| 241 | |
| 242 | // Spec §6.4: an intentional stop during the 10 s observation records |
| 243 | // nothing. GenCancelled only closes the observation. |
| 244 | // |
| 245 | // What it must not publish is the observed build, v2 (fix wave 1, item 6). |
| 246 | // The row's serving build is not required to stay empty: the scheduler last |
| 247 | // saw host a wanting v1 (registerThenInstall), and with the fake activator |
| 248 | // now serving v2 its loop may admit a swap back onto v1 at any moment, whose |
| 249 | // success acks v1 off this goroutine. Asserting "" made the test fail |
| 250 | // whenever that swap won the race (seen 3 in 300 runs beside the pointer |
| 251 | // test). The cancel's own writes are flushed before the row is read. |
| 252 | func TestACancelledObservationRecordsNothing(t *testing.T) { |
| 253 | h := newHarness(t, "a") |
| 254 | h.stage("v2") |
| 255 | h.registerThenInstall("v2", "a") |
| 256 | h.act.serve("a", "v2") |
| 257 | g, _ := h.act.Current("a") |
| 258 | h.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: h.rows.get("a"), Gen: g, FirstWake: true}) |
| 259 | h.flushAcks() |
| 260 | if _, ok := h.book.Lookup("bab", "blog", "v2"); ok { |
| 261 | t.Fatal("a cancelled observation recorded a failure") |
| 262 | } |
| 263 | if r := h.rows.get("a"); r.GoodVersion != "v1" || r.ServingVersion == "v2" { |
| 264 | t.Fatalf("row = %+v: a cancelled observation published a build", r) |
| 265 | } |
| 266 | } |
| 267 | |
| 268 | // Spec §7.9: a worker's serving build has one writer, the activator |
| 269 | // (ordered by incarnation). The scheduler records a worker's good build but |
| 270 | // leaves its ServingVersion alone, so its write can never interleave with |
| 271 | // the activator's and restore an older value. |
| 272 | func TestTheSchedulerLeavesAWorkersServingVersionAlone(t *testing.T) { |
| 273 | h := newHarness(t, "a") |
| 274 | h.act.serve("a", "v2") |
| 275 | g, _ := h.act.Current("a") |
| 276 | g.Worker = true |
| 277 | h.m.served("a", g, 0) |
| 278 | if r := h.rows.get("a"); r.ServingVersion != "" || r.GoodVersion != "v2" { |
| 279 | t.Fatalf("row = %+v: want the good build recorded and the serving build left to the activator", r) |
| 280 | } |
| 281 | } |
| 282 | |
| 283 | // Spec §8.5: a good build is written expecting the current one. A late |
| 284 | // success of an older generation (a completion delayed past a newer swap, a |
| 285 | // stale wake event) is not the generation the activator serves, so it never |
| 286 | // overwrites the newer good or serving build. Writing whatever arrived last |
| 287 | // regressed both to the older build. |
| 288 | func TestAStaleSuccessNeverOverwritesANewerBuild(t *testing.T) { |
| 289 | h := newHarness(t, "a") |
| 290 | h.act.serve("a", "v2") |
| 291 | older, _ := h.act.Current("a") |
| 292 | h.act.serve("a", "v3") // a newer generation took over |
| 293 | newer, _ := h.act.Current("a") |
| 294 | h.m.served("a", newer, 0) |
| 295 | h.m.served("a", older, 0) // the older one's completion arrives late |
| 296 | h.m.OnGen(activator.GenEvent{Kind: activator.GenSettled, Route: h.rows.get("a"), Gen: older, FirstWake: true}) |
| 297 | if r := h.rows.get("a"); r.GoodVersion != "v3" || r.ServingVersion != "v3" { |
| 298 | t.Fatalf("row = %+v: an older generation overwrote the newer good and serving build", r) |
| 299 | } |
| 300 | } |
| 301 | |
| 302 | func TestAFailedHoldFirstWakeHoldsTheInstanceDown(t *testing.T) { |
| 303 | h := newHarness(t, "a") |
| 304 | h.stage("v2") |
| 305 | h.registerThenInstall("v2", "a") |
| 306 | h.m.OnGen(activator.GenEvent{Kind: activator.GenFailed, Route: h.rows.get("a"), FirstWake: true, Mode: activator.ModeHold, |
| 307 | Failed: release.StagedIdentity{Version: "v2"}, Reason: "health check failed"}) |
| 308 | if p := h.m.Plan(context.Background(), h.rows.get("a")); p.Refuse == "" { |
| 309 | t.Fatalf("plan = %+v, want a refusal: recovery after a hold failure is explicit", p) |
| 310 | } |
| 311 | // Task 4's rule: a record never carries a zero PromotedAt. No pointer was |
| 312 | // read for this wake, so the failure's own time stands in. |
| 313 | if rec, ok := h.book.Lookup("bab", "blog", "v2"); !ok || !rec.PromotedAt.Equal(t0) || !rec.At.Equal(t0) { |
| 314 | t.Fatalf("record %+v %v: want PromotedAt = At = the failure's time", rec, ok) |
| 315 | } |
| 316 | if !h.events.has(textCohortHalted("v2", "box-1", status.ReasonHealth)) { |
| 317 | t.Fatal("no cohort-halted event with the reason word") |
| 318 | } |
| 319 | } |
| 320 | |
| 321 | func TestAWakeActivatesTheConfigItStartedWith(t *testing.T) { |
| 322 | h := newHarness(t, "a") |
| 323 | h.pass() |
| 324 | h.act.mu.Lock() |
| 325 | delete(h.act.gens, "a") |
| 326 | h.act.owned["a"] = true // waking: a process, not yet current |
| 327 | h.act.mu.Unlock() |
| 328 | var acks int |
| 329 | ack := h.m.DeferAck(func(context.Context, appconfig.ConfigAck) { acks++ }) |
| 330 | h.m.ConfigDelivered("a", "sha256:c1") |
| 331 | ack(appconfig.ConfigAck{Host: "a"}) |
| 332 | h.flushAcks() |
| 333 | if acks != 0 { |
| 334 | t.Fatal("acknowledged before any generation started with the config") |
| 335 | } |
| 336 | rt := h.rows.get("a") |
| 337 | p := h.m.Plan(context.Background(), rt) |
| 338 | // The activator carries the plan's token and revision onto the event. |
| 339 | h.m.OnGen(activator.GenEvent{Kind: activator.GenHealthy, Route: rt, Gen: activator.Gen{Build: "v1", Digest: "sha-v1"}, |
| 340 | Token: p.Token, ConfigRev: p.ConfigRev}) |
| 341 | h.flushAcks() |
| 342 | if acks != 1 { |
| 343 | t.Fatalf("acks = %d after the wake, want 1", acks) |
| 344 | } |
| 345 | } |
| 346 | |
| 347 | // Spec §8.1: a sleeping instance needs no config attempt. Its next wake |
| 348 | // starts with the delivered file, and config sync's ack goes out at |
| 349 | // delivery, as today, marked Hibernating. |
| 350 | func TestASleepersConfigNeedsNoAttempt(t *testing.T) { |
| 351 | h := newHarness(t, "a") |
| 352 | h.pass() |
| 353 | h.act.mu.Lock() |
| 354 | delete(h.act.gens, "a") // asleep: no process at all |
| 355 | h.act.mu.Unlock() |
| 356 | var acks []appconfig.ConfigAck |
| 357 | ack := h.m.DeferAck(func(_ context.Context, a appconfig.ConfigAck) { acks = append(acks, a) }) |
| 358 | h.m.ConfigDelivered("a", "sha256:c1") |
| 359 | ack(appconfig.ConfigAck{Host: "a", Restarted: true}) |
| 360 | h.flushAcks() |
| 361 | if len(acks) != 1 || !acks[0].Hibernating || acks[0].Restarted { |
| 362 | t.Fatalf("acks = %+v, want one at delivery, Hibernating", acks) |
| 363 | } |
| 364 | h.pass() |
| 365 | time.Sleep(50 * time.Millisecond) |
| 366 | if calls, _, _ := h.act.snapshot(); len(calls) != 0 { |
| 367 | t.Fatalf("attempts %+v: a sleeper got a config attempt", calls) |
| 368 | } |
| 369 | } |
| 370 | |
| 371 | // Spec §8.1: a failed first wake is published with no blocking read in |
| 372 | // between. The failing pointer's PromotedAt is read when the wake is |
| 373 | // planned and travels on the event. buildFailed read it again after the |
| 374 | // failure, before the record, leaving every sibling's wake free to plan the |
| 375 | // failed build meanwhile. |
| 376 | func TestAFailedFirstWakeIsRecordedWithNoPointerRead(t *testing.T) { |
| 377 | h := newHarness(t, "a") |
| 378 | h.stage("v2") |
| 379 | h.install("v2", "a") |
| 380 | promoted := t0.Add(-time.Hour) |
| 381 | h.pointer("stable", "v2", promoted) |
| 382 | p := h.m.Plan(context.Background(), h.rows.get("a")) |
| 383 | if !p.FirstWake || !p.FailedUnder.Equal(promoted) { |
| 384 | t.Fatalf("plan %+v: want a first wake carrying the pointer's PromotedAt", p) |
| 385 | } |
| 386 | var reads atomic.Int32 |
| 387 | h.mu.Lock() |
| 388 | h.pointerHook = func() { reads.Add(1) } |
| 389 | h.mu.Unlock() |
| 390 | h.m.OnGen(activator.GenEvent{Kind: activator.GenFailed, Route: h.rows.get("a"), FirstWake: true, Mode: activator.ModeOverlap, |
| 391 | Failed: p.Boot, Reason: "GET / answered 500", FailedUnder: p.FailedUnder}) |
| 392 | if n := reads.Load(); n != 0 { |
| 393 | t.Fatalf("%d pointer read(s) between the failure and its record", n) |
| 394 | } |
| 395 | rec, ok := h.book.Lookup("bab", "blog", "v2") |
| 396 | if !ok || !rec.PromotedAt.Equal(promoted) { |
| 397 | t.Fatalf("record %+v %v: want one carrying the planned PromotedAt", rec, ok) |
| 398 | } |
| 399 | } |
| 400 | |
| 401 | // Spec §6.4, §8.4: a cold managed host (nothing installed yet) has its build |
| 402 | // installed and stamped before the wake is planned, so the plan boots that |
| 403 | // identity, as a first wake, and respects its failure records. An empty |
| 404 | // plan sent the wake down the live-link path, which materialized the |
| 405 | // channel head and ran it unplanned, a held build included. |
| 406 | func TestAColdManagedHostIsInstalledBeforeItIsPlanned(t *testing.T) { |
| 407 | h := newHarness(t, "a") |
| 408 | h.stage("v2") |
| 409 | cold := func() { |
| 410 | r := h.rows.get("a") |
| 411 | r.Version = "" |
| 412 | h.rows.put(r) |
| 413 | } |
| 414 | cold() |
| 415 | installs := 0 |
| 416 | h.m.d.Materialize = func(_ context.Context, host string) error { |
| 417 | installs++ |
| 418 | r := h.rows.get(host) |
| 419 | r.Version = "v2" |
| 420 | h.rows.put(r) |
| 421 | return nil |
| 422 | } |
| 423 | if p := h.m.Plan(context.Background(), h.rows.get("a")); installs != 1 || p.Boot.Version != "v2" || !p.FirstWake { |
| 424 | t.Fatalf("installs %d, plan %+v: want v2 installed, then planned as a first wake", installs, p) |
| 425 | } |
| 426 | if _, err := h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "a", At: t0}); err != nil { |
| 427 | t.Fatal(err) |
| 428 | } |
| 429 | cold() |
| 430 | if p := h.m.Plan(context.Background(), h.rows.get("a")); p.Boot.Version == "v2" { |
| 431 | t.Fatalf("plan %+v: a cold host was booted onto a build its app holds", p) |
| 432 | } |
| 433 | h.m.d.Materialize = nil |
| 434 | cold() |
| 435 | if p := h.m.Plan(context.Background(), h.rows.get("a")); p.Refuse == "" || p.Boot.StagedPath != "" { |
| 436 | t.Fatalf("plan %+v: with nothing installed and no way to install, the wake must be refused", p) |
| 437 | } |
| 438 | } |
| 439 | |
| 440 | // A refusal's text reaches the member's "wake failed" event (Task 9). A |
| 441 | // failed install's error, which can name box paths, goes to the operator log |
| 442 | // only. |
| 443 | func TestAFailedColdInstallRefusesWithoutItsErrorText(t *testing.T) { |
| 444 | h := newHarness(t, "a") |
| 445 | r := h.rows.get("a") |
| 446 | r.Version = "" |
| 447 | h.rows.put(r) |
| 448 | h.m.d.Materialize = func(context.Context, string) error { |
| 449 | return errors.New("stage /opt/carlos/versions/bab/v2/blog: no space left on device") |
| 450 | } |
| 451 | p := h.m.Plan(context.Background(), h.rows.get("a")) |
| 452 | if p.Refuse == "" || p.Boot.StagedPath != "" { |
| 453 | t.Fatalf("plan %+v: a failed install must refuse the wake", p) |
| 454 | } |
| 455 | if strings.Contains(p.Refuse, "/opt") || strings.Contains(p.Refuse, "no space") { |
| 456 | t.Fatalf("refusal %q carries the raw error", p.Refuse) |
| 457 | } |
| 458 | } |
| 459 | |
| 460 | // Spec §4.2 (Task 3's carry, as resolve applies it to an attempt): a first |
| 461 | // wake whose canonical demand cannot be read or compared boots in hold, on |
| 462 | // the staged bytes hashed here, with no fallback. Missing or unverifiable |
| 463 | // data never permits an overlap. A staged file that is not the release's |
| 464 | // artifact is a staging refusal: the good build boots and root is asked to |
| 465 | // stage again. |
| 466 | func TestAPlannedWakeReadsTheDemandAsAnAttemptDoes(t *testing.T) { |
| 467 | h := newHarness(t, "a") |
| 468 | h.stage("v2") |
| 469 | h.install("v2", "a") |
| 470 | sum, err := release.FileSHA256(release.VersionPath(h.opt, "bab", "v2", "blog")) |
| 471 | if err != nil { |
| 472 | t.Fatal(err) |
| 473 | } |
| 474 | |
| 475 | h.mu.Lock() |
| 476 | h.demandErr["v2"] = errors.New("manifest unreadable") |
| 477 | h.mu.Unlock() |
| 478 | if p := h.m.Plan(context.Background(), h.rows.get("a")); !p.FirstWake || p.Mode != activator.ModeHold || p.Fallback != nil || p.Boot.Digest != sum { |
| 479 | t.Fatalf("plan %+v: an unreadable demand must boot v2 in hold on its hashed bytes", p) |
| 480 | } |
| 481 | |
| 482 | h.mu.Lock() |
| 483 | delete(h.demandErr, "v2") |
| 484 | h.noDigest["v2"] = true |
| 485 | h.mu.Unlock() |
| 486 | if p := h.m.Plan(context.Background(), h.rows.get("a")); !p.FirstWake || p.Mode != activator.ModeHold || p.Fallback != nil || p.Boot.Digest != sum { |
| 487 | t.Fatalf("plan %+v: a demand nothing compared must boot v2 in hold on its hashed bytes", p) |
| 488 | } |
| 489 | |
| 490 | h.mu.Lock() |
| 491 | delete(h.noDigest, "v2") |
| 492 | h.mismatch["v2"] = true |
| 493 | h.mu.Unlock() |
| 494 | p := h.m.Plan(context.Background(), h.rows.get("a")) |
| 495 | if p.FirstWake || p.Boot.Version != "v1" { |
| 496 | t.Fatalf("plan %+v: a mis-staged v2 must boot the good v1", p) |
| 497 | } |
| 498 | if h.touched.Load() != 1 || !h.events.has(textStagingRefused("v2")) { |
| 499 | t.Fatal("a staging refusal must ask root to stage again and say so") |
| 500 | } |
| 501 | } |
| 502 | |
| 503 | // Spec §7.4, §8.1–8.4 (P5 review finding 3): a plan carries its selection |
| 504 | // gate. A sibling's failure recorded after the plan was made refuses the |
| 505 | // planned build at the exec; the good build still passes. |
| 506 | func TestAPlansCheckRefusesABuildHeldSinceThePlan(t *testing.T) { |
| 507 | h := newHarness(t, "a") |
| 508 | h.stage("v2") |
| 509 | h.install("v2", "a") |
| 510 | p := h.m.Plan(context.Background(), h.rows.get("a")) |
| 511 | if !p.FirstWake || p.Boot.Version != "v2" || p.Check == nil { |
| 512 | t.Fatalf("plan %+v: want a first wake onto v2 with its check", p) |
| 513 | } |
| 514 | if err := p.Check("v2"); err != nil { |
| 515 | t.Fatalf("nothing holds v2 yet: %v", err) |
| 516 | } |
| 517 | if _, err := h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "b", At: t0}); err != nil { |
| 518 | t.Fatal(err) |
| 519 | } |
| 520 | err := p.Check("v2") |
| 521 | if err == nil { |
| 522 | t.Fatal("v2 failed on a sibling after the plan, and the check still let it boot") |
| 523 | } |
| 524 | // Approved wording (zdd.refuse.held-since-plan). |
| 525 | if want := "build v2 failed on this box while this instance was starting, so it was not started"; err.Error() != want { |
| 526 | t.Fatalf("refusal %q, want the draft %q", err, want) |
| 527 | } |
| 528 | if err := p.Check("v1"); err != nil { |
| 529 | t.Fatalf("the good build was refused: %v", err) |
| 530 | } |
| 531 | } |
| 532 | |
| 533 | // Spec §8.5: a record written after the plan does not refuse an instance with |
| 534 | // no good build either. It is not moved onto anything else, and refusing it |
| 535 | // would turn a sibling's failure into an outage. A hold-down still refuses. |
| 536 | func TestAPlansCheckPassesAHostWithNoGoodBuildWhenARecordLandsAfterThePlan(t *testing.T) { |
| 537 | h := newHarness(t, "a") |
| 538 | h.stage("v2") |
| 539 | h.install("v2", "a") |
| 540 | r := h.rows.get("a") |
| 541 | r.GoodVersion, r.GoodDigest = "", "" |
| 542 | h.rows.put(r) |
| 543 | p := h.m.Plan(context.Background(), h.rows.get("a")) |
| 544 | if !p.FirstWake || p.Boot.Version != "v2" || p.Fallback != nil || p.Check == nil { |
| 545 | t.Fatalf("plan %+v: want a first wake onto v2, no fallback, with its check", p) |
| 546 | } |
| 547 | if _, err := h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "b", At: t0}); err != nil { |
| 548 | t.Fatal(err) |
| 549 | } |
| 550 | if err := p.Check("v2"); err != nil { |
| 551 | t.Fatalf("no good build, a sibling's record since the plan: the check must pass, got %v", err) |
| 552 | } |
| 553 | if _, err := h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "a", HeldDown: []string{"a"}, At: t0}); err != nil { |
| 554 | t.Fatal(err) |
| 555 | } |
| 556 | if err := p.Check("v2"); err == nil { |
| 557 | t.Fatal("held down on this instance since the plan: the check must still refuse") |
| 558 | } |
| 559 | } |
| 560 | |
| 561 | // Spec §2.9 (rev 17), §6.4, §8.3: an overlap attempt on a foreign-layout row |
| 562 | // runs as a hold, and the activator says so (SwapResult.Mode). Its failure |
| 563 | // is a hold build's failure: held down, so no wake boots the good build in |
| 564 | // its place. And while an attempt runs for an instance, no wake of it is |
| 565 | // planned, so a hold that went down is recorded before anything wakes it. |
| 566 | // Recording under the dispatched ModeOverlap left it un-held, and the next |
| 567 | // wake booted the good build against the database the hold build ran on. |
| 568 | func TestAForeignRowsFailedHoldIsHeldDown(t *testing.T) { |
| 569 | h := newHarness(t, "a") |
| 570 | h.stage("v2") |
| 571 | h.act.outcome = func(activator.Attempt) activator.SwapOutcome { return activator.SwapDown } |
| 572 | h.act.mode = activator.ModeHold // the effective mode, as plan 3's Swap reports it |
| 573 | gate := h.act.hold("a") |
| 574 | h.install("v2", "a") |
| 575 | h.pass() |
| 576 | waitFor(t, "the attempt to start", func() bool { _, started, _ := h.act.snapshot(); return len(started) == 1 }) |
| 577 | // Approved wording (zdd.refuse.attempt-running, zdd.refuse.held-down). |
| 578 | if p := h.m.Plan(context.Background(), h.rows.get("a")); p.Refuse != "this instance is still moving to another build" { |
| 579 | t.Fatalf("a wake was planned while the instance's attempt ran: %+v", p) |
| 580 | } |
| 581 | close(gate) |
| 582 | waitFor(t, "the hold-down record", func() bool { return h.book.HeldDown("bab", "blog", "v2", "a") }) |
| 583 | wantDown := "build v2 failed in hold mode on this instance, so it stays down until you roll back or retry the build" |
| 584 | if p := h.m.Plan(context.Background(), h.rows.get("a")); p.Refuse != wantDown || p.Boot.StagedPath != "" { |
| 585 | t.Fatalf("after the hold went down the wake planned %+v, want refused %q (held down)", p, wantDown) |
| 586 | } |
| 587 | } |
| 588 | |
| 589 | // The activator's sweep calls OnDrift (Nudge) on a goroutine of its own for |
| 590 | // every drifted entry on every sweep, with no dedupe of its own (Task 9). |
| 591 | // Nudges for one host collapse: while one is reading the row, the others |
| 592 | // return at once and leave it one more look, so nothing piles up behind a |
| 593 | // slow read and nothing a later nudge was for is lost. And a nudge for the |
| 594 | // build an attempt is already running onto neither queues it again nor |
| 595 | // supersedes it. |
| 596 | func TestNudgesForOneHostCollapse(t *testing.T) { |
| 597 | h := newHarness(t, "a") |
| 598 | h.stage("v2") |
| 599 | h.pass() |
| 600 | release1 := make(chan struct{}) |
| 601 | var gets atomic.Int32 |
| 602 | h.rows.mu.Lock() |
| 603 | h.rows.getHook = func(host string) { |
| 604 | if host == "a" && gets.Add(1) == 1 { |
| 605 | <-release1 |
| 606 | } |
| 607 | } |
| 608 | h.rows.mu.Unlock() |
| 609 | first := make(chan struct{}) |
| 610 | go func() { h.m.Nudge("a"); close(first) }() |
| 611 | waitFor(t, "the first nudge to read the row", func() bool { return gets.Load() == 1 }) |
| 612 | for range 20 { |
| 613 | h.m.Nudge("a") // returns at once: the first nudge owns this host |
| 614 | } |
| 615 | if n := gets.Load(); n != 1 { |
| 616 | t.Fatalf("%d row reads while one nudge was in flight, want 1", n) |
| 617 | } |
| 618 | close(release1) |
| 619 | <-first |
| 620 | if n := gets.Load(); n != 2 { |
| 621 | t.Fatalf("%d row reads, want 2: the in-flight nudge looks once more for the ones it absorbed", n) |
| 622 | } |
| 623 | |
| 624 | h.rows.mu.Lock() |
| 625 | h.rows.getHook = nil |
| 626 | h.rows.mu.Unlock() |
| 627 | gate := h.act.hold("a") |
| 628 | h.install("v2", "a") |
| 629 | h.m.Nudge("a") |
| 630 | waitFor(t, "the attempt onto v2 to start", func() bool { _, started, _ := h.act.snapshot(); return len(started) == 1 }) |
| 631 | for range 5 { |
| 632 | h.m.Nudge("a") |
| 633 | } |
| 634 | close(gate) |
| 635 | waitFor(t, "the attempt to finish", func() bool { calls, _, _ := h.act.snapshot(); return len(calls) == 1 }) |
| 636 | calls, started, _ := h.act.snapshot() |
| 637 | if calls[0].outcome != activator.SwapDone || len(started) != 1 { |
| 638 | t.Fatalf("calls %+v started %v: repeated nudges re-queued or superseded the running attempt", calls, started) |
| 639 | } |
| 640 | } |
| 641 | |
| 642 | // wakeHost makes host a sleeper whose wake is in flight: no current |
| 643 | // generation, but the activator owns its entry, as it does from the wake's |
| 644 | // plan to its outcome. |
| 645 | func (h *harness) wakeHost(host string) { |
| 646 | h.act.mu.Lock() |
| 647 | defer h.act.mu.Unlock() |
| 648 | delete(h.act.gens, host) |
| 649 | h.act.owned[host] = true |
| 650 | } |
| 651 | |
| 652 | // I1 (Task 16's simulation, seeds 8, 70 and 102): a wake planned onto build |
| 653 | // B, a sibling's failure of B recorded while the wake is still on its way to |
| 654 | // the exec, then a promotion that moves every route on the box off B. The |
| 655 | // next Pass must not prune B's record: the plan's check reads only the book, |
| 656 | // so a pruned record let the wake boot the failed B, in hold mode against |
| 657 | // its database. The record goes once nothing planned still carries B: the |
| 658 | // wake's outcome, a newer plan for the host, or the activator letting the |
| 659 | // host go. |
| 660 | func TestAPlannedWakeKeepsItsBuildsFailureRecordThroughAPrune(t *testing.T) { |
| 661 | sibling := FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "b", At: t0} |
| 662 | setup := func(t *testing.T) (*harness, activator.WakePlan) { |
| 663 | h := newHarness(t, "a") |
| 664 | h.stage("v2", "v3") |
| 665 | h.install("v2", "a") |
| 666 | h.wakeHost("a") |
| 667 | p := h.m.Plan(context.Background(), h.rows.get("a")) |
| 668 | if !p.FirstWake || p.Boot.Version != "v2" || p.Check == nil { |
| 669 | t.Fatalf("plan %+v: want a first wake onto v2 with its check", p) |
| 670 | } |
| 671 | if _, err := h.book.Record(sibling); err != nil { |
| 672 | t.Fatal(err) |
| 673 | } |
| 674 | h.install("v3", "a") // the promotion: no route on the box installs v2 now |
| 675 | h.pass() |
| 676 | if _, ok := h.book.Lookup("bab", "blog", "v2"); !ok { |
| 677 | t.Fatal("the Pass pruned v2's record while a planned wake still carried v2") |
| 678 | } |
| 679 | if err := p.Check("v2"); err == nil { |
| 680 | t.Fatal("the wake's check let v2 boot: it failed on a sibling after the plan") |
| 681 | } |
| 682 | return h, p |
| 683 | } |
| 684 | pruned := func(t *testing.T, h *harness, why string) { |
| 685 | t.Helper() |
| 686 | h.pass() |
| 687 | if _, ok := h.book.Lookup("bab", "blog", "v2"); ok { |
| 688 | t.Fatalf("%s, and the next Pass still kept v2's record", why) |
| 689 | } |
| 690 | } |
| 691 | t.Run("the wake's outcome", func(t *testing.T) { |
| 692 | h, p := setup(t) |
| 693 | h.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: h.rows.get("a"), FirstWake: true, Token: p.Token}) |
| 694 | pruned(t, h, "the wake reported its outcome") |
| 695 | }) |
| 696 | t.Run("a newer plan", func(t *testing.T) { |
| 697 | h, _ := setup(t) |
| 698 | if p := h.m.Plan(context.Background(), h.rows.get("a")); p.Boot.Version != "v3" { |
| 699 | t.Fatalf("replanned %+v, want v3", p) |
| 700 | } |
| 701 | pruned(t, h, "the host's newer plan boots v3") |
| 702 | }) |
| 703 | t.Run("the activator let the host go", func(t *testing.T) { |
| 704 | h, _ := setup(t) |
| 705 | h.act.mu.Lock() |
| 706 | delete(h.act.owned, "a") // the wake ended with no event: refused at its lease |
| 707 | h.act.mu.Unlock() |
| 708 | pruned(t, h, "the activator no longer owns the host") |
| 709 | }) |
| 710 | } |
| 711 | |
| 712 | // blockDemand makes the demand read for ver block (once per call to it) |
| 713 | // until the returned release is called. blocked closes when the read starts. |
| 714 | func (h *harness) blockDemand(ver string) (blocked <-chan struct{}, release func()) { |
| 715 | in, out := make(chan struct{}), make(chan struct{}) |
| 716 | var n atomic.Int32 |
| 717 | h.mu.Lock() |
| 718 | h.demandHook = func(v string) { |
| 719 | if v == ver && n.Add(1) == 1 { |
| 720 | close(in) |
| 721 | <-out |
| 722 | } |
| 723 | } |
| 724 | h.mu.Unlock() |
| 725 | var once atomic.Bool |
| 726 | return in, func() { |
| 727 | if once.CompareAndSwap(false, true) { |
| 728 | close(out) |
| 729 | } |
| 730 | } |
| 731 | } |
| 732 | |
| 733 | // planAsync runs Plan for host on its own goroutine, as the activator's wake |
| 734 | // does, and returns its plan on the channel. |
| 735 | func (h *harness) planAsync(rt registry.Route) <-chan activator.WakePlan { |
| 736 | done := make(chan activator.WakePlan, 1) |
| 737 | go func() { done <- h.m.Plan(context.Background(), rt) }() |
| 738 | return done |
| 739 | } |
| 740 | |
| 741 | func waitClosed(t *testing.T, what string, c <-chan struct{}) { |
| 742 | t.Helper() |
| 743 | select { |
| 744 | case <-c: |
| 745 | case <-time.After(5 * time.Second): |
| 746 | t.Fatalf("timed out waiting for %s", what) |
| 747 | } |
| 748 | } |
| 749 | |
| 750 | // I1, the planner's own window (Task 10 round 3): Plan reads the book, then |
| 751 | // makes its slow reads (the canonical demand, the failing pointer, the |
| 752 | // policy) before it returns. A sibling's failure of B recorded while Plan is |
| 753 | // in those reads, then a promotion that moves the box off B, then a Pass: |
| 754 | // had Plan registered B only at its return, the Pass would prune B's record |
| 755 | // (no route installs B, nothing tracks it) and the exec would boot the failed |
| 756 | // B. Plan registers B before its reads, so the record stands, and it reads |
| 757 | // the row again after them, so a wake whose row moved is planned against the |
| 758 | // fresh row. |
| 759 | func TestAPlanInItsSlowReadsKeepsItsBuildsFailureRecord(t *testing.T) { |
| 760 | sibling := FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "b", At: t0} |
| 761 | notBootable := func(t *testing.T, p activator.WakePlan) { |
| 762 | t.Helper() |
| 763 | if p.Boot.Version == "v2" && (p.Check == nil || p.Check("v2") == nil) { |
| 764 | t.Fatalf("plan %+v: the exec may boot v2, which failed on a sibling during the plan", p) |
| 765 | } |
| 766 | } |
| 767 | |
| 768 | t.Run("the route moves during the reads", func(t *testing.T) { |
| 769 | h := newHarness(t, "a") |
| 770 | h.stage("v2", "v3") |
| 771 | h.install("v2", "a") |
| 772 | h.wakeHost("a") |
| 773 | blocked, release := h.blockDemand("v2") |
| 774 | defer release() |
| 775 | done := h.planAsync(h.rows.get("a")) |
| 776 | waitClosed(t, "Plan to read v2's demand", blocked) |
| 777 | if _, err := h.book.Record(sibling); err != nil { |
| 778 | t.Fatal(err) |
| 779 | } |
| 780 | h.install("v3", "a") // the promotion: no route on the box installs v2 now |
| 781 | h.pass() |
| 782 | _, kept := h.book.Lookup("bab", "blog", "v2") |
| 783 | release() |
| 784 | p := <-done |
| 785 | notBootable(t, p) |
| 786 | if !kept { |
| 787 | t.Fatal("the Pass pruned v2's record while Plan was still planning v2") |
| 788 | } |
| 789 | if !p.FirstWake || p.Boot.Version != "v3" { |
| 790 | t.Fatalf("plan %+v: the row moved to v3 during the plan, want a first wake onto v3", p) |
| 791 | } |
| 792 | }) |
| 793 | |
| 794 | t.Run("the row was stale before the plan", func(t *testing.T) { |
| 795 | // The activator read the row (v2) before a promotion landed; the |
| 796 | // sibling's record was pruned before Plan began. Plan's registration |
| 797 | // comes too late for that prune, and the read of the row after the |
| 798 | // slow reads catches it. |
| 799 | h := newHarness(t, "a") |
| 800 | h.stage("v2", "v3") |
| 801 | h.install("v2", "a") |
| 802 | h.wakeHost("a") |
| 803 | stale := h.rows.get("a") |
| 804 | if _, err := h.book.Record(sibling); err != nil { |
| 805 | t.Fatal(err) |
| 806 | } |
| 807 | h.install("v3", "a") |
| 808 | h.pass() |
| 809 | if _, ok := h.book.Lookup("bab", "blog", "v2"); ok { |
| 810 | t.Fatal("setup: nothing installs or plans v2, so the Pass should have pruned it") |
| 811 | } |
| 812 | p := h.m.Plan(context.Background(), stale) |
| 813 | if p.Boot.Version != "v3" { |
| 814 | t.Fatalf("plan %+v from a stale row: want the fresh row's v3", p) |
| 815 | } |
| 816 | }) |
| 817 | |
| 818 | t.Run("the route moves twice", func(t *testing.T) { |
| 819 | // The row moves during the first plan and again during the replan: |
| 820 | // the plan refuses rather than chase the row. |
| 821 | h := newHarness(t, "a") |
| 822 | h.stage("v2", "v3", "v4") |
| 823 | h.install("v2", "a") |
| 824 | h.wakeHost("a") |
| 825 | blocked, release := h.blockDemand("v2") |
| 826 | defer release() |
| 827 | done := h.planAsync(h.rows.get("a")) |
| 828 | waitClosed(t, "Plan to read v2's demand", blocked) |
| 829 | h.install("v3", "a") |
| 830 | blocked3, release3 := h.blockDemand("v3") |
| 831 | defer release3() |
| 832 | release() |
| 833 | waitClosed(t, "the replan to read v3's demand", blocked3) |
| 834 | h.install("v4", "a") |
| 835 | release3() |
| 836 | if p := <-done; p.Refuse == "" || p.Boot.StagedPath != "" { |
| 837 | t.Fatalf("plan %+v: the row moved twice during the plan, want refused", p) |
| 838 | } |
| 839 | }) |
| 840 | } |
| 841 | |
amadan