| 1 | package rollout |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "fmt" |
| 8 | "os" |
| 9 | "path/filepath" |
| 10 | "strings" |
| 11 | "sync" |
| 12 | "testing" |
| 13 | "time" |
| 14 | |
| 15 | "github.com/carlosframework/platform/internal/activator" |
| 16 | "github.com/carlosframework/platform/internal/appconfig" |
| 17 | "github.com/carlosframework/platform/internal/status" |
| 18 | ) |
| 19 | |
| 20 | var markV2 = HoldRan{Account: "bab", App: "blog", Build: "v2"} |
| 21 | |
| 22 | // Final review wave 2, N2: decideWake's rule for an instance with a |
| 23 | // hold-ran marker. The good build never boots in place of the installed |
| 24 | // build, and (Astra R3 N7, ruling 205) the marker lifts no failure record |
| 25 | // and the installed good build waits for a signed rollback. |
| 26 | func TestDecideWakeWithAHoldRanMarker(t *testing.T) { |
| 27 | overlap := DefaultPolicy() |
| 28 | book := func(recs ...FailureRecord) *FailureBook { |
| 29 | b := &FailureBook{} |
| 30 | for _, r := range recs { |
| 31 | b.Record(r) |
| 32 | } |
| 33 | b.MarkHoldRan("a", markV2) |
| 34 | return b |
| 35 | } |
| 36 | sibV2 := FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "x", At: t0} |
| 37 | sibV3 := FailureRecord{Account: "bab", App: "blog", Build: "v3", Host: "x", At: t0} |
| 38 | |
| 39 | // N7: the marked build, blocked for the instance by a sibling's record, |
| 40 | // with a good build: refused, never v1 and never v2. |
| 41 | p, err := decideWake(inputs(wakeRoute("v2", "v1"), book(sibV2), overlap)) |
| 42 | if err != nil || p.Refuse != HeldDownRefusal("v2") || p.Boot.StagedPath != "" { |
| 43 | t.Fatalf("installed is the marked build, blocked: want the wake refused in the hold-down words, got %+v, %v", p, err) |
| 44 | } |
| 45 | // With no good build a record alone refuses nothing (spec §8.5): the |
| 46 | // marked build boots, hold, no fallback. |
| 47 | noGood := wakeRoute("v2", "") |
| 48 | noGood.GoodDigest = "" |
| 49 | if p, err := decideWake(inputs(noGood, book(sibV2), overlap)); err != nil || p.Refuse != "" || p.Boot.Version != "v2" || |
| 50 | !p.FirstWake || p.Fallback != nil || p.Mode != activator.ModeHold { |
| 51 | t.Fatalf("installed is the marked build, blocked, no good build: want a hold first wake of v2, got %+v, %v", p, err) |
| 52 | } |
| 53 | // Unblocked, and its release allows overlap: still hold, no fallback. |
| 54 | if p, _ := decideWake(inputs(wakeRoute("v2", "v1"), book(), overlap)); p.Boot.Version != "v2" || p.Fallback != nil || p.Mode != activator.ModeHold { |
| 55 | t.Fatalf("installed is the marked build, unblocked: want hold with no fallback, got %+v", p) |
| 56 | } |
| 57 | // Held down on the instance: the hold-down refusal still comes first. |
| 58 | down := book(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "a", HeldDown: []string{"a"}, At: t0}) |
| 59 | if p, _ := decideWake(inputs(wakeRoute("v2", "v1"), down, overlap)); p.Refuse != HeldDownRefusal("v2") { |
| 60 | t.Fatalf("marked build held down: want the hold-down refusal, got %+v", p) |
| 61 | } |
| 62 | // Another installed build held for the instance: refused, never v1. |
| 63 | if p, _ := decideWake(inputs(wakeRoute("v3", "v1"), book(sibV3), overlap)); p.Refuse != HeldDownRefusal("v2") || p.Boot.StagedPath != "" { |
| 64 | t.Fatalf("another held build: want the wake refused in the hold-down words, got %+v", p) |
| 65 | } |
| 66 | // Another, unblocked build: a first wake with no fallback, in its own mode. |
| 67 | if p, _ := decideWake(inputs(wakeRoute("v3", "v1"), book(), overlap)); p.Refuse != "" || p.Boot.Version != "v3" || !p.FirstWake || |
| 68 | p.Fallback != nil || p.Mode != activator.ModeOverlap { |
| 69 | t.Fatalf("another unblocked build: want an overlap first wake of v3 with no fallback, got %+v", p) |
| 70 | } |
| 71 | // Ruling 205: the good build installed is no proof of a rollback (root's |
| 72 | // sibling repoint installs it too). The planner refuses while the marker |
| 73 | // stands; the pointer pass lifts the marker on a signed rollback. |
| 74 | if p, _ := decideWake(inputs(wakeRoute("v1", "v1"), book(), overlap)); p.Refuse != HeldDownRefusal("v2") || p.Boot.StagedPath != "" { |
| 75 | t.Fatalf("the good build installed under a marker: want the wake refused, got %+v", p) |
| 76 | } |
| 77 | // Task 3 fix round 1: the marked build is also the good build, and it is |
| 78 | // installed. The marked-build rule applies, not the good-build arm: it |
| 79 | // boots, in hold mode with no fallback. A sibling's record of it does not |
| 80 | // hold an instance that served it (spec §8.4, Blocks with served); only a |
| 81 | // hold-down refuses it. |
| 82 | for name, b := range map[string]*FailureBook{"no record": book(), "a sibling's record": book(sibV2)} { |
| 83 | if p, err := decideWake(inputs(wakeRoute("v2", "v2"), b, overlap)); err != nil || p.Refuse != "" || p.Boot.Version != "v2" || |
| 84 | !p.FirstWake || p.Fallback != nil || p.Mode != activator.ModeHold { |
| 85 | t.Fatalf("marked build = installed = good, %s: want a hold first wake of v2 with no fallback, got %+v, %v", name, p, err) |
| 86 | } |
| 87 | } |
| 88 | downGood := book(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "a", HeldDown: []string{"a"}, At: t0}) |
| 89 | if p, _ := decideWake(inputs(wakeRoute("v2", "v2"), downGood, overlap)); p.Refuse != HeldDownRefusal("v2") || p.Boot.StagedPath != "" { |
| 90 | t.Fatalf("marked build = installed = good, held down: want the hold-down refusal, got %+v", p) |
| 91 | } |
| 92 | // A staging refusal never boots the good build instead. |
| 93 | in := inputs(wakeRoute("v2", "v1"), book(), overlap) |
| 94 | in.Demand = func(string, string) (string, string, error) { return "", "", errors.New("digest mismatch") } |
| 95 | if p, err := decideWake(in); err == nil || p.Refuse == "" || p.Boot.StagedPath != "" { |
| 96 | t.Fatalf("a staging refusal under the marker: want the wake refused, got %+v, %v", p, err) |
| 97 | } |
| 98 | // Another app's marker under the same host name changes nothing. |
| 99 | other := &FailureBook{} |
| 100 | other.Record(sibV2) |
| 101 | other.MarkHoldRan("a", HoldRan{Account: "bab", App: "other", Build: "v2"}) |
| 102 | if p, _ := decideWake(inputs(wakeRoute("v2", "v1"), other, overlap)); p.Boot.Version != "v1" || p.FirstWake { |
| 103 | t.Fatalf("another app's marker: want the usual recovery onto v1, got %+v", p) |
| 104 | } |
| 105 | } |
| 106 | |
| 107 | // Final review wave 2, N2: a planned wake onto the good build made before |
| 108 | // the marker existed (a recovery for the blocked v2) is refused at its exec |
| 109 | // once the marker stands. The marker comes from a hold first wake whose |
| 110 | // observation was cancelled after it ran. The refusal uses existing words. |
| 111 | func TestAHoldRanMarkerRefusesTheGoodBuildAtTheExec(t *testing.T) { |
| 112 | ctx := context.Background() |
| 113 | h := newHarness(t, "h") |
| 114 | h.stage("v2") |
| 115 | h.install("v2", "h") |
| 116 | h.sleepHost("h") |
| 117 | h.pass() |
| 118 | h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", At: t0}) |
| 119 | h.wakeHost("h") |
| 120 | recovery := h.m.Plan(ctx, h.rows.get("h")) |
| 121 | if recovery.Boot.Version != "v1" || recovery.FirstWake || recovery.Check == nil { |
| 122 | t.Fatalf("plan %+v: want the recovery onto the good v1 with its check (v2 is blocked, no marker yet)", recovery) |
| 123 | } |
| 124 | if err := recovery.Check("v1"); err != nil { |
| 125 | t.Fatalf("before the marker the good build passes the check: %v", err) |
| 126 | } |
| 127 | h.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: h.rows.get("h"), Gen: activator.Gen{Build: "v2"}, |
| 128 | Mode: activator.ModeHold, FirstWake: true, Executed: true}) |
| 129 | err := recovery.Check("v1") |
| 130 | if err == nil { |
| 131 | t.Fatal("the recovery onto v1 passed its exec check after a hold build ran on h") |
| 132 | } |
| 133 | if want := HeldDownRefusal("v2"); err.Error() != want { |
| 134 | t.Fatalf("refusal %q, want the existing words %q", err, want) |
| 135 | } |
| 136 | } |
| 137 | |
| 138 | // Final review wave 2, N2: a planned hold first wake whose observation was |
| 139 | // cancelled after its build ran (the activator's GenCancelled, Executed) |
| 140 | // leaves the marker, so a sibling's failure of the build afterwards does not |
| 141 | // send the instance's next wake to the good build. The marker is a |
| 142 | // restriction on the instance, not a failure: nothing is recorded or halted. |
| 143 | // The instance passing on the build (a settled first wake records it good) |
| 144 | // clears the marker, and with it the restriction. |
| 145 | func TestAFirstWakeCancelledAfterItsHoldBuildRanAndItsClearingByAPass(t *testing.T) { |
| 146 | ctx := context.Background() |
| 147 | h := newHarness(t, "s") |
| 148 | h.swap = appconfig.SwapHold |
| 149 | h.stage("v2", "v3") |
| 150 | h.install("v2", "s") |
| 151 | h.sleepHost("s") |
| 152 | h.pass() |
| 153 | h.wakeHost("s") |
| 154 | p := h.m.Plan(ctx, h.rows.get("s")) |
| 155 | if !p.FirstWake || p.Boot.Version != "v2" || p.Mode != activator.ModeHold { |
| 156 | t.Fatalf("plan %+v: want a hold first wake of v2", p) |
| 157 | } |
| 158 | g := activator.Gen{N: 7, Build: "v2", StagedPath: p.Boot.StagedPath, Digest: p.Boot.Digest} |
| 159 | h.m.OnGen(activator.GenEvent{Kind: activator.GenHealthy, Route: h.rows.get("s"), Gen: g, Mode: p.Mode, FirstWake: true, Token: p.Token}) |
| 160 | h.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: h.rows.get("s"), Gen: g, Mode: p.Mode, FirstWake: true, |
| 161 | Token: p.Token, Executed: true}) |
| 162 | if got, ok := h.book.HoldRanOn("s"); !ok || !got.same(markV2) || !got.At.Equal(t0) { |
| 163 | t.Fatalf("marker = %+v, %v: want v2's, stamped at its wake's plan", got, ok) |
| 164 | } |
| 165 | if _, ok := h.book.Lookup("bab", "blog", "v2"); ok { |
| 166 | t.Fatal("a cancelled observation recorded a failure") |
| 167 | } |
| 168 | |
| 169 | // A sibling then fails v2: the record holds v2 for s too (Astra R3 N7), |
| 170 | // and the marker holds v1, so s stays down rather than boot either. |
| 171 | h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", PromotedAt: t0, At: t0}) |
| 172 | h.sleepHost("s") |
| 173 | h.wakeHost("s") |
| 174 | if p := h.m.Plan(ctx, h.rows.get("s")); p.Refuse != HeldDownRefusal("v2") { |
| 175 | t.Fatalf("plan %+v: want the wake refused while the sibling's record of v2 stands", p) |
| 176 | } |
| 177 | // `carlos promote --retry` of v2 clears the record; v2 boots again, in |
| 178 | // hold mode with nothing behind it, never the good v1. |
| 179 | h.advance(time.Minute) |
| 180 | h.pointer("stable", "v2", h.clock()) |
| 181 | h.pass() |
| 182 | if _, ok := h.book.Lookup("bab", "blog", "v2"); ok { |
| 183 | t.Fatal("the retry pointer left v2's record standing") |
| 184 | } |
| 185 | h.wakeHost("s") |
| 186 | p = h.m.Plan(ctx, h.rows.get("s")) |
| 187 | if p.Refuse != "" || p.Boot.Version != "v2" || !p.FirstWake || p.Fallback != nil || p.Mode != activator.ModeHold { |
| 188 | t.Fatalf("plan %+v: want v2 again, never the good v1", p) |
| 189 | } |
| 190 | |
| 191 | // It runs and settles: v2 is s's good build now, and the marker goes. |
| 192 | h.act.serve("s", "v2") |
| 193 | cur, _ := h.act.Current("s") |
| 194 | h.m.OnGen(activator.GenEvent{Kind: activator.GenHealthy, Route: h.rows.get("s"), Gen: cur, Mode: p.Mode, FirstWake: true, Token: p.Token}) |
| 195 | h.m.OnGen(activator.GenEvent{Kind: activator.GenSettled, Route: h.rows.get("s"), Gen: cur, Mode: p.Mode, FirstWake: true, Token: p.Token}) |
| 196 | if r := h.rows.get("s"); r.GoodVersion != "v2" { |
| 197 | t.Fatalf("good build %q after the settle, want v2", r.GoodVersion) |
| 198 | } |
| 199 | if got, ok := h.book.HoldRanOn("s"); ok { |
| 200 | t.Fatalf("marker %+v survived the instance passing on a build", got) |
| 201 | } |
| 202 | // The usual rule again: a held v3 sends the wake to the good build. |
| 203 | h.install("v3", "s") |
| 204 | h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v3", Host: "sib", At: t0}) |
| 205 | h.sleepHost("s") |
| 206 | h.wakeHost("s") |
| 207 | if p := h.m.Plan(ctx, h.rows.get("s")); p.Boot.Version != "v2" || p.FirstWake { |
| 208 | t.Fatalf("plan %+v after the marker cleared: want the recovery onto the good v2", p) |
| 209 | } |
| 210 | } |
| 211 | |
| 212 | // Final review wave 2, N2: `carlos instances retry` clears the host's |
| 213 | // marker, with or without a failure record to clear beside it. |
| 214 | func TestRetryHostClearsTheHoldRanMarker(t *testing.T) { |
| 215 | t.Run("with a record", func(t *testing.T) { |
| 216 | h := newHarness(t, "a") |
| 217 | h.stage("v2") |
| 218 | h.install("v2", "a") |
| 219 | h.sleepHost("a") |
| 220 | h.pass() |
| 221 | h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", At: t0}) |
| 222 | h.book.MarkHoldRan("a", markV2) |
| 223 | if !h.retry("a", "v2") { |
| 224 | t.Fatal("RetryHost reported nothing cleared") |
| 225 | } |
| 226 | if _, ok := h.book.HoldRanOn("a"); ok { |
| 227 | t.Fatal("the marker survived the host's retry") |
| 228 | } |
| 229 | }) |
| 230 | t.Run("marker only", func(t *testing.T) { |
| 231 | h := newHarness(t, "a", "b") |
| 232 | h.stage("v2") |
| 233 | h.install("v2", "a") |
| 234 | h.sleepHost("a") |
| 235 | h.pass() |
| 236 | h.book.MarkHoldRan("a", markV2) |
| 237 | h.book.MarkHoldRan("b", markV2) |
| 238 | if !h.retry("a", "v2") { |
| 239 | t.Fatal("RetryHost reported nothing cleared with a marker standing") |
| 240 | } |
| 241 | if _, ok := h.book.HoldRanOn("a"); ok { |
| 242 | t.Fatal("the marker survived the host's retry") |
| 243 | } |
| 244 | if _, ok := h.book.HoldRanOn("b"); !ok { |
| 245 | t.Fatal("a's retry cleared b's marker") |
| 246 | } |
| 247 | }) |
| 248 | } |
| 249 | |
| 250 | // Final review wave 2, N2, the scheduler's half: which attempt outcomes set |
| 251 | // the marker. A hold adoption that ran and did not pass, and was not held |
| 252 | // down, sets it; a failed one sets it in the same save as its record. One |
| 253 | // that ran nothing, an overlap attempt, a restart, a hold-down and a pass do |
| 254 | // not. |
| 255 | func TestWhichAttemptOutcomesLeaveTheHoldRanMarker(t *testing.T) { |
| 256 | // Started is the dispatch's (scheduler.execute): the marker's At, on disk |
| 257 | // too, whichever save sets it (Task 3 fix round 1). |
| 258 | started := t0.Add(-time.Minute) |
| 259 | adopt := dispatch{ID: Identity{Host: "a", Cause: activator.CauseRelease, Build: "v2"}, SchedRev: 1, Class: classAdopt, Cohort: cohortKey("v2", classAdopt), Started: started} |
| 260 | restart := dispatch{ID: Identity{Host: "a", Cause: activator.CauseRestart, Build: "v1", RestartID: "r1"}, SchedRev: 1, Class: classRecover, Cohort: cohortKey("v1", classRecover)} |
| 261 | for _, tc := range []struct { |
| 262 | name string |
| 263 | d dispatch |
| 264 | res activator.SwapResult |
| 265 | mode activator.SwapMode |
| 266 | want bool |
| 267 | }{ |
| 268 | {"hold, superseded after it ran", adopt, activator.SwapResult{Outcome: activator.SwapAbandoned, Executed: true}, activator.ModeHold, true}, |
| 269 | {"hold, failed after it ran, not held down", adopt, activator.SwapResult{Outcome: activator.SwapFailed, Reason: "health check failed", Executed: true}, activator.ModeHold, true}, |
| 270 | {"hold, abandoned before it ran", adopt, activator.SwapResult{Outcome: activator.SwapAbandoned}, activator.ModeHold, false}, |
| 271 | {"hold, down: held down instead", adopt, activator.SwapResult{Outcome: activator.SwapDown, Reason: "health check failed", Executed: true}, activator.ModeHold, false}, |
| 272 | {"overlap, superseded after it ran", adopt, activator.SwapResult{Outcome: activator.SwapAbandoned, Executed: true}, activator.ModeOverlap, false}, |
| 273 | {"hold restart, superseded after it ran", restart, activator.SwapResult{Outcome: activator.SwapAbandoned, Executed: true}, activator.ModeHold, false}, |
| 274 | } { |
| 275 | t.Run(tc.name, func(t *testing.T) { |
| 276 | h := newHarness(t, "a") |
| 277 | dir := t.TempDir() |
| 278 | b, err := OpenFailureBook(dir) |
| 279 | if err != nil { |
| 280 | t.Fatal(err) |
| 281 | } |
| 282 | h.book, h.m.d.Book = b, b |
| 283 | s := h.m.scheduler(appKey{"bab", "blog"}) |
| 284 | s.mu.Lock() |
| 285 | slot := s.q.slot("a", t0) |
| 286 | slot.serving, slot.good = "v1", "v1" |
| 287 | d := tc.d |
| 288 | slot.running = &d |
| 289 | s.q.running++ |
| 290 | s.mu.Unlock() |
| 291 | s.done(tc.d, tc.res, tc.mode) |
| 292 | |
| 293 | again, err := OpenFailureBook(dir) |
| 294 | if err != nil { |
| 295 | t.Fatal(err) |
| 296 | } |
| 297 | got, ok := again.HoldRanOn("a") |
| 298 | if ok != tc.want || (ok && (!got.same(markV2) || !got.At.Equal(started))) { |
| 299 | t.Fatalf("marker on disk = %+v, %v; want set %v, stamped at the dispatch %v", got, ok, tc.want, started) |
| 300 | } |
| 301 | if tc.res.Outcome == activator.SwapFailed { |
| 302 | if _, ok := again.Lookup("bab", "blog", "v2"); !ok { |
| 303 | t.Fatal("the failure record is not on disk beside the marker") |
| 304 | } |
| 305 | } |
| 306 | }) |
| 307 | } |
| 308 | } |
| 309 | |
| 310 | // Final review wave 2, N2: status shows an instance with a marker as held, |
| 311 | // in the existing words (HeldBy, AttemptDown), never new ones. |
| 312 | // |
| 313 | // Astra R3 N7 and ruling 205 add two down cases: the marked build itself |
| 314 | // held by a sibling's record (d), and the good build installed with no |
| 315 | // signed rollback yet (c). Their wakes are refused, so they are down. |
| 316 | func TestStatusShowsAHoldRanInstanceAsHeld(t *testing.T) { |
| 317 | h := newHarness(t, "a", "b", "c") |
| 318 | h.stage("v2", "v3") |
| 319 | h.install("v2", "a") |
| 320 | h.install("v3", "b") |
| 321 | h.sleepHost("a") |
| 322 | h.sleepHost("b") |
| 323 | h.sleepHost("c") |
| 324 | h.pass() |
| 325 | h.book.MarkHoldRan("a", markV2) |
| 326 | h.book.MarkHoldRan("b", markV2) |
| 327 | h.book.MarkHoldRan("c", markV2) |
| 328 | h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v3", Host: "sib", Reason: "exited early", At: t0}) |
| 329 | r := h.m.Status() |
| 330 | if got := r.Hosts["a"]; got.HeldBy != "v2" || got.Attempt != "" { |
| 331 | t.Fatalf("a = %+v: want held by v2 (its wake boots v2, nothing else)", got) |
| 332 | } |
| 333 | if got := r.Hosts["b"]; got.HeldBy != "v3" || got.Attempt != status.AttemptDown || got.Reason != status.ReasonExit { |
| 334 | t.Fatalf("b = %+v: want down, held by v3 (its wakes are refused, not fallen back)", got) |
| 335 | } |
| 336 | if got := r.Hosts["c"]; got.HeldBy != "v2" || got.Attempt != status.AttemptDown { |
| 337 | t.Fatalf("c = %+v: want down, held by v2 (its good build v1 is installed, but no signed rollback lifted the marker)", got) |
| 338 | } |
| 339 | |
| 340 | d := newHarness(t, "d") |
| 341 | d.stage("v2") |
| 342 | d.install("v2", "d") |
| 343 | d.sleepHost("d") |
| 344 | d.pass() |
| 345 | d.book.MarkHoldRan("d", markV2) |
| 346 | d.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", Reason: "exited early", At: t0}) |
| 347 | if got := d.m.Status().Hosts["d"]; got.HeldBy != "v2" || got.Attempt != status.AttemptDown || got.Reason != status.ReasonExit { |
| 348 | t.Fatalf("d = %+v: want down, held by v2 (the marked build is held by a sibling's record)", got) |
| 349 | } |
| 350 | |
| 351 | // Task 3 fix round 1: the marked build is also the installed good build. |
| 352 | // Its wake boots it (the marked-build rule), so it is held, not down. |
| 353 | e := newHarness(t, "e") |
| 354 | e.stage("v2") |
| 355 | r2 := e.rows.get("e") |
| 356 | r2.Version, r2.GoodVersion, r2.GoodDigest = "v2", "v2", "sha-v2" |
| 357 | e.rows.put(r2) |
| 358 | e.sleepHost("e") |
| 359 | e.pass() |
| 360 | e.book.MarkHoldRan("e", markV2) |
| 361 | e.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", Reason: "exited early", At: t0}) |
| 362 | if got := e.m.Status().Hosts["e"]; got.HeldBy != "v2" || got.Attempt != "" { |
| 363 | t.Fatalf("e = %+v: want held by v2 and not down (its marked build is its installed good build, which its wake boots)", got) |
| 364 | } |
| 365 | } |
| 366 | |
| 367 | // Task 3 fix round 1: when the marker names the installed good build, the |
| 368 | // marked-build rule applies at the plan and at the exec, even with a |
| 369 | // sibling's record of the build standing (the instance served it). The good |
| 370 | // build is still refused at the exec of any other plan while the marker |
| 371 | // stands. |
| 372 | func TestAMarkerOnTheInstalledGoodBuildBootsIt(t *testing.T) { |
| 373 | ctx := context.Background() |
| 374 | h := newHarness(t, "h") |
| 375 | h.stage("v2", "v3") |
| 376 | r := h.rows.get("h") |
| 377 | r.Version, r.GoodVersion, r.GoodDigest = "v2", "v2", "sha-v2" |
| 378 | h.rows.put(r) |
| 379 | h.book.MarkHoldRan("h", markV2) |
| 380 | h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", At: t0}) |
| 381 | h.sleepHost("h") |
| 382 | h.wakeHost("h") |
| 383 | p := h.m.Plan(ctx, h.rows.get("h")) |
| 384 | if p.Refuse != "" || p.Boot.Version != "v2" || p.Mode != activator.ModeHold || p.Fallback != nil || p.Check == nil { |
| 385 | t.Fatalf("plan %+v: want a hold wake of v2 with no fallback", p) |
| 386 | } |
| 387 | if err := p.Check("v2"); err != nil { |
| 388 | t.Fatalf("the plan's check refuses v2, the marked and installed good build: %v", err) |
| 389 | } |
| 390 | // Another build installed: the good v2 is not the installed build, and |
| 391 | // the good-build arm refuses it at that plan's exec. |
| 392 | h.install("v3", "h") |
| 393 | other := h.rows.get("h") |
| 394 | check := h.m.planCheck(other, activator.WakePlan{Boot: p.Boot, Mode: activator.ModeHold, FirstWake: true}) |
| 395 | if err := check("v2"); err == nil || err.Error() != HeldDownRefusal("v2") { |
| 396 | t.Fatalf("check of the good v2 with v3 installed = %v, want the hold-down refusal", err) |
| 397 | } |
| 398 | } |
| 399 | |
| 400 | // Final review wave 2, N2: the markers survive a reopen; a record written |
| 401 | // with MarkHoldRan carries its marker in the same file and never stores the |
| 402 | // instruction; a clear is saved too. |
| 403 | func TestTheHoldRanMarkerSurvivesReopen(t *testing.T) { |
| 404 | dir := t.TempDir() |
| 405 | b, err := OpenFailureBook(dir) |
| 406 | if err != nil { |
| 407 | t.Fatal(err) |
| 408 | } |
| 409 | if err := b.MarkHoldRan("a.example.com", markV2); err != nil { |
| 410 | t.Fatal(err) |
| 411 | } |
| 412 | r := rec("v3", "c.example.com") |
| 413 | r.MarkHoldRan = true |
| 414 | if _, err := b.Record(r); err != nil { |
| 415 | t.Fatal(err) |
| 416 | } |
| 417 | raw, err := os.ReadFile(filepath.Join(dir, FailureFile)) |
| 418 | if err != nil { |
| 419 | t.Fatal(err) |
| 420 | } |
| 421 | if strings.Contains(strings.ToLower(string(raw)), "markholdran") || strings.Contains(string(raw), "mark_hold_ran") { |
| 422 | t.Fatalf("the instruction was stored:\n%s", raw) |
| 423 | } |
| 424 | again, err := OpenFailureBook(dir) |
| 425 | if err != nil { |
| 426 | t.Fatal(err) |
| 427 | } |
| 428 | if got, ok := again.HoldRanOn("a.example.com"); !ok || !got.same(markV2) { |
| 429 | t.Fatalf("a's marker after reopen = %+v, %v", got, ok) |
| 430 | } |
| 431 | if got, ok := again.HoldRanOn("c.example.com"); !ok || got.Build != "v3" { |
| 432 | t.Fatalf("c's marker (set with its record) after reopen = %+v, %v", got, ok) |
| 433 | } |
| 434 | if rec, _ := again.Lookup("bab", "blog", "v3"); rec.MarkHoldRan { |
| 435 | t.Fatal("a record came back carrying the instruction") |
| 436 | } |
| 437 | if cleared, err := again.ClearHoldRan("a.example.com"); !cleared || err != nil { |
| 438 | t.Fatalf("ClearHoldRan = %v, %v", cleared, err) |
| 439 | } |
| 440 | third, err := OpenFailureBook(dir) |
| 441 | if err != nil { |
| 442 | t.Fatal(err) |
| 443 | } |
| 444 | if _, ok := third.HoldRanOn("a.example.com"); ok { |
| 445 | t.Fatal("the clear was not saved") |
| 446 | } |
| 447 | if _, ok := third.HoldRanOn("c.example.com"); !ok { |
| 448 | t.Fatal("clearing a's marker dropped c's") |
| 449 | } |
| 450 | } |
| 451 | |
| 452 | // Final review wave 2, N2: the markers are additive to schema 1. A file an |
| 453 | // older agent wrote (no hold_ran key) opens with no markers. A file with |
| 454 | // markers still parses into the old shape (plain encoding/json skips the |
| 455 | // unknown key) with every record intact. Since Task 5 it is written as |
| 456 | // schema 2, and since the crash-epoch hard limit as schema 3, which an older |
| 457 | // agent's reader refuses on purpose rather than read it and drop the |
| 458 | // markers and intents (TestTheBookIsUnreadableToAnOlderReader). |
| 459 | func TestTheHoldRanMarkerIsSchemaAdditive(t *testing.T) { |
| 460 | dir := t.TempDir() |
| 461 | old := `{"schema": 1, "records": [{"account": "bab", "app": "blog", "build": "v2", "host": "a", "cause": "release", |
| 462 | "mode": "hold", "reason": "health check failed", "promoted_at": "2023-11-14T22:13:20Z", "at": "2023-11-14T22:13:20Z"}]}` |
| 463 | if err := os.WriteFile(filepath.Join(dir, FailureFile), []byte(old), 0o644); err != nil { |
| 464 | t.Fatal(err) |
| 465 | } |
| 466 | b, err := OpenFailureBook(dir) |
| 467 | if err != nil { |
| 468 | t.Fatalf("an older agent's file: %v", err) |
| 469 | } |
| 470 | if _, ok := b.HoldRanOn("a"); ok { |
| 471 | t.Fatal("a file without markers came back with one") |
| 472 | } |
| 473 | if !b.Blocks("bab", "blog", "v2", "x", false) { |
| 474 | t.Fatal("the older file's record was lost") |
| 475 | } |
| 476 | if err := b.MarkHoldRan("a", markV2); err != nil { |
| 477 | t.Fatal(err) |
| 478 | } |
| 479 | raw, err := os.ReadFile(filepath.Join(dir, FailureFile)) |
| 480 | if err != nil { |
| 481 | t.Fatal(err) |
| 482 | } |
| 483 | // The schema 1 shape an agent before the markers decodes into. |
| 484 | var older struct { |
| 485 | Schema int `json:"schema"` |
| 486 | Records []FailureRecord `json:"records"` |
| 487 | } |
| 488 | if err := json.Unmarshal(raw, &older); err != nil { |
| 489 | t.Fatalf("an older agent cannot parse the file with markers: %v", err) |
| 490 | } |
| 491 | if older.Schema != 3 || len(older.Records) != 1 || older.Records[0].Build != "v2" { |
| 492 | t.Fatalf("older agent's view = %+v: want schema 3 and the one record", older) |
| 493 | } |
| 494 | if !strings.Contains(string(raw), `"hold_ran"`) { |
| 495 | t.Fatalf("the marker is not in the file:\n%s", raw) |
| 496 | } |
| 497 | } |
| 498 | |
| 499 | // Astra R3 N7: a marker never turns into permission to run a build a |
| 500 | // sibling's failure record holds. The instance has a good build, so it stays |
| 501 | // down, at the plan and at the exec of a plan made before the record. |
| 502 | func TestAMarkedBuildStillHonoursASiblingsFailureRecord(t *testing.T) { |
| 503 | ctx := context.Background() |
| 504 | h := newHarness(t, "h", "s") |
| 505 | h.stage("v2") |
| 506 | h.install("v2", "h", "s") |
| 507 | h.book.MarkHoldRan("h", markV2) |
| 508 | h.sleepHost("h") |
| 509 | h.wakeHost("h") |
| 510 | before := h.m.Plan(ctx, h.rows.get("h")) |
| 511 | if before.Refuse != "" || before.Boot.Version != "v2" || before.Mode != activator.ModeHold || before.Fallback != nil { |
| 512 | t.Fatalf("plan %+v: want a hold first wake of v2 while no record holds it", before) |
| 513 | } |
| 514 | h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "s", At: t0}) // a sibling's plain failure |
| 515 | if err := before.Check("v2"); err == nil { |
| 516 | t.Fatal("the earlier plan's check lets v2 boot after a sibling's record of it landed") |
| 517 | } |
| 518 | h.wakeHost("h") |
| 519 | if p := h.m.Plan(ctx, h.rows.get("h")); p.Refuse != HeldDownRefusal("v2") { |
| 520 | t.Fatalf("plan %+v: want the hold-down refusal for v2", p) |
| 521 | } |
| 522 | } |
| 523 | |
| 524 | // Ruling 205 (Astra R3): the installed good build is no proof of an explicit |
| 525 | // rollback, because root's automatic sibling repoint installs it too. Only a |
| 526 | // signed pointer naming the good build, newer than the marker (`carlos |
| 527 | // rollback` re-signs one), lifts the marker; the pointer pass does it. Until |
| 528 | // then the plan is refused, and so is the good build at the exec of any |
| 529 | // earlier plan. |
| 530 | func TestASignedRollbackClearsTheMarkerAndTheGoodBuildBoots(t *testing.T) { |
| 531 | ctx := context.Background() |
| 532 | h := newHarness(t, "h") |
| 533 | h.stage("v2") |
| 534 | h.sleepHost("h") |
| 535 | h.wakeHost("h") |
| 536 | early := h.m.Plan(ctx, h.rows.get("h")) |
| 537 | if early.Refuse != "" || early.Boot.Version != "v1" || early.FirstWake || early.Check == nil { |
| 538 | t.Fatalf("plan %+v: want the ordinary wake of the installed good v1 before any marker", early) |
| 539 | } |
| 540 | // Hold v2 ran on h, and root repointed h to its good build: v1 is |
| 541 | // installed, under a marker stamped at t0. v1's pointer predates it. |
| 542 | h.book.MarkHoldRan("h", markV2) |
| 543 | h.pointer("stable", "v1", t0.Add(-time.Hour)) |
| 544 | h.advance(time.Minute) |
| 545 | h.pass() |
| 546 | if _, ok := h.book.HoldRanOn("h"); !ok { |
| 547 | t.Fatal("a pointer older than the marker cleared it") |
| 548 | } |
| 549 | h.wakeHost("h") |
| 550 | if p := h.m.Plan(ctx, h.rows.get("h")); p.Refuse != HeldDownRefusal("v2") { |
| 551 | t.Fatalf("no rollback pointer: plan %+v, want refused", p) |
| 552 | } |
| 553 | if err := early.Check("v1"); err == nil || err.Error() != HeldDownRefusal("v2") { |
| 554 | t.Fatalf("the earlier plan's check on v1 = %v, want the hold-down refusal for v2", err) |
| 555 | } |
| 556 | |
| 557 | // `carlos rollback` signs v1's pointer afresh. |
| 558 | h.pointer("stable", "v1", h.clock()) |
| 559 | h.advance(pointerReadEvery) |
| 560 | h.pass() |
| 561 | if got, ok := h.book.HoldRanOn("h"); ok { |
| 562 | t.Fatalf("marker %+v survived a signed rollback newer than it", got) |
| 563 | } |
| 564 | h.wakeHost("h") |
| 565 | p := h.m.Plan(ctx, h.rows.get("h")) |
| 566 | if p.Refuse != "" || p.Boot.Version != "v1" || p.FirstWake { |
| 567 | t.Fatalf("after a signed rollback: plan %+v, want v1's ordinary wake", p) |
| 568 | } |
| 569 | if err := p.Check("v1"); err != nil { |
| 570 | t.Fatalf("after a signed rollback the check refuses v1: %v", err) |
| 571 | } |
| 572 | } |
| 573 | |
| 574 | // Ruling 205, in the order it happens (Task 3 fix round 1): hold build v2 is |
| 575 | // dispatched and running on h; `carlos rollback` signs v1 and root installs |
| 576 | // v1 on h; the attempt is superseded after v2 ran, and marks h. The marker's |
| 577 | // At is the attempt's dispatch, so the rollback signed while v2 ran is newer |
| 578 | // than it, and the pass after the mark clears it at once (setting the marker |
| 579 | // made the pointer read due; the pass that saw the install came before any |
| 580 | // marker). The sleeping h's next wake boots v1. |
| 581 | // |
| 582 | // A pointer naming v1 signed before the attempt started clears nothing: it |
| 583 | // says nothing about recovering from v2. |
| 584 | func TestAnInstallOntoTheGoodBuildMakesTheRollbackReadDueAtOnce(t *testing.T) { |
| 585 | ctx := context.Background() |
| 586 | for _, tc := range []struct { |
| 587 | name string |
| 588 | signedAt time.Duration // the v1 pointer's PromotedAt, from the attempt's dispatch |
| 589 | cleared bool |
| 590 | }{ |
| 591 | {"rollback signed while the hold build ran", 5 * time.Second, true}, |
| 592 | {"pointer signed before the hold build started", -time.Minute, false}, |
| 593 | } { |
| 594 | t.Run(tc.name, func(t *testing.T) { |
| 595 | h := newHarness(t, "h") |
| 596 | h.swap = appconfig.SwapHold |
| 597 | h.stage("v2", "v3") |
| 598 | h.act.mode = activator.ModeHold |
| 599 | h.act.executed = func(a activator.Attempt, _ activator.SwapOutcome) bool { return a.Build == "v2" } |
| 600 | // A record of another build keeps the app's pointers read; the |
| 601 | // first pass reads them, so no read is due for pointerReadEvery |
| 602 | // unless something makes it so. |
| 603 | h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v3", Host: "sib", PromotedAt: t0, At: t0}) |
| 604 | gate := h.act.hold("h") |
| 605 | h.install("v2", "h") |
| 606 | h.pass() |
| 607 | waitFor(t, "h's hold attempt onto v2", func() bool { |
| 608 | _, started, _ := h.act.snapshot() |
| 609 | return len(started) == 1 && started[0] == "h:v2" |
| 610 | }) |
| 611 | start := h.clock() |
| 612 | |
| 613 | h.advance(5 * time.Second) |
| 614 | h.pointer("stable", "v1", start.Add(tc.signedAt)) // `carlos rollback` to v1 (or an older pointer) |
| 615 | h.install("v1", "h") // root installs it |
| 616 | h.pass() // the newer desire supersedes the attempt |
| 617 | h.advance(5 * time.Second) |
| 618 | close(gate) |
| 619 | waitFor(t, "h's attempt to finish", func() bool { return !h.m.attemptRunning("h") }) |
| 620 | got, ok := h.book.HoldRanOn("h") |
| 621 | if !ok || !got.same(markV2) || !got.At.Equal(start) { |
| 622 | t.Fatalf("marker = %+v, %v: want v2's, stamped at the attempt's dispatch %v", got, ok, start) |
| 623 | } |
| 624 | |
| 625 | h.sleepHost("h") |
| 626 | h.pass() |
| 627 | if _, still := h.book.HoldRanOn("h"); still == tc.cleared { |
| 628 | t.Fatalf("after the pass the marker stands: %v, want %v", still, !tc.cleared) |
| 629 | } |
| 630 | h.wakeHost("h") |
| 631 | p := h.m.Plan(ctx, h.rows.get("h")) |
| 632 | if tc.cleared { |
| 633 | if p.Refuse != "" || p.Boot.Version != "v1" || p.FirstWake { |
| 634 | t.Fatalf("plan %+v after the rollback: want v1's ordinary wake", p) |
| 635 | } |
| 636 | if err := p.Check("v1"); err != nil { |
| 637 | t.Fatalf("after the rollback the check refuses v1: %v", err) |
| 638 | } |
| 639 | return |
| 640 | } |
| 641 | if p.Refuse != HeldDownRefusal("v2") { |
| 642 | t.Fatalf("plan %+v with only an older pointer: want refused", p) |
| 643 | } |
| 644 | }) |
| 645 | } |
| 646 | } |
| 647 | |
| 648 | // Ruling 205 (Task 3 fix round 1): a planned wake's marker is stamped with |
| 649 | // its own plan's time, the start of its execution, kept per token: a newer |
| 650 | // plan for the host does not change an older wake's start. A wake whose |
| 651 | // start is not known (no token) falls back to the book's clock, the safe |
| 652 | // direction. |
| 653 | func TestAWakesHoldRanMarkerIsStampedWithItsPlanTime(t *testing.T) { |
| 654 | ctx := context.Background() |
| 655 | h := newHarness(t, "s") |
| 656 | h.swap = appconfig.SwapHold |
| 657 | h.stage("v2") |
| 658 | h.install("v2", "s") |
| 659 | h.sleepHost("s") |
| 660 | h.pass() |
| 661 | h.wakeHost("s") |
| 662 | first := h.m.Plan(ctx, h.rows.get("s")) |
| 663 | if !first.FirstWake || first.Boot.Version != "v2" || first.Mode != activator.ModeHold { |
| 664 | t.Fatalf("plan %+v: want a hold first wake of v2", first) |
| 665 | } |
| 666 | planned := h.clock() |
| 667 | h.advance(time.Minute) |
| 668 | h.wakeHost("s") |
| 669 | if second := h.m.Plan(ctx, h.rows.get("s")); second.Token == first.Token || second.Boot.Version != "v2" { |
| 670 | t.Fatalf("second plan %+v: want another plan of v2", second) |
| 671 | } |
| 672 | h.advance(time.Minute) |
| 673 | h.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: h.rows.get("s"), Gen: activator.Gen{Build: "v2"}, |
| 674 | Mode: activator.ModeHold, FirstWake: true, Token: first.Token, Executed: true}) |
| 675 | if got, ok := h.book.HoldRanOn("s"); !ok || !got.same(markV2) || !got.At.Equal(planned) { |
| 676 | t.Fatalf("marker = %+v, %v: want v2's, stamped at its wake's plan %v", got, ok, planned) |
| 677 | } |
| 678 | |
| 679 | h.book.ClearHoldRan("s") |
| 680 | h.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: h.rows.get("s"), Gen: activator.Gen{Build: "v2"}, |
| 681 | Mode: activator.ModeHold, FirstWake: true, Executed: true}) |
| 682 | if got, ok := h.book.HoldRanOn("s"); !ok || !got.At.Equal(h.clock()) { |
| 683 | t.Fatalf("marker = %+v, %v: a wake of unknown start, want the book's clock %v", got, ok, h.clock()) |
| 684 | } |
| 685 | } |
| 686 | |
| 687 | // Ruling 205's book half: a marker keeps the At its caller gives (the |
| 688 | // execution's start), and one given none is stamped by the book's clock; a |
| 689 | // repeated report of the same execution keeps the first stamp, and writes |
| 690 | // nothing (another execution renews it: bookseq_test.go). The rollback |
| 691 | // clear lifts only (acct, app)'s marker, and only for a pointer newer than |
| 692 | // it. |
| 693 | func TestTheHoldRanMarkerStampAndItsRollbackClear(t *testing.T) { |
| 694 | now := t0 |
| 695 | b := &FailureBook{now: func() time.Time { return now }} |
| 696 | one := markV2 |
| 697 | one.Exec = "a1" |
| 698 | b.MarkHoldRan("a", one) |
| 699 | now = now.Add(time.Minute) |
| 700 | b.MarkHoldRan("a", one) |
| 701 | if got, _ := b.HoldRanOn("a"); !got.At.Equal(t0) { |
| 702 | t.Fatalf("marker At %v after a repeated report of its execution, want %v", got.At, t0) |
| 703 | } |
| 704 | if ok, _ := b.ClearHoldRanOnPointer("a", "bab", "blog", t0); ok { |
| 705 | t.Fatal("a pointer as old as the marker cleared it") |
| 706 | } |
| 707 | if ok, _ := b.ClearHoldRanOnPointer("a", "bab", "other", t0.Add(time.Second)); ok { |
| 708 | t.Fatal("another app's pointer cleared the marker") |
| 709 | } |
| 710 | if ok, _ := b.ClearHoldRanOnPointer("a", "bab", "blog", t0.Add(time.Second)); !ok { |
| 711 | t.Fatal("a newer pointer left the marker standing") |
| 712 | } |
| 713 | b.MarkHoldRan("a", HoldRan{Account: "bab", App: "blog", Build: "v3"}) |
| 714 | if got, _ := b.HoldRanOn("a"); got.Build != "v3" || !got.At.Equal(now) { |
| 715 | t.Fatalf("marker %+v: want v3's, stamped at %v", got, now) |
| 716 | } |
| 717 | // A report of an execution that started before the standing marker's |
| 718 | // (Task 4 fix round 1) names its build but keeps the later stamp: At |
| 719 | // never moves back. |
| 720 | started := now.Add(-30 * time.Second) |
| 721 | b.MarkHoldRan("a", HoldRan{Account: "bab", App: "blog", Build: "v4", At: started}) |
| 722 | if got, _ := b.HoldRanOn("a"); got.Build != "v4" || !got.At.Equal(now) { |
| 723 | t.Fatalf("marker %+v: want v4's, keeping the later stamp %v", got, now) |
| 724 | } |
| 725 | b.ClearHoldRan("a") |
| 726 | b.MarkHoldRan("a", HoldRan{Account: "bab", App: "blog", Build: "v4", At: started}) |
| 727 | if got, _ := b.HoldRanOn("a"); got.Build != "v4" || !got.At.Equal(started) { |
| 728 | t.Fatalf("marker %+v: want v4's, stamped at its execution's start %v, not the book's clock", got, started) |
| 729 | } |
| 730 | } |
| 731 | |
| 732 | // Ruling 205 at the scheduler (Task 3 fix round 1): an awake instance (a |
| 733 | // socket instance, or a worker) serving v2 under v2's hold-ran marker, with |
| 734 | // its good v1 installed (root's repoint, say), is not moved onto v1 until a |
| 735 | // signed rollback lifts the marker. The gated attempt ends abandoned with |
| 736 | // nothing run, is derived again after the abandon deferral, and is gated |
| 737 | // again; once the pointer pass sees `carlos rollback`'s pointer and clears |
| 738 | // the marker, the next derivation runs the move. |
| 739 | func TestAMoveOntoTheInstalledGoodBuildWaitsForASignedRollback(t *testing.T) { |
| 740 | for _, worker := range []bool{false, true} { |
| 741 | name := "socket instance" |
| 742 | if worker { |
| 743 | name = "worker" |
| 744 | } |
| 745 | t.Run(name, func(t *testing.T) { |
| 746 | var h *harness |
| 747 | if worker { |
| 748 | h = workerHarness(t, "h") |
| 749 | } else { |
| 750 | h = newHarness(t, "h") |
| 751 | } |
| 752 | var ( |
| 753 | mu sync.Mutex |
| 754 | gated int |
| 755 | ) |
| 756 | h.m.d.Logf = func(format string, args ...any) { |
| 757 | if strings.Contains(fmt.Sprintf(format, args...), "waiting for a signed rollback") { |
| 758 | mu.Lock() |
| 759 | gated++ |
| 760 | mu.Unlock() |
| 761 | } |
| 762 | } |
| 763 | gatedTimes := func() int { |
| 764 | mu.Lock() |
| 765 | defer mu.Unlock() |
| 766 | return gated |
| 767 | } |
| 768 | moves := func() int { |
| 769 | calls, _, _ := h.act.snapshot() |
| 770 | return len(calls) + len(h.act.replacements()) |
| 771 | } |
| 772 | h.stage("v2") |
| 773 | h.act.serve("h", "v2") |
| 774 | h.book.MarkHoldRan("h", HoldRan{Account: "bab", App: "blog", Build: "v2", At: t0}) |
| 775 | h.pass() |
| 776 | waitFor(t, "the move onto v1 to be gated", func() bool { return gatedTimes() == 1 && !h.m.attemptRunning("h") }) |
| 777 | if n := moves(); n != 0 { |
| 778 | t.Fatalf("%d Swap/ReplaceWorker calls while v2's marker stands, want none", n) |
| 779 | } |
| 780 | // After the abandon deferral it is derived again, and gated again. |
| 781 | h.advance(abandonDeferral + time.Second) |
| 782 | h.pass() |
| 783 | waitFor(t, "the move onto v1 to be gated again", func() bool { return gatedTimes() == 2 && !h.m.attemptRunning("h") }) |
| 784 | if n := moves(); n != 0 { |
| 785 | t.Fatalf("%d Swap/ReplaceWorker calls while v2's marker stands, want none", n) |
| 786 | } |
| 787 | |
| 788 | // `carlos rollback` signs v1; the pointer pass clears the marker, |
| 789 | // and the next derivation moves h onto v1. |
| 790 | h.pointer("stable", "v1", h.clock()) |
| 791 | h.advance(pointerReadEvery) |
| 792 | h.pass() |
| 793 | if got, ok := h.book.HoldRanOn("h"); ok { |
| 794 | t.Fatalf("marker %+v survived the signed rollback", got) |
| 795 | } |
| 796 | waitFor(t, "the move onto v1", func() bool { return moves() == 1 }) |
| 797 | if calls, _, _ := h.act.snapshot(); len(calls) == 1 && (worker || calls[0].a.Build != "v1") { |
| 798 | t.Fatalf("swap %+v: want one onto v1 (and none for a worker)", calls[0].a) |
| 799 | } |
| 800 | if r := h.act.replacements(); len(r) == 1 && (!worker || r[0].a.Build != "v1") { |
| 801 | t.Fatalf("replacement %+v: want one onto v1 (and none for a socket instance)", r[0].a) |
| 802 | } |
| 803 | }) |
| 804 | } |
| 805 | } |
| 806 | |
amadan