| 1 | //go:build linux |
| 2 | |
| 3 | package activator |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "crypto/sha256" |
| 8 | "encoding/hex" |
| 9 | "errors" |
| 10 | "os" |
| 11 | "os/exec" |
| 12 | "path/filepath" |
| 13 | "slices" |
| 14 | "strings" |
| 15 | "sync" |
| 16 | "testing" |
| 17 | "time" |
| 18 | |
| 19 | "github.com/carlosframework/platform/internal/hostbox" |
| 20 | "github.com/carlosframework/platform/internal/hostclaim" |
| 21 | "github.com/carlosframework/platform/internal/hostwrite" |
| 22 | "github.com/carlosframework/platform/internal/registry" |
| 23 | "github.com/carlosframework/platform/internal/release" |
| 24 | ) |
| 25 | |
| 26 | // The claimed (CARLOS_HOST_CLAIM) path records its build before every exec |
| 27 | // (settings spec §4.2, settings plan 4 Task 5d). These tests launch through |
| 28 | // the root launch fixture (newRootLaunchFixture), so startAdmitted's |
| 29 | // beforeStart really runs inside RoleLaunch.Start, immediately before root's |
| 30 | // Launch: f.root.launches is the exec count. |
| 31 | |
| 32 | // claimedBuildOf is the staged identity of the fixture's executable, as one |
| 33 | // resolution of the live link names it. |
| 34 | func claimedBuildOf(t *testing.T, f *rootLaunchFixture, ver string) release.StagedIdentity { |
| 35 | t.Helper() |
| 36 | raw, err := os.ReadFile(f.exe) |
| 37 | if err != nil { |
| 38 | t.Fatal(err) |
| 39 | } |
| 40 | sum := sha256.Sum256(raw) |
| 41 | return release.StagedIdentity{Version: ver, StagedPath: f.exe, Digest: hex.EncodeToString(sum[:])} |
| 42 | } |
| 43 | |
| 44 | // rootLaunched is s launching through f: the launcher, its attempt and its |
| 45 | // command, the staged identity the snapshot hashed (v1) and the version it |
| 46 | // read (v1). |
| 47 | func rootLaunched(t *testing.T, f *rootLaunchFixture, s ClaimStart) ClaimStart { |
| 48 | t.Helper() |
| 49 | s.RoleLauncher, s.PrepareLaunch = &f.launch, nil |
| 50 | s.CurrentAttempt = func() hostclaim.WakeRetryAttempt { return f.attempt } |
| 51 | s.Build = f.build |
| 52 | s.Staged, s.Version = claimedBuildOf(t, f, "v1"), "v1" |
| 53 | if s.OpenProbe != nil { |
| 54 | // A probe's window opens at the start the journal was offered. |
| 55 | s.OpenProbe = func(_ context.Context, at time.Time) (time.Time, error) { return at, nil } |
| 56 | } |
| 57 | return s |
| 58 | } |
| 59 | |
| 60 | // launchedChild grants a launch on a real child process (sleep), so a start |
| 61 | // that root granted can be stopped and its exit watched; the child is |
| 62 | // killed when the test ends. |
| 63 | func launchedChild(t *testing.T) func(hostbox.RoleLaunchRequest) (hostbox.RoleLaunchGrant, error) { |
| 64 | return func(q hostbox.RoleLaunchRequest) (hostbox.RoleLaunchGrant, error) { |
| 65 | child := exec.Command("sleep", "30") |
| 66 | if err := child.Start(); err != nil { |
| 67 | return hostbox.RoleLaunchGrant{}, err |
| 68 | } |
| 69 | t.Cleanup(func() { _ = child.Process.Kill(); _ = child.Wait() }) |
| 70 | start, err := roleProcessStart(child.Process.Pid) |
| 71 | if err != nil { |
| 72 | return hostbox.RoleLaunchGrant{}, err |
| 73 | } |
| 74 | return hostbox.RoleLaunchGrant{ReservationID: q.Reservation.ID, Unit: "carlos-instance@tenant.test.service", UnitInvocationID: "unit-1", PID: child.Process.Pid, ProcessStart: start, |
| 75 | Group: hostbox.LaunchGroup{RelativePath: q.Reservation.Group.RelativePath, Device: 3, Inode: 4, InvocationID: "unit-1"}}, nil |
| 76 | } |
| 77 | } |
| 78 | |
| 79 | // claimedRecord is a claims-on Service whose Prepare hands startAdmitted a |
| 80 | // start launching through the root fixture. |
| 81 | type claimedRecord struct { |
| 82 | a *Service |
| 83 | f *rootLaunchFixture |
| 84 | obs hostclaim.Observed |
| 85 | log *execLog |
| 86 | mu sync.Mutex |
| 87 | reserved int |
| 88 | failed int |
| 89 | restored int |
| 90 | } |
| 91 | |
| 92 | func newClaimedRecord(t *testing.T, mutate func(*ClaimStart)) *claimedRecord { |
| 93 | t.Helper() |
| 94 | f := newRootLaunchFixture(t) |
| 95 | c := &claimedRecord{f: f, obs: claimedObs("tenant.test")} |
| 96 | c.log = &execLog{spawns: func() int { f.root.mu.Lock(); defer f.root.mu.Unlock(); return len(f.root.launches) }} |
| 97 | work, _ := (hostwrite.Config{}).Select(t.Context(), c.obs.Host, "spawn") |
| 98 | c.a = &Service{cfg: Config{HostWrites: hostwrite.Config{Enabled: true}, Claims: &ClaimRuntime{Box: "box", Bucket: "data", |
| 99 | Read: func(context.Context, string) (hostclaim.Observed, error) { return c.obs, nil }, |
| 100 | Prepare: func(context.Context, registry.Route, hostclaim.Observed, startPolicy) (ClaimStart, error) { |
| 101 | s := rootLaunched(t, f, claimedStartFor(c.obs, work, "v1")) |
| 102 | s.Reserve = func(context.Context, int) error { c.mu.Lock(); c.reserved++; c.mu.Unlock(); return nil } |
| 103 | s.Failed = func(context.Context, int) error { c.mu.Lock(); c.failed++; c.mu.Unlock(); return nil } |
| 104 | s.Restore = func(context.Context) error { c.mu.Lock(); c.restored++; c.mu.Unlock(); return nil } |
| 105 | s.Replicate = func(context.Context, string) (Rep, error) { return &fakeRep{}, nil } |
| 106 | if mutate != nil { |
| 107 | mutate(&s) |
| 108 | } |
| 109 | return s, nil |
| 110 | }}}} |
| 111 | c.a.BeforeExec = c.log.hook |
| 112 | return c |
| 113 | } |
| 114 | |
| 115 | func (c *claimedRecord) counts() (reserved, failed, restored, launches int) { |
| 116 | c.mu.Lock() |
| 117 | reserved, failed, restored = c.reserved, c.failed, c.restored |
| 118 | c.mu.Unlock() |
| 119 | c.f.root.mu.Lock() |
| 120 | defer c.f.root.mu.Unlock() |
| 121 | return reserved, failed, restored, len(c.f.root.launches) |
| 122 | } |
| 123 | |
| 124 | func (c *claimedRecord) start(t *testing.T, e *actEntry, maxStarts int) error { |
| 125 | t.Helper() |
| 126 | return c.a.startAdmitted(t.Context(), e, c.obs, startPolicy{MaxStarts: maxStarts, Window: 30 * time.Second}) |
| 127 | } |
| 128 | |
| 129 | // One record of the claimed build, at the claim holder's Location in the |
| 130 | // claim actor's bucket, before each exec. A start that failed for any other |
| 131 | // reason is retried, and the retry is a new exec: it records again. |
| 132 | func TestAClaimedStartRecordsBeforeEachExec(t *testing.T) { |
| 133 | t.Parallel() |
| 134 | c := newClaimedRecord(t, nil) |
| 135 | launches := 0 |
| 136 | granted := launchedChild(t) |
| 137 | c.f.root.grant = func(q hostbox.RoleLaunchRequest) (hostbox.RoleLaunchGrant, error) { |
| 138 | if launches++; launches == 1 { |
| 139 | return hostbox.RoleLaunchGrant{}, errors.New("connection reset") |
| 140 | } |
| 141 | return granted(q) |
| 142 | } |
| 143 | e := &actEntry{host: c.obs.Host} |
| 144 | if err := c.start(t, e, 2); err != nil { |
| 145 | t.Fatalf("startAdmitted = %v", err) |
| 146 | } |
| 147 | calls, at, locs := c.log.seen() |
| 148 | if !slices.Equal(calls, []string{"tenant.test@v1", "tenant.test@v1"}) || !slices.Equal(at, []int{0, 1}) { |
| 149 | t.Fatalf("records %v at exec counts %v, want v1 before each of the two execs", calls, at) |
| 150 | } |
| 151 | want := BuildStateLocation{Bucket: "data", Prefix: c.obs.Claim.Holder.Location} |
| 152 | for _, l := range locs { |
| 153 | if l != want { |
| 154 | t.Fatalf("recorded at %+v, want beside the claimed lease %+v (never a g-<gen>/ prefix)", l, want) |
| 155 | } |
| 156 | } |
| 157 | if _, failed, _, n := c.counts(); failed != 1 || n != 2 { |
| 158 | t.Fatalf("failed=%d launches=%d, want the first start failed and retried", failed, n) |
| 159 | } |
| 160 | // The identity it ran rides on the generation, so a legacy sidecar |
| 161 | // restart runs the same file (Task 5d, C). |
| 162 | g := e.claimResult.gen |
| 163 | id := claimedBuildOf(t, c.f, "v1") |
| 164 | if g.Build != "v1" || g.StagedPath != id.StagedPath || g.Digest != id.Digest { |
| 165 | t.Fatalf("generation %+v, want the hashed v1 by path and digest", g) |
| 166 | } |
| 167 | } |
| 168 | |
| 169 | // A refused record is no failed attempt of the build (ruling |
| 170 | // R-EXEC-CLAIMRETRY): nothing execs, the claim's WakeRetry is not written |
| 171 | // (s.Failed), and the loop does not retry; the wake fails as fenced. |
| 172 | func TestAClaimedRecordRefusalIsTerminal(t *testing.T) { |
| 173 | c := newClaimedRecord(t, nil) |
| 174 | c.log.fail = failAll |
| 175 | began := time.Now() |
| 176 | err := c.start(t, &actEntry{host: c.obs.Host}, 5) |
| 177 | if !errors.Is(err, errBuildStateUnwritten) || errors.Is(err, errClaimsAdmission) { |
| 178 | t.Fatalf("startAdmitted = %v, want the record's refusal, not the admission's", err) |
| 179 | } |
| 180 | if got := memberSafeWakeReason(err); got != memberSafeWakeReason(ErrFenced) { |
| 181 | t.Fatalf("member reads %q, want the fence's wording", got) |
| 182 | } |
| 183 | reserved, failed, _, launches := c.counts() |
| 184 | if launches != 0 || failed != 0 || reserved != 1 { |
| 185 | t.Fatalf("launches=%d failed=%d reserved=%d, want one attempt, nothing execed, no failed attempt", launches, failed, reserved) |
| 186 | } |
| 187 | if calls, _, _ := c.log.seen(); len(calls) != 1 { |
| 188 | t.Fatalf("records %v, want one refused record and no retry", calls) |
| 189 | } |
| 190 | if took := time.Since(began); took >= time.Second { |
| 191 | t.Fatalf("took %v: the refusal waited out a retry backoff", took) |
| 192 | } |
| 193 | } |
| 194 | |
| 195 | // Ruling R-EXEC-CLAIMVERSION: the record names the version of the bytes the |
| 196 | // snapshot hashed. A start whose hashed path names v1 while its version |
| 197 | // read says v2 (a repoint between the hash and the read) is refused before |
| 198 | // anything touches the database or execs, and says why. |
| 199 | func TestAClaimedVersionMismatchIsRefused(t *testing.T) { |
| 200 | c := newClaimedRecord(t, func(s *ClaimStart) { s.Version = "v2" }) |
| 201 | err := c.start(t, &actEntry{host: c.obs.Host}, 5) |
| 202 | if !errors.Is(err, errBuildStateUnwritten) || !strings.Contains(err.Error(), "v1") || !strings.Contains(err.Error(), "v2") { |
| 203 | t.Fatalf("startAdmitted = %v, want a refusal naming v1 and v2", err) |
| 204 | } |
| 205 | if _, failed, restored, launches := c.counts(); launches != 0 || failed != 0 || restored != 0 { |
| 206 | t.Fatalf("launches=%d failed=%d restored=%d, want nothing run", launches, failed, restored) |
| 207 | } |
| 208 | if calls, _, _ := c.log.seen(); len(calls) != 0 { |
| 209 | t.Fatalf("records %v, want none", calls) |
| 210 | } |
| 211 | // With no hook (a box with no replica bucket) the start is today's. |
| 212 | c = newClaimedRecord(t, func(s *ClaimStart) { s.Version = "v2" }) |
| 213 | c.a.BeforeExec = nil |
| 214 | c.f.root.grant = func(hostbox.RoleLaunchRequest) (hostbox.RoleLaunchGrant, error) { |
| 215 | return hostbox.RoleLaunchGrant{}, ErrAttemptOver |
| 216 | } |
| 217 | if err := c.start(t, &actEntry{host: c.obs.Host}, 1); !errors.Is(err, ErrAttemptOver) { |
| 218 | t.Fatalf("no hook: startAdmitted = %v, want today's launch", err) |
| 219 | } |
| 220 | } |
| 221 | |
| 222 | // Ruling R-EXEC-PROBES: a moves probe runs a build against the moved |
| 223 | // database through the same beforeStart, so it records too, beside the |
| 224 | // holder's lease. Both entries: ProbeWake (a landing) and RecoveryProbe. |
| 225 | func TestAClaimedProbeRecords(t *testing.T) { |
| 226 | refused := func(hostbox.RoleLaunchRequest) (hostbox.RoleLaunchGrant, error) { |
| 227 | return hostbox.RoleLaunchGrant{}, ErrAttemptOver |
| 228 | } |
| 229 | t.Run("ProbeWake", func(t *testing.T) { |
| 230 | r := &pwRuntime{observed: pwFlip()} |
| 231 | a, _ := pwService(t, r) |
| 232 | f := newRootLaunchFixture(t) |
| 233 | f.launch.Target.Host = pwHost |
| 234 | f.root.grant = refused |
| 235 | a.cfg.Claims.Bucket = "data" |
| 236 | prepare := a.cfg.Claims.Prepare |
| 237 | a.cfg.Claims.Prepare = func(ctx context.Context, rt registry.Route, o hostclaim.Observed, p startPolicy) (ClaimStart, error) { |
| 238 | s, err := prepare(ctx, rt, o, p) |
| 239 | return rootLaunched(t, f, s), err |
| 240 | } |
| 241 | l := &execLog{spawns: func() int { f.root.mu.Lock(); defer f.root.mu.Unlock(); return len(f.root.launches) }} |
| 242 | a.BeforeExec = l.hook |
| 243 | if _, err := a.ProbeWake(t.Context(), pwConfig()); !errors.Is(err, ErrProbeFailed) || !errors.Is(err, ErrAttemptOver) { |
| 244 | t.Fatalf("err = %v", err) |
| 245 | } |
| 246 | calls, at, locs := l.seen() |
| 247 | want := BuildStateLocation{Bucket: "data", Prefix: r.admitted().Claim.Holder.Location} |
| 248 | if !slices.Equal(calls, []string{pwHost + "@v1"}) || at[0] != 0 || locs[0] != want { |
| 249 | t.Fatalf("records %v at exec counts %v, %+v; want v1 before the exec at %+v", calls, at, locs, want) |
| 250 | } |
| 251 | }) |
| 252 | t.Run("RecoveryProbe", func(t *testing.T) { |
| 253 | obs := probeObserved(probeClaim(hostclaim.KindRestoring, hostclaim.PhaseProbing, "box")) |
| 254 | a, r, _ := probeService(t, obs) |
| 255 | f := newRootLaunchFixture(t) |
| 256 | f.launch.Target.Host = probeHost |
| 257 | f.root.grant = refused |
| 258 | r.Bucket = "data" |
| 259 | prepare := r.Prepare |
| 260 | r.Prepare = func(ctx context.Context, rt registry.Route, o hostclaim.Observed, p startPolicy) (ClaimStart, error) { |
| 261 | s, err := prepare(ctx, rt, o, p) |
| 262 | return rootLaunched(t, f, s), err |
| 263 | } |
| 264 | l := &execLog{spawns: func() int { f.root.mu.Lock(); defer f.root.mu.Unlock(); return len(f.root.launches) }} |
| 265 | a.BeforeExec = l.hook |
| 266 | if err := a.RecoveryProbe(t.Context(), registry.Route{Host: probeHost}, probePolicy(probeInputs())); !errors.Is(err, ErrAttemptOver) { |
| 267 | t.Fatalf("err = %v", err) |
| 268 | } |
| 269 | calls, at, locs := l.seen() |
| 270 | want := BuildStateLocation{Bucket: "data", Prefix: obs.Claim.Holder.Location} |
| 271 | if !slices.Equal(calls, []string{probeHost + "@v1"}) || at[0] != 0 || locs[0] != want { |
| 272 | t.Fatalf("records %v at exec counts %v, %+v; want v1 before the exec at %+v", calls, at, locs, want) |
| 273 | } |
| 274 | }) |
| 275 | } |
| 276 | |
| 277 | // A claimed instance's legacy sidecar restart (Task 5d, C) runs the file its |
| 278 | // primary ran, by path, and records that build first. It names no location |
| 279 | // on its context: with claims on the agent locates the host through its |
| 280 | // claim's Holder.Location (internal/host buildStates.locate, ruling |
| 281 | // R-CLAIMPREFIX-2), which is the claimed lease's. |
| 282 | func TestAClaimedSidecarRestartRecordsTheBuildItsPrimaryRan(t *testing.T) { |
| 283 | t.Parallel() |
| 284 | claimedSidecarRestart(t, false) |
| 285 | } |
| 286 | |
| 287 | // The same for a pinned claimed start (Task 12a): every claimed wake is |
| 288 | // pinned once a planner is on, and with no generation number the pin's |
| 289 | // start must still hand the sidecar the file it ran, by path, with the |
| 290 | // pin's demand. Prepare's start was checked against the pin (s.Version is |
| 291 | // Pin.Build), and s.Staged is what actually ran. |
| 292 | func TestAPinnedClaimedSidecarRestartRunsTheClaimedFile(t *testing.T) { |
| 293 | t.Parallel() |
| 294 | claimedSidecarRestart(t, true) |
| 295 | } |
| 296 | |
| 297 | func claimedSidecarRestart(t *testing.T, pinned bool) { |
| 298 | t.Helper() |
| 299 | f := newRootLaunchFixture(t) |
| 300 | f.root.grant = launchedChild(t) |
| 301 | host := f.launch.Target.Host |
| 302 | obs := claimedObs(host) |
| 303 | obs.Claim.Holder.WriterBox = "box" |
| 304 | admitted := probeAdmitted(obs) |
| 305 | work, _ := (hostwrite.Config{}).Select(t.Context(), host, "spawn") |
| 306 | var mu sync.Mutex |
| 307 | reads := 0 |
| 308 | b := newPlanGenBackend() |
| 309 | b.sidecarStaged = true |
| 310 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 311 | if err != nil { |
| 312 | t.Fatal(err) |
| 313 | } |
| 314 | a := New(Config{Idle: time.Hour, HostWrites: hostwrite.Config{Enabled: true}, Claims: &ClaimRuntime{Box: "box", Bucket: "data", |
| 315 | Read: func(context.Context, string) (hostclaim.Observed, error) { |
| 316 | mu.Lock() |
| 317 | defer mu.Unlock() |
| 318 | if reads++; reads > 1 { |
| 319 | return admitted, nil |
| 320 | } |
| 321 | return obs, nil |
| 322 | }, |
| 323 | StopForeign: func(context.Context, string) error { return nil }, |
| 324 | Prerequisites: func(context.Context, registry.Route, hostclaim.Observed, startPolicy) error { return nil }, |
| 325 | Admit: func(context.Context, registry.Route, hostclaim.Observed, startPolicy) (hostclaim.Observed, error) { |
| 326 | return admitted, nil |
| 327 | }, |
| 328 | FailedAdmission: func(context.Context, hostclaim.Observed, error) error { return nil }, |
| 329 | Prepare: func(context.Context, registry.Route, hostclaim.Observed, startPolicy) (ClaimStart, error) { |
| 330 | s := rootLaunched(t, f, claimedStartFor(admitted, work, "v1")) |
| 331 | s.Replicate = func(context.Context, string) (Rep, error) { return &fakeRep{}, nil } |
| 332 | return s, nil |
| 333 | }}}, b, reg) |
| 334 | l := &execLog{} |
| 335 | a.BeforeExec = l.hook |
| 336 | id := claimedBuildOf(t, f, "v1") |
| 337 | wantDemand := "" |
| 338 | if pinned { |
| 339 | boot := id |
| 340 | boot.Demand, wantDemand = "hold", "hold" |
| 341 | a.Planner = func(context.Context, registry.Route) WakePlan { |
| 342 | return WakePlan{Boot: boot, Mode: ModeHold} |
| 343 | } |
| 344 | } |
| 345 | rt := registry.Route{Host: host, Kind: "instance", Hibernate: true, Socket: filepath.Join(f.runDir, "app.sock"), DBPath: filepath.Join(f.launchDir, "app.db")} |
| 346 | done, err := a.beginRequest(t.Context(), rt) |
| 347 | if err != nil { |
| 348 | t.Fatalf("the claimed wake: %v", err) |
| 349 | } |
| 350 | done() |
| 351 | g, ok := a.entryGen(host) |
| 352 | if !ok || g.Build != "v1" || g.StagedPath != id.StagedPath || g.Digest != id.Digest || g.Demand != wantDemand { |
| 353 | t.Fatalf("serving %+v (%v), want the hashed v1 by path and digest, demand %q", g, ok, wantDemand) |
| 354 | } |
| 355 | a.sweepOnce() |
| 356 | waitFor(t, "the sidecar start", func() bool { return len(b.sidecarsOn()) == 1 && a.sidecarSettled(host) }) |
| 357 | if sc := b.sidecarsOn(); sc[0].StagedPath != id.StagedPath || sc[0].Digest != id.Digest || sc[0].Build != "v1" || b.sidecarStartCount() != 0 { |
| 358 | t.Fatalf("sidecar on %+v (%d live-link starts), want the primary's v1 by path", sc[0], b.sidecarStartCount()) |
| 359 | } |
| 360 | calls, _, locs := l.seen() |
| 361 | if !slices.Equal(calls, []string{host + "@v1", host + "@v1"}) { |
| 362 | t.Fatalf("records %v, want the primary's v1, then the sidecar's v1", calls) |
| 363 | } |
| 364 | if locs[0] != (BuildStateLocation{Bucket: "data", Prefix: admitted.Claim.Holder.Location}) || locs[1] != (BuildStateLocation{}) { |
| 365 | t.Fatalf("locations %+v: want the primary's at the claimed lease, the sidecar's left to the claim read", locs) |
| 366 | } |
| 367 | } |
| 368 | |
| 369 | // The claimed exec asks its attempt gate again after the record, as |
| 370 | // startSidecar does (Task 12a): a fence raised or a lease lost while the |
| 371 | // build state was being written (the S3 PUT) execs nothing. |
| 372 | func TestAClaimedStartExecsOnlyIfStillGatedAfterItsRecord(t *testing.T) { |
| 373 | c := newClaimedRecord(t, nil) |
| 374 | var mu sync.Mutex |
| 375 | fenced, gates := false, 0 |
| 376 | record := c.log.hook |
| 377 | c.a.BeforeExec = func(ctx context.Context, host, build string) error { |
| 378 | err := record(ctx, host, build) |
| 379 | mu.Lock() |
| 380 | fenced = true // the fence lands during the PUT |
| 381 | mu.Unlock() |
| 382 | return err |
| 383 | } |
| 384 | gate := func() error { |
| 385 | mu.Lock() |
| 386 | defer mu.Unlock() |
| 387 | gates++ |
| 388 | if fenced { |
| 389 | return ErrFenced |
| 390 | } |
| 391 | return nil |
| 392 | } |
| 393 | err := c.a.startAdmitted(t.Context(), &actEntry{host: c.obs.Host}, c.obs, startPolicy{MaxStarts: 1, Window: 30 * time.Second, Gate: gate}) |
| 394 | if !errors.Is(err, ErrFenced) { |
| 395 | t.Fatalf("startAdmitted = %v, want the fence raised during the record", err) |
| 396 | } |
| 397 | if calls, _, _ := c.log.seen(); len(calls) != 1 { |
| 398 | t.Fatalf("records %v, want the one record before the fence", calls) |
| 399 | } |
| 400 | mu.Lock() |
| 401 | asked := gates |
| 402 | mu.Unlock() |
| 403 | if _, _, _, launches := c.counts(); launches != 0 || asked != 2 { |
| 404 | t.Fatalf("launches=%d gates=%d, want the gate asked before and after the record and nothing execed", launches, asked) |
| 405 | } |
| 406 | } |
| 407 | |
amadan