| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "runtime" |
| 7 | "slices" |
| 8 | "sync/atomic" |
| 9 | "testing" |
| 10 | "time" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/appconfig" |
| 13 | "github.com/carlosframework/platform/internal/registry" |
| 14 | "github.com/carlosframework/platform/internal/release" |
| 15 | "github.com/carlosframework/platform/internal/telemetry" |
| 16 | ) |
| 17 | |
| 18 | // The box's GET loops read only on change or at the backstop age (S3 |
| 19 | // change-index design §5, codex 9): config-gen.json, canary channel |
| 20 | // pointers and telemetry.json. |
| 21 | |
| 22 | // gateClock is a fake clock a test moves by hand. |
| 23 | type gateClock struct{ t time.Time } |
| 24 | |
| 25 | func (c *gateClock) now() time.Time { return c.t } |
| 26 | func (c *gateClock) add(d time.Duration) { c.t = c.t.Add(d) } |
| 27 | func newGateClock() *gateClock { return &gateClock{t: time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC)} } |
| 28 | func pollWatch(t *testing.T, w *release.ChangeWatch) { |
| 29 | t.Helper() |
| 30 | if _, err := w.Poll(context.Background()); err != nil { |
| 31 | t.Fatalf("poll the change index: %v", err) |
| 32 | } |
| 33 | } |
| 34 | |
| 35 | // noIndexStore hides every optional extension of the store it wraps, so a |
| 36 | // watch over it is unusable: the fleet store door's shape (spec §3.4). |
| 37 | type noIndexStore struct{ release.Store } |
| 38 | |
| 39 | // flakyGetStore fails Get of one key while fail is set. |
| 40 | type flakyGetStore struct { |
| 41 | *ciCountingStore |
| 42 | key string |
| 43 | fail atomic.Bool |
| 44 | } |
| 45 | |
| 46 | func (s *flakyGetStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 47 | if key == s.key && s.fail.Load() { |
| 48 | s.note("Get", key) |
| 49 | return nil, "", errors.New("503 slow down") |
| 50 | } |
| 51 | return s.ciCountingStore.Get(ctx, key) |
| 52 | } |
| 53 | |
| 54 | // genGateKit is an indexed counting store with config generations for the |
| 55 | // given apps under account bab, a polled watch over it, and a gated cache on |
| 56 | // a fake clock. |
| 57 | func genGateKit(t *testing.T, apps ...string) (*ciCountingStore, *release.ChangeWatch, *genCache, *gateClock, []registry.Route) { |
| 58 | t.Helper() |
| 59 | cs := newCICountingStore(t) |
| 60 | var routes []registry.Route |
| 61 | for _, app := range apps { |
| 62 | touch(t, context.Background(), cs, "bab", app) |
| 63 | routes = append(routes, execRoute(app+".example.com", "bab", app)) |
| 64 | } |
| 65 | w := release.NewChangeWatch(cs) |
| 66 | pollWatch(t, w) |
| 67 | clock := newGateClock() |
| 68 | cache := newGenCache() |
| 69 | cache.gate = release.NewReadGate(w, release.ChangeIndexBackstop) |
| 70 | cache.now = clock.now |
| 71 | cs.reset() |
| 72 | return cs, w, cache, clock, routes |
| 73 | } |
| 74 | |
| 75 | func genGets(cs *ciCountingStore, app string) int { |
| 76 | return cs.count("Get", appconfig.GenKey("bab", app)) |
| 77 | } |
| 78 | |
| 79 | func TestConfigGenSkipsUnmovedApp(t *testing.T) { |
| 80 | t.Parallel() |
| 81 | cs, w, cache, clock, routes := genGateKit(t, "shop") |
| 82 | ctx := context.Background() |
| 83 | genPollPass(ctx, routes, cs, cache) |
| 84 | clock.add(3 * time.Second) |
| 85 | pollWatch(t, w) |
| 86 | if exec, unit, errs := genPollPass(ctx, routes, cs, cache); exec || unit || len(errs) != 0 { |
| 87 | t.Fatalf("an unmoved app reported exec %v unit %v errs %v", exec, unit, errs) |
| 88 | } |
| 89 | if n := genGets(cs, "shop"); n != 1 { |
| 90 | t.Fatalf("two ticks with no bump read config-gen.json %d time(s), want 1 (the first): %v", n, cs.got()) |
| 91 | } |
| 92 | } |
| 93 | |
| 94 | func TestConfigGenReadsMovedApp(t *testing.T) { |
| 95 | t.Parallel() |
| 96 | cs, w, cache, clock, routes := genGateKit(t, "shop", "cafe") |
| 97 | ctx := context.Background() |
| 98 | genPollPass(ctx, routes, cs, cache) |
| 99 | touch(t, ctx, cs, "bab", "shop") |
| 100 | cs.reset() |
| 101 | clock.add(3 * time.Second) |
| 102 | pollWatch(t, w) |
| 103 | exec, _, errs := genPollPass(ctx, routes, cs, cache) |
| 104 | if !exec || len(errs) != 0 { |
| 105 | t.Fatalf("a moved app's config: exec %v errs %v, want a sync", exec, errs) |
| 106 | } |
| 107 | if genGets(cs, "shop") != 1 || genGets(cs, "cafe") != 0 { |
| 108 | t.Fatalf("after shop's bump the tick read %v, want shop's config-gen.json only", cs.got()) |
| 109 | } |
| 110 | } |
| 111 | |
| 112 | func TestConfigGenReadsAtMaxAge(t *testing.T) { |
| 113 | t.Parallel() |
| 114 | cs, w, cache, clock, routes := genGateKit(t, "shop") |
| 115 | ctx := context.Background() |
| 116 | genPollPass(ctx, routes, cs, cache) |
| 117 | clock.add(release.ChangeIndexBackstop - time.Second) |
| 118 | pollWatch(t, w) |
| 119 | genPollPass(ctx, routes, cs, cache) |
| 120 | if n := genGets(cs, "shop"); n != 1 { |
| 121 | t.Fatalf("a read younger than the backstop was repeated: %d reads", n) |
| 122 | } |
| 123 | clock.add(time.Second) |
| 124 | genPollPass(ctx, routes, cs, cache) |
| 125 | if n := genGets(cs, "shop"); n != 2 { |
| 126 | t.Fatalf("a read the backstop's age old was not repeated: %d reads, want 2", n) |
| 127 | } |
| 128 | } |
| 129 | |
| 130 | func TestConfigGenRetriesAfterError(t *testing.T) { |
| 131 | t.Parallel() |
| 132 | cs, w, cache, clock, routes := genGateKit(t, "shop") |
| 133 | ctx := context.Background() |
| 134 | fs := &flakyGetStore{ciCountingStore: cs, key: appconfig.GenKey("bab", "shop")} |
| 135 | genPollPass(ctx, routes, fs, cache) |
| 136 | touch(t, ctx, cs, "bab", "shop") |
| 137 | pollWatch(t, w) |
| 138 | cs.reset() |
| 139 | fs.fail.Store(true) |
| 140 | clock.add(3 * time.Second) |
| 141 | if exec, _, errs := genPollPass(ctx, routes, fs, cache); exec || len(errs) != 1 { |
| 142 | t.Fatalf("the failed read: exec %v errs %v, want one error and no sync", exec, errs) |
| 143 | } |
| 144 | // No new bump: the failed read alone makes the next tick read again, |
| 145 | // and that read sees the change. |
| 146 | fs.fail.Store(false) |
| 147 | clock.add(3 * time.Second) |
| 148 | pollWatch(t, w) |
| 149 | if exec, _, errs := genPollPass(ctx, routes, fs, cache); !exec || len(errs) != 0 { |
| 150 | t.Fatalf("the tick after a failed read: exec %v errs %v, want the change synced", exec, errs) |
| 151 | } |
| 152 | if n := genGets(cs, "shop"); n != 2 { |
| 153 | t.Fatalf("config-gen.json read %d time(s) after the bump, want 2 (the failure, the retry)", n) |
| 154 | } |
| 155 | } |
| 156 | |
| 157 | // ---- canary pointers ------------------------------------------------------ |
| 158 | |
| 159 | // canaryKit is a box with one canary route (fastapp on canary/dev, account |
| 160 | // bab) whose head is v1, over an indexed counting store. |
| 161 | type canaryKit struct { |
| 162 | env *fastFixture |
| 163 | ds *release.DirStore |
| 164 | cs *ciCountingStore |
| 165 | d *fastDeps |
| 166 | clock *gateClock |
| 167 | } |
| 168 | |
| 169 | const canaryHost = "canary.example.com" |
| 170 | |
| 171 | func newCanaryKit(t *testing.T, usable bool) *canaryKit { |
| 172 | t.Helper() |
| 173 | env := newFastFixture(t) |
| 174 | cs := newCICountingStore(t) |
| 175 | k := &canaryKit{env: env, ds: cs.DirStore, cs: cs, clock: newGateClock()} |
| 176 | k.ship(t, "v1") |
| 177 | env.addRoute(t, canaryHost, "canary/dev") |
| 178 | var store release.Store = cs |
| 179 | if !usable { |
| 180 | store = noIndexStore{cs} |
| 181 | } |
| 182 | env.store = store |
| 183 | k.d = env.deps(nil) |
| 184 | k.d.watch = release.NewChangeWatch(store) |
| 185 | k.d.canaryGate = release.NewReadGate(k.d.watch, release.ChangeIndexBackstop) |
| 186 | cs.reset() |
| 187 | return k |
| 188 | } |
| 189 | |
| 190 | func (k *canaryKit) ship(t *testing.T, ver string) { |
| 191 | t.Helper() |
| 192 | ctx := context.Background() |
| 193 | plat := runtime.GOOS + "-" + runtime.GOARCH |
| 194 | m := release.Manifest{App: "fastapp", Version: ver, Kind: "binary", Artifacts: map[string]release.Artifact{plat: {}}} |
| 195 | if err := release.Ship(ctx, k.ds, "bab", m, map[string][]byte{plat: []byte("build " + ver)}); err != nil { |
| 196 | t.Fatalf("Ship %s: %v", ver, err) |
| 197 | } |
| 198 | if err := release.Promote(ctx, k.ds, k.env.priv, "bab", "fastapp", "canary/dev", ver); err != nil { |
| 199 | t.Fatalf("Promote %s: %v", ver, err) |
| 200 | } |
| 201 | } |
| 202 | |
| 203 | // tick runs one fastTick at the kit's clock, checks the canary is still a |
| 204 | // member, and returns how many times the pointer has been read so far. |
| 205 | func (k *canaryKit) tick(t *testing.T, d time.Duration) int { |
| 206 | t.Helper() |
| 207 | k.clock.add(d) |
| 208 | if m := k.d.fastMembers(k.clock.now()); !slices.Contains(m, canaryHost) { |
| 209 | t.Fatalf("the canary route left the sweep: members %v", m) |
| 210 | } |
| 211 | fastTick(context.Background(), k.d, k.clock.now()) |
| 212 | return k.cs.count("Get", release.ChannelKey("bab", "fastapp", "canary/dev")) |
| 213 | } |
| 214 | |
| 215 | func (k *canaryKit) live(t *testing.T) string { |
| 216 | t.Helper() |
| 217 | v, err := release.LiveVersion(k.env.optDir, canaryHost) |
| 218 | if err != nil { |
| 219 | t.Fatal(err) |
| 220 | } |
| 221 | return v |
| 222 | } |
| 223 | |
| 224 | func TestCanaryPointerGated(t *testing.T) { |
| 225 | t.Parallel() |
| 226 | k := newCanaryKit(t, true) |
| 227 | if n := k.tick(t, 0); n != 1 || k.live(t) != "v1" { |
| 228 | t.Fatalf("first tick: %d pointer reads, live %q; want 1 and v1", n, k.live(t)) |
| 229 | } |
| 230 | for i := range 3 { |
| 231 | if n := k.tick(t, 2*time.Second); n != 1 { |
| 232 | t.Fatalf("quiet tick %d read the pointer again (%d reads)", i, n) |
| 233 | } |
| 234 | } |
| 235 | // A promote bumps the app's scope: the next tick reads and adopts. |
| 236 | k.ship(t, "v2") |
| 237 | if n := k.tick(t, 2*time.Second); n != 2 || k.live(t) != "v2" { |
| 238 | t.Fatalf("after a promote: %d pointer reads, live %q; want 2 and v2", n, k.live(t)) |
| 239 | } |
| 240 | if n := k.tick(t, 2*time.Second); n != 2 { |
| 241 | t.Fatalf("the tick after the adopt read the pointer again (%d reads)", n) |
| 242 | } |
| 243 | // The age bound: a pointer read the backstop's age ago is read again. |
| 244 | if n := k.tick(t, release.ChangeIndexBackstop); n != 3 { |
| 245 | t.Fatalf("at the backstop age: %d pointer reads, want 3", n) |
| 246 | } |
| 247 | } |
| 248 | |
| 249 | // A canary group whose routes are not live on the head its last read |
| 250 | // returned (here the live link was moved off it, as a rolled-back restart |
| 251 | // leaves it) keeps today's read on every tick until it is. |
| 252 | func TestCanaryPointerReadWhileUnsettled(t *testing.T) { |
| 253 | t.Parallel() |
| 254 | k := newCanaryKit(t, true) |
| 255 | k.tick(t, 0) |
| 256 | k.env.stageLive(t, canaryHost, "v0") |
| 257 | if n := k.tick(t, 2*time.Second); n != 2 || k.live(t) != "v1" { |
| 258 | t.Fatalf("an unsettled group: %d pointer reads, live %q; want 2 and v1", n, k.live(t)) |
| 259 | } |
| 260 | if n := k.tick(t, 2*time.Second); n != 2 { |
| 261 | t.Fatalf("the group settled again but its pointer was read (%d reads)", n) |
| 262 | } |
| 263 | } |
| 264 | |
| 265 | // A tick whose own index poll failed reads the canary pointer (amadan #336 |
| 266 | // final review, finding 3). Here another caller's index GET hangs holding |
| 267 | // the watch's poll slot, and the tick gives up waiting for it: before the |
| 268 | // fix that give-up left the frozen observation vouching, and the gate |
| 269 | // skipped the read. The next good poll restores the gate. |
| 270 | func TestCanaryPointerReadWhenTheTicksIndexPollFails(t *testing.T) { |
| 271 | t.Parallel() |
| 272 | k := newCanaryKit(t, true) |
| 273 | if n := k.tick(t, 0); n != 1 { |
| 274 | t.Fatalf("first tick: %d pointer reads, want 1", n) |
| 275 | } |
| 276 | if n := k.tick(t, 2*time.Second); n != 1 { |
| 277 | t.Fatalf("a quiet tick read the pointer (%d reads)", n) |
| 278 | } |
| 279 | unstall := k.cs.stallIndex() |
| 280 | polls := k.cs.count("GetIfChanged", release.ChangeIndexKey) |
| 281 | hung := make(chan error, 1) |
| 282 | go func() { _, err := k.d.watch.Poll(context.Background()); hung <- err }() |
| 283 | for k.cs.count("GetIfChanged", release.ChangeIndexKey) == polls { |
| 284 | time.Sleep(time.Millisecond) |
| 285 | } |
| 286 | if n := k.tick(t, 2*time.Second); n != 2 { |
| 287 | unstall() |
| 288 | t.Fatalf("a tick whose index poll gave up skipped the pointer read (%d reads, want 2): it trusted a frozen observation", n) |
| 289 | } |
| 290 | unstall() |
| 291 | if err := <-hung; err != nil { |
| 292 | t.Fatalf("the hung poll, released: %v", err) |
| 293 | } |
| 294 | if n := k.tick(t, 2*time.Second); n != 2 { |
| 295 | t.Fatalf("after the watch recovered the quiet tick read the pointer (%d reads)", n) |
| 296 | } |
| 297 | } |
| 298 | |
| 299 | // Final review R1 finding 6 (amadan #336): two canary channels of one app. |
| 300 | // canary/dev's promote fails its restart and rolls back, so its route sits |
| 301 | // out the sweep in its restart hold; canary/beta reads its own pointer |
| 302 | // meanwhile, which marks the app's generation. When dev's hold ends its |
| 303 | // cached head still matches its rolled-back live version, but its own last |
| 304 | // read predates the app's generation, so it reads again and retries the |
| 305 | // promote. Before, beta's read settled dev until the backstop. |
| 306 | func TestCanaryGroupFreshnessIsPerChannel(t *testing.T) { |
| 307 | t.Parallel() |
| 308 | k := newCanaryKit(t, true) |
| 309 | ctx := context.Background() |
| 310 | const betaHost = "beta.example.com" |
| 311 | if err := release.Promote(ctx, k.ds, k.env.priv, "bab", "fastapp", "canary/beta", "v1"); err != nil { |
| 312 | t.Fatal(err) |
| 313 | } |
| 314 | k.env.addRoute(t, betaHost, "canary/beta") |
| 315 | devKey := release.ChannelKey("bab", "fastapp", "canary/dev") |
| 316 | betaKey := release.ChannelKey("bab", "fastapp", "canary/beta") |
| 317 | tick := func(d time.Duration) (dev, beta int) { |
| 318 | k.clock.add(d) |
| 319 | fastTick(ctx, k.d, k.clock.now()) |
| 320 | return k.cs.count("Get", devKey), k.cs.count("Get", betaKey) |
| 321 | } |
| 322 | if dev, beta := tick(0); dev != 1 || beta != 1 { |
| 323 | t.Fatalf("first tick: dev %d beta %d pointer reads, want 1 each", dev, beta) |
| 324 | } |
| 325 | if dev, beta := tick(2 * time.Second); dev != 1 || beta != 1 { |
| 326 | t.Fatalf("a quiet tick read a pointer: dev %d beta %d", dev, beta) |
| 327 | } |
| 328 | |
| 329 | k.ship(t, "v2") // canary/dev -> v2; the restart fails and rolls back: |
| 330 | k.d.restartHold = map[string]*restartFailure{canaryHost: {ver: "v2", step: time.Minute, notBefore: k.clock.now().Add(time.Minute)}} |
| 331 | if dev, beta := tick(2 * time.Second); dev != 1 || beta != 2 { |
| 332 | t.Fatalf("with dev held: dev %d beta %d pointer reads, want dev untouched and beta read", dev, beta) |
| 333 | } |
| 334 | if k.live(t) != "v1" { |
| 335 | t.Fatalf("fixture: the held route moved to %q", k.live(t)) |
| 336 | } |
| 337 | delete(k.d.restartHold, canaryHost) // the hold ends |
| 338 | dev, beta := tick(2 * time.Second) |
| 339 | if dev != 2 { |
| 340 | t.Fatalf("dev's pointer read %d time(s) after its hold ended, want 2: beta's read settled dev's group", dev) |
| 341 | } |
| 342 | if beta != 2 { |
| 343 | t.Fatalf("beta's pointer read again (%d) with nothing moved", beta) |
| 344 | } |
| 345 | if k.live(t) != "v2" { |
| 346 | t.Fatalf("dev did not retry the promote: live %q", k.live(t)) |
| 347 | } |
| 348 | } |
| 349 | |
| 350 | // A stable route keeps today's path: the gate never skips its pointer. |
| 351 | func TestCanaryGateLeavesStableRoutes(t *testing.T) { |
| 352 | t.Parallel() |
| 353 | k := newCanaryKit(t, true) |
| 354 | skip, read := k.d.canaryPointerGate(k.clock.now())("bab", "fastapp", "stable", nil) |
| 355 | if skip || read != nil { |
| 356 | t.Fatalf("the canary gate took a stable group: skip %v read %v", skip, read != nil) |
| 357 | } |
| 358 | } |
| 359 | |
| 360 | // ---- telemetry ------------------------------------------------------------ |
| 361 | |
| 362 | func telemetryRoutes(t *testing.T, acct string, apps ...string) *registry.Registry { |
| 363 | t.Helper() |
| 364 | reg := edgesTestRegistry(t) |
| 365 | for _, app := range apps { |
| 366 | if err := reg.Upsert(registry.Route{Host: app + ".example.com", Kind: "instance", Account: acct, App: app, Status: "ready"}); err != nil { |
| 367 | t.Fatal(err) |
| 368 | } |
| 369 | } |
| 370 | return reg |
| 371 | } |
| 372 | |
| 373 | // Armed, an account's telemetry.json is read once per pass, however many of |
| 374 | // its apps the box serves. |
| 375 | func TestTelemetryAccountReadOncePerPass(t *testing.T) { |
| 376 | t.Parallel() |
| 377 | ctx := context.Background() |
| 378 | cs := newCICountingStore(t) |
| 379 | reg := telemetryRoutes(t, "bab", "shop", "cafe", "deli") |
| 380 | if err := telemetry.Update(ctx, cs, telemetry.AccountKey("bab"), func(st *telemetry.Settings) { |
| 381 | st.Analytics = &telemetry.Analytics{Count: telemetry.CountEdge} |
| 382 | }); err != nil { |
| 383 | t.Fatal(err) |
| 384 | } |
| 385 | w := release.NewChangeWatch(cs) |
| 386 | pollWatch(t, w) |
| 387 | cs.reset() |
| 388 | on, err := telemetrySwitchesWith(ctx, cs, reg, newTelemetryReads(w), newGateClock().now()) |
| 389 | if err != nil { |
| 390 | t.Fatal(err) |
| 391 | } |
| 392 | if n := cs.count("Get", telemetry.AccountKey("bab")); n != 1 { |
| 393 | t.Fatalf("three apps in one account read its telemetry.json %d time(s), want 1: %v", n, cs.got()) |
| 394 | } |
| 395 | if !on["bab/shop"] || !on["bab/cafe"] || !on["bab/deli"] { |
| 396 | t.Fatalf("switches %v, want every app on from the account", on) |
| 397 | } |
| 398 | } |
| 399 | |
| 400 | // onceFailingGetStore fails the first Get of one key, then answers. |
| 401 | type onceFailingGetStore struct { |
| 402 | *ciCountingStore |
| 403 | key string |
| 404 | failed atomic.Bool |
| 405 | } |
| 406 | |
| 407 | func (s *onceFailingGetStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 408 | if key == s.key && s.failed.CompareAndSwap(false, true) { |
| 409 | s.note("Get", key) |
| 410 | return nil, "", errors.New("503 slow down") |
| 411 | } |
| 412 | return s.ciCountingStore.Get(ctx, key) |
| 413 | } |
| 414 | |
| 415 | // Disarmed, the pass is main's (final review R1, finding 5): each app |
| 416 | // resolves both levels on its own, so one transient account read failure |
| 417 | // leaves only that app off, never the account's other apps. |
| 418 | func TestTelemetryDisarmedResolvesEachAppOnItsOwn(t *testing.T) { |
| 419 | t.Parallel() |
| 420 | ctx := context.Background() |
| 421 | cs := newCICountingStore(t) |
| 422 | reg := telemetryRoutes(t, "bab", "shop", "cafe", "deli") |
| 423 | if err := telemetry.Update(ctx, cs, telemetry.AccountKey("bab"), func(st *telemetry.Settings) { |
| 424 | st.Analytics = &telemetry.Analytics{Count: telemetry.CountEdge} |
| 425 | }); err != nil { |
| 426 | t.Fatal(err) |
| 427 | } |
| 428 | fs := &onceFailingGetStore{ciCountingStore: cs, key: telemetry.AccountKey("bab")} |
| 429 | cs.reset() |
| 430 | on, err := telemetrySwitchesWith(ctx, fs, reg, newTelemetryReads(nil), newGateClock().now()) |
| 431 | if err != nil { |
| 432 | t.Fatal(err) |
| 433 | } |
| 434 | if n := cs.count("Get", telemetry.AccountKey("bab")); n != 3 { |
| 435 | t.Fatalf("three apps read the account's telemetry.json %d time(s) disarmed, want 3 (one each, as before the index)", n) |
| 436 | } |
| 437 | if len(on) != 2 { |
| 438 | t.Fatalf("switches %v: one failed account read must leave only its own app off", on) |
| 439 | } |
| 440 | } |
| 441 | |
| 442 | func TestTelemetryGatedWithAgeBound(t *testing.T) { |
| 443 | t.Parallel() |
| 444 | ctx := context.Background() |
| 445 | cs := newCICountingStore(t) |
| 446 | reg := telemetryRoutes(t, "bab", "shop", "cafe") |
| 447 | if err := telemetry.Update(ctx, cs, telemetry.AccountKey("bab"), func(st *telemetry.Settings) { |
| 448 | st.Analytics = &telemetry.Analytics{Count: telemetry.CountEdge} |
| 449 | }); err != nil { |
| 450 | t.Fatal(err) |
| 451 | } |
| 452 | w := release.NewChangeWatch(cs) |
| 453 | pollWatch(t, w) |
| 454 | r := newTelemetryReads(w) |
| 455 | clock := newGateClock() |
| 456 | pass := func() map[string]bool { |
| 457 | t.Helper() |
| 458 | on, err := telemetrySwitchesWith(ctx, cs, reg, r, clock.now()) |
| 459 | if err != nil { |
| 460 | t.Fatal(err) |
| 461 | } |
| 462 | return on |
| 463 | } |
| 464 | cs.reset() |
| 465 | if on := pass(); !on["bab/shop"] || !on["bab/cafe"] { |
| 466 | t.Fatalf("first pass switches %v", on) |
| 467 | } |
| 468 | if n := cs.count("Get", ""); n != 3 { |
| 469 | t.Fatalf("the first pass made %d reads, want 3 (account, shop, cafe): %v", n, cs.got()) |
| 470 | } |
| 471 | clock.add(time.Minute) |
| 472 | pollWatch(t, w) |
| 473 | cs.reset() |
| 474 | if on := pass(); !on["bab/shop"] || !on["bab/cafe"] { |
| 475 | t.Fatalf("a pass that skipped its reads lost the last good values: %v", on) |
| 476 | } |
| 477 | if n := cs.count("Get", ""); n != 0 { |
| 478 | t.Fatalf("a quiet pass read %v", cs.got()) |
| 479 | } |
| 480 | // cafe's own setting moves: only cafe's file is read. |
| 481 | if err := telemetry.Update(ctx, cs, telemetry.AppKey("bab", "cafe"), func(st *telemetry.Settings) { |
| 482 | st.Analytics = &telemetry.Analytics{Count: telemetry.CountOff} |
| 483 | }); err != nil { |
| 484 | t.Fatal(err) |
| 485 | } |
| 486 | clock.add(time.Minute) |
| 487 | pollWatch(t, w) |
| 488 | cs.reset() |
| 489 | if on := pass(); !on["bab/shop"] || on["bab/cafe"] { |
| 490 | t.Fatalf("after cafe's change switches %v, want shop on, cafe off", on) |
| 491 | } |
| 492 | if got := cs.got(); !slices.Equal(got, []string{"Get " + telemetry.AppKey("bab", "cafe")}) { |
| 493 | t.Fatalf("after cafe's change the pass read %v, want cafe's telemetry.json only", got) |
| 494 | } |
| 495 | // The age bound: every file is read again at the backstop's age. |
| 496 | cs.reset() |
| 497 | clock.add(release.ChangeIndexBackstop) |
| 498 | pass() |
| 499 | if n := cs.count("Get", ""); n != 3 { |
| 500 | t.Fatalf("at the backstop age the pass made %d reads, want 3: %v", n, cs.got()) |
| 501 | } |
| 502 | } |
| 503 | |
| 504 | // A failed read keeps the last good value and is retried on the next pass. |
| 505 | func TestTelemetryKeepsLastGoodOnError(t *testing.T) { |
| 506 | t.Parallel() |
| 507 | ctx := context.Background() |
| 508 | cs := newCICountingStore(t) |
| 509 | reg := telemetryRoutes(t, "bab", "shop") |
| 510 | if err := telemetry.Update(ctx, cs, telemetry.AccountKey("bab"), func(st *telemetry.Settings) { |
| 511 | st.Analytics = &telemetry.Analytics{Count: telemetry.CountEdge} |
| 512 | }); err != nil { |
| 513 | t.Fatal(err) |
| 514 | } |
| 515 | fs := &flakyGetStore{ciCountingStore: cs, key: telemetry.AccountKey("bab")} |
| 516 | w := release.NewChangeWatch(cs) |
| 517 | pollWatch(t, w) |
| 518 | r := newTelemetryReads(w) |
| 519 | clock := newGateClock() |
| 520 | if on, _ := telemetrySwitchesWith(ctx, fs, reg, r, clock.now()); !on["bab/shop"] { |
| 521 | t.Fatalf("first pass %v", on) |
| 522 | } |
| 523 | clock.add(release.ChangeIndexBackstop) // due by age |
| 524 | fs.fail.Store(true) |
| 525 | cs.reset() |
| 526 | if on, _ := telemetrySwitchesWith(ctx, fs, reg, r, clock.now()); !on["bab/shop"] { |
| 527 | t.Fatalf("a failed account read dropped the last good value: %v", on) |
| 528 | } |
| 529 | if n := cs.count("Get", telemetry.AccountKey("bab")); n != 1 { |
| 530 | t.Fatalf("the due pass read the account %d time(s), want 1", n) |
| 531 | } |
| 532 | fs.fail.Store(false) |
| 533 | cs.reset() |
| 534 | clock.add(time.Second) |
| 535 | telemetrySwitchesWith(ctx, fs, reg, r, clock.now()) |
| 536 | if n := cs.count("Get", telemetry.AccountKey("bab")); n != 1 { |
| 537 | t.Fatalf("the pass after a failed read read the account %d time(s), want 1 (a retry)", n) |
| 538 | } |
| 539 | } |
| 540 | |
| 541 | // ---- an unusable watch ---------------------------------------------------- |
| 542 | |
| 543 | // Review Focus 5: a watch that cannot vouch (here, a store with no |
| 544 | // conditional read, the fleet door's shape) makes every gated read due on |
| 545 | // every tick. The gates are the production ones, built from the armed watch. |
| 546 | func TestGatesDueWhenWatchUnusable(t *testing.T) { |
| 547 | cs := newCICountingStore(t) |
| 548 | plain := noIndexStore{cs} |
| 549 | w := release.NewChangeWatch(plain) |
| 550 | if _, err := w.Poll(context.Background()); err == nil { |
| 551 | t.Fatal("fixture: a watch over a store with no conditional read polled cleanly") |
| 552 | } |
| 553 | armedChangeWatch.Store(w) |
| 554 | t.Cleanup(func() { armedChangeWatch.Store(nil) }) |
| 555 | ctx := context.Background() |
| 556 | clock := newGateClock() |
| 557 | |
| 558 | // config-gen.json |
| 559 | touch(t, ctx, cs, "bab", "shop") |
| 560 | cache := newGenCache() |
| 561 | cache.gate = newChangeGate() |
| 562 | cache.now = clock.now |
| 563 | routes := []registry.Route{execRoute("shop.example.com", "bab", "shop")} |
| 564 | cs.reset() |
| 565 | for range 3 { |
| 566 | clock.add(3 * time.Second) |
| 567 | genPollPass(ctx, routes, plain, cache) |
| 568 | } |
| 569 | if n := genGets(cs, "shop"); n != 3 { |
| 570 | t.Fatalf("config-gen.json read %d time(s) in three ticks, want 3", n) |
| 571 | } |
| 572 | |
| 573 | // telemetry.json, through the production state |
| 574 | reg := telemetryRoutes(t, "bab", "shop") |
| 575 | cs.reset() |
| 576 | for range 3 { |
| 577 | clock.add(time.Minute) |
| 578 | if _, err := telemetrySwitchesWith(ctx, plain, reg, currentTelemetryReads(), clock.now()); err != nil { |
| 579 | t.Fatal(err) |
| 580 | } |
| 581 | } |
| 582 | if a, b := cs.count("Get", telemetry.AccountKey("bab")), cs.count("Get", telemetry.AppKey("bab", "shop")); a != 3 || b != 3 { |
| 583 | t.Fatalf("telemetry reads in three passes: account %d app %d, want 3 each", a, b) |
| 584 | } |
| 585 | |
| 586 | // canary pointers |
| 587 | k := newCanaryKit(t, false) |
| 588 | k.d.canaryGate = newChangeGate() |
| 589 | k.d.watch = nil // fastTick's own poll would fail the same way; the gate reads the armed watch |
| 590 | for i := range 3 { |
| 591 | if n := k.tick(t, 2*time.Second); n != i+1 { |
| 592 | t.Fatalf("tick %d: %d pointer reads, want %d", i, n, i+1) |
| 593 | } |
| 594 | } |
| 595 | } |
| 596 | |
amadan