amadan

carlos / platform Public

Wake a sidecar whose route this pass just minted

sidecarWakeChanges treated every first sight of a sidecar as seed-don't-
replay. That rule is right for an AGENT RESTART, where replaying every
historical wake would be a thundering herd — and wrong for a route the
pass has only just minted. A `carlos sidecars wake` that lands before the
box's next tick was therefore swallowed by that very tick: it mints the
route and seeds the wake object's etag in one pass, while the CLI reports
success. Since an operator wake is a hibernating sidecar's only first-run
mechanism (the timer needs a next-due only a running child can set), the
feature's primary flow failed on its most common timing — exactly what a
scripted enroll, create, wake does.

The etag cache now carries the one bit that tells the two cases apart:
whether the arming BASELINE has been established. Before it, first sight
is history and is recorded silently; after it, a first sight is a route
minted under a running agent, and a wake object already standing at one is
replayed. A spurious extra wake is idempotent and cheap; a lost one is a
bot that never runs. A minted route with no wake object stays silent, and
an arming pass whose route list could not be read leaves the baseline to
the first leg pass rather than reading an empty cache as "everything here
is new".
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014WvcQVWggXkE5BAkjRDgWF
Paul Campbell pushed by paul@keymail.dev 9f83895136ed6f3c25639bd370c678cb79974dd1 parent ddee127
3 files changed, +281 −63
  • internal/host/agent.go +3 −3
  • internal/host/sidecarwake.go +86 −28
  • internal/host/sidecarwake_test.go +192 −32
diff --git a/internal/host/agent.go b/internal/host/agent.go
index befe7821e..85284c7fd 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -1353,8 +1353,8 @@ func fastConvergeLoop(reg *registry.Registry, svc *activator.Service) {
// same reason: one silent read per served sidecar at arming so the
// FIRST wake request after a boot is acted on rather than swallowed as
// the baseline (sidecarWakeSeed's own doc comment).
- sidecarWakeCache := map[[3]string]string{}
- sidecarWakeSeed(ctx, reg, store, sidecarWakeCache)
+ sidecarWakeMem := newSidecarWakeMemory()
+ sidecarWakeSeed(ctx, reg, store, sidecarWakeMem)
sidecarWake := wakeFn(svc)
d := &fastDeps{
reg: reg, store: store, pub: pub, optDir: optDir,
@@ -1401,7 +1401,7 @@ func fastConvergeLoop(reg *registry.Registry, svc *activator.Service) {
log.Printf("agent: sidecar wake check: list routes: %v", err)
return
}
- sidecarWakeLeg(ctx, routes, store, sidecarWakeCache, sidecarWake)
+ sidecarWakeLeg(ctx, routes, store, sidecarWakeMem, sidecarWake)
},
}
log.Printf("agent: fast converge loop armed (2s + jitter: serial poll + canary/* + unstaged routes)")
diff --git a/internal/host/sidecarwake.go b/internal/host/sidecarwake.go
index b3c4246b2..02c378014 100644
--- a/internal/host/sidecarwake.go
+++ b/internal/host/sidecarwake.go
@@ -55,24 +55,61 @@ func sidecarsServed(routes []registry.Route) map[[3]string]registry.Route {
return out
}
+// sidecarWakeMemory is the wake leg's memory across ticks: the etag last
+// seen for each served sidecar, plus the one bit that says whether the
+// arming BASELINE has been established.
+//
+// The two live together deliberately. "First sight" means two completely
+// different things either side of that baseline — history a booting box must
+// not replay, versus a route that has just appeared under a running agent —
+// and a bare map cannot tell them apart, which is exactly how the
+// create-then-wake flow came to swallow its own first wake.
+type sidecarWakeMemory struct {
+ etags map[[3]string]string
+ // seeded is set by the first completed pass (the arming one in
+ // production, sidecarWakeSeed; the first leg pass when arming's route
+ // list could not be read). Before it, first sight is the baseline; after
+ // it, first sight is a NEW route.
+ seeded bool
+}
+
+func newSidecarWakeMemory() *sidecarWakeMemory {
+ return &sidecarWakeMemory{etags: map[[3]string]string{}}
+}
+
// sidecarWakeChanges is sidecar-wake's counterpart to restartGenPass: read
// the wake-object etag of every sidecar this box serves and report which
// ones MOVED, as their routes. It performs no wake itself — see
// dispatchSidecarWakes for the half that can block.
//
-// - First sight of a sidecar records silently and is never reported as
-// moved. A box (or an agent process) that has just started must not
-// wake every sidecar it serves because it has never seen these etags
-// before — that is the seed-don't-replay rule the serial watcher
-// already applies (agent.go's fastTick), and it is the whole safety
-// property of this feature. Without it, an agent restart would replay
-// every wake this app's sidecars have ever received — a thundering
-// herd on boot, and for a hibernating fleet a real cost, not a
-// cosmetic one. sidecarWakeSeed runs one such silent pass at arming so
+// - First sight BEFORE the baseline exists records silently and is never
+// reported as moved. A box (or an agent process) that has just started
+// must not wake every sidecar it serves because it has never seen these
+// etags before — that is the seed-don't-replay rule the serial watcher
+// already applies (agent.go's fastTick). Without it, an agent restart
+// would replay every wake this app's sidecars have ever received — a
+// thundering herd on boot, and for a hibernating fleet a real cost, not
+// a cosmetic one. sidecarWakeSeed runs one such silent pass at arming so
// the FIRST request after a boot is acted on rather than swallowed.
+// - First sight AFTER it, with a wake object already present, is reported
+// as MOVED. Such a route did not exist when this agent last looked, so
+// it has just been minted — and a wake object standing at a route this
+// very pass minted is a wake somebody already asked for and the CLI
+// already reported as accepted. That is the create-then-wake flow, and
+// it is the feature's PRIMARY one: an operator wake is a hibernating
+// sidecar's only first-run mechanism (the timer needs a next-due that
+// only a running child can set), and `carlos sidecars create` followed
+// by `carlos sidecars wake` routinely lands both inside a single tick —
+// the tick that mints the route would otherwise seed the wake's etag and
+// silently drop the wake. A spurious extra wake is idempotent and cheap;
+// a lost one is a bot that never runs. (A route that vanished and came
+// back is a first sight too, and is woken once for the same reason: it
+// has just been re-minted, and its wake object outlived it.)
// - An unchanged etag does nothing.
// - Absent that stays absent does nothing, forever: a sidecar nobody has
// ever woken this way has no object and never will until someone asks.
+// A brand-new route with no wake object is therefore silent too — only a
+// PRESENT object makes a first sight a wake.
// - Any movement counts: the etag changed, or absent -> present (the
// sidecar's first ever wake request).
// - A wake object naming a sidecar this box does not serve is never even
@@ -82,16 +119,15 @@ func sidecarsServed(routes []registry.Route) map[[3]string]registry.Route {
// - A read error for one sidecar leaves its cache entry untouched and the
// rest of the pass keeps going; the error comes back for the caller to
// aggregate into its own log line, same discipline as restartGenPass.
-// - A sidecar route that vanishes from the route set loses its cache
-// entry: a re-added sidecar is a first sight again, which is right,
-// because a route that has just been (re-)minted is starting fresh
-// anyway.
+// After the baseline that means its next sighting is a first sight, so
+// it may be woken once — the cheap direction of an unknown.
//
-// The cache is mutated in place and is expected to live for the process
-// lifetime, exactly like restartGenPass's.
-func sidecarWakeChanges(ctx context.Context, routes []registry.Route, store release.Store, cache map[[3]string]string) (moved []registry.Route, readErrs []error) {
+// The memory is mutated in place and is expected to live for the process
+// lifetime, exactly like restartGenPass's cache.
+func sidecarWakeChanges(ctx context.Context, routes []registry.Route, store release.Store, mem *sidecarWakeMemory) (moved []registry.Route, readErrs []error) {
served := sidecarsServed(routes)
live := make(map[[3]string]bool, len(served))
+ baseline := !mem.seeded
for k, rt := range served {
live[k] = true
_, etag, err := store.Get(ctx, release.SidecarWakeKey(k[0], k[1], k[2]))
@@ -102,17 +138,30 @@ func sidecarWakeChanges(ctx context.Context, routes []registry.Route, store rele
readErrs = append(readErrs, fmt.Errorf("%s/%s/%s: %w", k[0], k[1], k[2], err))
continue // cache entry untouched: unknown this tick, not changed
}
- prev, seen := cache[k]
- cache[k] = etag
- if seen && prev != etag {
+ prev, seen := mem.etags[k]
+ mem.etags[k] = etag
+ switch {
+ case seen:
+ if prev != etag {
+ moved = append(moved, rt)
+ }
+ case baseline:
+ // The agent-restart protection: history, recorded silently.
+ case etag != genAbsent:
+ // A route minted since the last pass, with a wake already
+ // standing at it — the create-then-wake flow.
moved = append(moved, rt)
}
}
- for k := range cache {
+ for k := range mem.etags {
if !live[k] {
- delete(cache, k)
+ delete(mem.etags, k)
}
}
+ // One completed pass IS the baseline, whichever caller ran it: arming's
+ // silent seed in production, or the first leg pass on a box whose arming
+ // route list could not be read.
+ mem.seeded = true
return moved, readErrs
}
@@ -150,8 +199,8 @@ func dispatchSidecarWakes(moved []registry.Route, wake func(rt registry.Route) e
// moved (for logging/tests); the actual wake calls happen off this
// function's return path (dispatchSidecarWakes), so it never blocks on a
// slow or unresponsive sidecar.
-func sidecarWakeLeg(ctx context.Context, routes []registry.Route, store release.Store, cache map[[3]string]string, wake func(rt registry.Route) error) (moved []registry.Route, readErrs []error) {
- moved, readErrs = sidecarWakeChanges(ctx, routes, store, cache)
+func sidecarWakeLeg(ctx context.Context, routes []registry.Route, store release.Store, mem *sidecarWakeMemory, wake func(rt registry.Route) error) (moved []registry.Route, readErrs []error) {
+ moved, readErrs = sidecarWakeChanges(ctx, routes, store, mem)
if len(readErrs) > 0 {
log.Printf("agent: sidecar wake check: %d sidecars unreadable: %v", len(readErrs), readErrs[0])
}
@@ -165,20 +214,29 @@ func sidecarWakeLeg(ctx context.Context, routes []registry.Route, store release.
// sidecarWakeSeed is sidecarWakeChanges' counterpart to restartSeed: at
// arming, read each served sidecar's wake-object etag once, silently, so
-// the cache sidecarWakeLeg compares against exists before the first serial
+// the memory sidecarWakeLeg compares against exists before the first serial
// move. Without it the first wake request after a boot is the sweep that
// establishes the baseline, and is therefore the one request that does
// nothing.
//
-// A read error leaves that sidecar out of the cache (its next sighting is a
-// first sight — safe, never a spurious wake).
-func sidecarWakeSeed(ctx context.Context, reg *registry.Registry, store release.Store, cache map[[3]string]string) {
+// This pass IS the agent-restart protection: only a sidecar this box already
+// served when the agent armed has its history recorded silently. Everything
+// that appears afterwards is a route minted under a running agent, and a
+// wake object already standing at one of those is acted on (see
+// sidecarWakeChanges).
+//
+// A LIST failure leaves the memory unseeded on purpose: the first leg pass
+// then establishes the baseline itself rather than reading an empty cache as
+// "every sidecar on this box is brand new" and waking the lot. A per-sidecar
+// read error is narrower — that one sidecar is left out of the cache and may
+// be woken once on its next sighting.
+func sidecarWakeSeed(ctx context.Context, reg *registry.Registry, store release.Store, mem *sidecarWakeMemory) {
routes, err := reg.List()
if err != nil {
log.Printf("agent: sidecar wake baseline: list routes: %v (the first request may need a second try)", err)
return
}
- _, readErrs := sidecarWakeChanges(ctx, routes, store, cache)
+ _, readErrs := sidecarWakeChanges(ctx, routes, store, mem)
if len(readErrs) > 0 {
log.Printf("agent: sidecar wake baseline: %d sidecars unreadable: %v", len(readErrs), readErrs[0])
}
diff --git a/internal/host/sidecarwake_test.go b/internal/host/sidecarwake_test.go
index 3c082d2cc..16933a8d0 100644
--- a/internal/host/sidecarwake_test.go
+++ b/internal/host/sidecarwake_test.go
@@ -43,11 +43,11 @@ func TestSidecarWakeChangesEtagMoveReportsExactlyThatSidecar(t *testing.T) {
wakeSidecarRoute("bot.eleven.sidecar.internal", "bdm", "eleven", "bot"),
wakeSidecarRoute("other.eleven.sidecar.internal", "bdm", "eleven", "other"),
}
- cache := map[[3]string]string{}
- sidecarWakeChanges(ctx, routes, s, cache) // seed: first sight of both, silently
+ mem := newSidecarWakeMemory()
+ sidecarWakeChanges(ctx, routes, s, mem) // seed: first sight of both, silently
touchSidecarWake(t, ctx, s, "bdm", "eleven", "bot")
- got, errs := sidecarWakeChanges(ctx, routes, s, cache)
+ got, errs := sidecarWakeChanges(ctx, routes, s, mem)
if len(errs) != 0 {
t.Fatalf("unexpected read errors: %v", errs)
}
@@ -60,10 +60,10 @@ func TestSidecarWakeChangesUnchangedEtagReportsNothing(t *testing.T) {
ctx, s := genKit(t)
routes := []registry.Route{wakeSidecarRoute("bot.eleven.sidecar.internal", "bdm", "eleven", "bot")}
touchSidecarWake(t, ctx, s, "bdm", "eleven", "bot")
- cache := map[[3]string]string{}
- sidecarWakeChanges(ctx, routes, s, cache) // seed
+ mem := newSidecarWakeMemory()
+ sidecarWakeChanges(ctx, routes, s, mem) // seed
- got, errs := sidecarWakeChanges(ctx, routes, s, cache)
+ got, errs := sidecarWakeChanges(ctx, routes, s, mem)
if len(errs) != 0 {
t.Fatalf("unexpected read errors: %v", errs)
}
@@ -72,19 +72,22 @@ func TestSidecarWakeChangesUnchangedEtagReportsNothing(t *testing.T) {
}
}
-// TestSidecarWakeChangesFirstSightDoesNotReplay is the no-replay-on-absence
-// rule: an unseeded cache must not treat "I have never looked at this
-// before" as movement, or every agent restart would replay every wake this
-// sidecar has ever received — a thundering herd on boot.
-func TestSidecarWakeChangesFirstSightDoesNotReplay(t *testing.T) {
+// TestSidecarWakeChangesBaselinePassDoesNotReplay is the no-replay rule at
+// the point it applies: the BASELINE pass — arming, i.e. an agent restart —
+// must not treat "I have never looked at this before" as movement, or every
+// restart would replay every wake this sidecar has ever received, a
+// thundering herd on boot. (First sight AFTER the baseline is a different
+// fact entirely — a route minted under a running agent — and is woken; see
+// TestSidecarWakeChangesRouteMintedAfterTheBaselineIsWoken.)
+func TestSidecarWakeChangesBaselinePassDoesNotReplay(t *testing.T) {
ctx, s := genKit(t)
// Wake object already exists BEFORE this box ever looks — the shape a
// long-lived wake request or a box that just (re)started sees.
touchSidecarWake(t, ctx, s, "bdm", "eleven", "bot")
routes := []registry.Route{wakeSidecarRoute("bot.eleven.sidecar.internal", "bdm", "eleven", "bot")}
- cache := map[[3]string]string{}
+ mem := newSidecarWakeMemory()
- got, errs := sidecarWakeChanges(ctx, routes, s, cache)
+ got, errs := sidecarWakeChanges(ctx, routes, s, mem)
if len(errs) != 0 {
t.Fatalf("unexpected read errors: %v", errs)
}
@@ -92,13 +95,13 @@ func TestSidecarWakeChangesFirstSightDoesNotReplay(t *testing.T) {
t.Fatalf("first sight reported %v — a box that just started must not replay every past wake", got)
}
want := sidecarWakeStoreEtag(t, ctx, s, "bdm", "eleven", "bot")
- if got := cache[[3]string{"bdm", "eleven", "bot"}]; got != want {
+ if got := mem.etags[[3]string{"bdm", "eleven", "bot"}]; got != want {
t.Fatalf("cache etag = %q, want the store's %q", got, want)
}
// A SECOND wake request after the seed IS movement.
touchSidecarWake(t, ctx, s, "bdm", "eleven", "bot")
- got, errs = sidecarWakeChanges(ctx, routes, s, cache)
+ got, errs = sidecarWakeChanges(ctx, routes, s, mem)
if len(errs) != 0 {
t.Fatalf("unexpected read errors: %v", errs)
}
@@ -107,6 +110,78 @@ func TestSidecarWakeChangesFirstSightDoesNotReplay(t *testing.T) {
}
}
+// TestSidecarWakeChangesRouteMintedAfterTheBaselineIsWoken is the
+// create-then-wake flow, which is the feature's PRIMARY one: an operator
+// wake is a hibernating sidecar's only first-run mechanism, and
+// `carlos sidecars create` + `carlos sidecars wake` routinely land in the
+// same tick — the tick that mints the route. Seeding that route's etag
+// silently (the baseline rule, over-applied) loses the wake outright while
+// the CLI reports success: a bot that never runs.
+func TestSidecarWakeChangesRouteMintedAfterTheBaselineIsWoken(t *testing.T) {
+ ctx, s := genKit(t)
+ existing := wakeSidecarRoute("old.eleven.sidecar.internal", "bdm", "eleven", "old")
+ mem := newSidecarWakeMemory()
+ sidecarWakeChanges(ctx, []registry.Route{existing}, s, mem) // arming: this box's baseline
+
+ // The console creates a sidecar and wakes it; this box's next tick mints
+ // the route AND sees the wake object for the first time, together.
+ minted := wakeSidecarRoute("bot.eleven.sidecar.internal", "bdm", "eleven", "bot")
+ touchSidecarWake(t, ctx, s, "bdm", "eleven", "bot")
+
+ got, errs := sidecarWakeChanges(ctx, []registry.Route{existing, minted}, s, mem)
+ if len(errs) != 0 {
+ t.Fatalf("unexpected read errors: %v", errs)
+ }
+ if len(got) != 1 || got[0].Host != "bot.eleven.sidecar.internal" {
+ t.Fatalf("reported %v, want the just-minted sidecar — a wake standing at a route this pass minted is a wake somebody already asked for", got)
+ }
+}
+
+// A route minted after the baseline with NO wake object is silent: only a
+// PRESENT object makes a first sight a wake, so ordinary provisioning never
+// wakes a hibernating sidecar nobody asked to run.
+func TestSidecarWakeChangesRouteMintedWithNoWakeObjectIsSilent(t *testing.T) {
+ ctx, s := genKit(t)
+ mem := newSidecarWakeMemory()
+ sidecarWakeChanges(ctx, nil, s, mem) // arming on a box with no sidecars
+
+ minted := wakeSidecarRoute("bot.eleven.sidecar.internal", "bdm", "eleven", "bot")
+ got, errs := sidecarWakeChanges(ctx, []registry.Route{minted}, s, mem)
+ if len(errs) != 0 {
+ t.Fatalf("unexpected read errors: %v", errs)
+ }
+ if len(got) != 0 {
+ t.Fatalf("reported %v — a newly minted sidecar nobody has ever asked to wake must stay asleep", got)
+ }
+
+ // …and it behaves as an ordinary member from then on: the first real
+ // request moves the etag and wakes it.
+ touchSidecarWake(t, ctx, s, "bdm", "eleven", "bot")
+ got, _ = sidecarWakeChanges(ctx, []registry.Route{minted}, s, mem)
+ if len(got) != 1 || got[0].Host != "bot.eleven.sidecar.internal" {
+ t.Fatalf("reported %v, want the sidecar on its first real wake request", got)
+ }
+}
+
+// An arming pass whose route LIST failed must not leave the box thinking
+// every sidecar it serves is brand new: the memory stays unseeded, so the
+// first leg pass establishes the baseline itself instead of replaying every
+// historical wake at once.
+func TestSidecarWakeSeedListFailureLeavesTheBaselineToTheFirstPass(t *testing.T) {
+ ctx, s := genKit(t)
+ touchSidecarWake(t, ctx, s, "bdm", "eleven", "bot") // history, from before this agent
+ routes := []registry.Route{wakeSidecarRoute("bot.eleven.sidecar.internal", "bdm", "eleven", "bot")}
+ mem := newSidecarWakeMemory() // arming never completed: no List, nothing seeded
+
+ got, errs := sidecarWakeChanges(ctx, routes, s, mem)
+ if len(errs) != 0 {
+ t.Fatalf("unexpected read errors: %v", errs)
+ }
+ if len(got) != 0 {
+ t.Fatalf("reported %v — with no baseline established, the first pass IS the baseline, not a replay of every past wake", got)
+ }
+}
+
func sidecarWakeStoreEtag(t *testing.T, ctx context.Context, s release.Store, acct, app, name string) string {
t.Helper()
_, etag, err := s.Get(ctx, release.SidecarWakeKey(acct, app, name))
@@ -125,14 +200,14 @@ func TestSidecarWakeChangesReadErrorDoesNotSuppressOthers(t *testing.T) {
wakeSidecarRoute("bot.eleven.sidecar.internal", "bdm", "eleven", "bot"),
wakeSidecarRoute("worker.titogo.sidecar.internal", "bdm", "titogo", "worker"),
}
- cache := map[[3]string]string{}
- sidecarWakeChanges(ctx, routes, s0, cache) // seed both, absent
+ mem := newSidecarWakeMemory()
+ sidecarWakeChanges(ctx, routes, s0, mem) // seed both, absent
touchSidecarWake(t, ctx, s0, "bdm", "eleven", "bot")
touchSidecarWake(t, ctx, s0, "bdm", "titogo", "worker")
failing := &failingKeyStore{Store: s0, failKey: release.SidecarWakeKey("bdm", "eleven", "bot"), err: errors.New("bucket blip")}
- got, errs := sidecarWakeChanges(ctx, routes, failing, cache)
+ got, errs := sidecarWakeChanges(ctx, routes, failing, mem)
if len(errs) != 1 {
t.Fatalf("errs = %v, want exactly one (the failing key)", errs)
}
@@ -148,18 +223,18 @@ func TestSidecarWakeChangesIgnoresUnservedSidecar(t *testing.T) {
ctx, s := genKit(t)
touchSidecarWake(t, ctx, s, "bdm", "eleven", "ghost") // no route for "ghost" at all
routes := []registry.Route{wakeSidecarRoute("bot.eleven.sidecar.internal", "bdm", "eleven", "bot")}
- cache := map[[3]string]string{}
- sidecarWakeChanges(ctx, routes, s, cache) // seed
+ mem := newSidecarWakeMemory()
+ sidecarWakeChanges(ctx, routes, s, mem) // seed
touchSidecarWake(t, ctx, s, "bdm", "eleven", "ghost") // "ghost" moves again
- got, errs := sidecarWakeChanges(ctx, routes, s, cache)
+ got, errs := sidecarWakeChanges(ctx, routes, s, mem)
if len(errs) != 0 {
t.Fatalf("unexpected read errors: %v", errs)
}
if len(got) != 0 {
t.Fatalf("an unserved sidecar's wake object was reported: %v, want none", got)
}
- if _, ok := cache[[3]string{"bdm", "eleven", "ghost"}]; ok {
+ if _, ok := mem.etags[[3]string{"bdm", "eleven", "ghost"}]; ok {
t.Fatal("a sidecar this box does not serve must never appear in its cache")
}
}
@@ -176,17 +251,17 @@ func TestSidecarWakeChangesIgnoresNonSidecarRouteWithMatchingSlug(t *testing.T)
notSidecar := execRoute("app.example.com", "bdm", "eleven")
notSidecar.Slug = "bot" // same name a sidecar could carry, wrong Kind
routes := []registry.Route{notSidecar}
- cache := map[[3]string]string{}
+ mem := newSidecarWakeMemory()
- got, errs := sidecarWakeChanges(ctx, routes, s, cache)
+ got, errs := sidecarWakeChanges(ctx, routes, s, mem)
if len(errs) != 0 {
t.Fatalf("unexpected read errors: %v", errs)
}
if len(got) != 0 {
t.Fatalf("a non-sidecar route with a matching Slug was reported: %v, want none", got)
}
- if len(cache) != 0 {
- t.Fatalf("cache = %v, want empty — a non-sidecar route must never seed a sidecar cache entry", cache)
+ if len(mem.etags) != 0 {
+ t.Fatalf("cache = %v, want empty — a non-sidecar route must never seed a sidecar cache entry", mem.etags)
}
}
@@ -342,8 +417,8 @@ func TestSidecarWakeLegActsOnBumpedGeneration(t *testing.T) {
t.Fatalf("upsert %s: %v", rt.Host, err)
}
}
- cache := map[[3]string]string{}
- sidecarWakeSeed(ctx, reg, s, cache)
+ mem := newSidecarWakeMemory()
+ sidecarWakeSeed(ctx, reg, s, mem)
touchSidecarWake(t, ctx, s, "bdm", "eleven", "bot")
var mu sync.Mutex
@@ -354,7 +429,7 @@ func TestSidecarWakeLegActsOnBumpedGeneration(t *testing.T) {
mu.Unlock()
return nil
}
- moved, errs := sidecarWakeLeg(ctx, routes, s, cache, wake)
+ moved, errs := sidecarWakeLeg(ctx, routes, s, mem, wake)
if len(errs) != 0 {
t.Fatalf("unexpected read errors: %v", errs)
}
@@ -368,14 +443,99 @@ func TestSidecarWakeLegActsOnBumpedGeneration(t *testing.T) {
})
}
+// TestSidecarWakeLegCreateThenWakeInOneTickStillWakesIt is the blocker as a
+// member would meet it, through the leg fastConvergeLoop actually calls: the
+// agent arms with no sidecars, then one serial move carries BOTH the record
+// (so this tick mints the route) and the wake object. The wake must land.
+func TestSidecarWakeLegCreateThenWakeInOneTickStillWakesIt(t *testing.T) {
+ ctx, s := genKit(t)
+ reg := restartTestRegistry(t)
+ mem := newSidecarWakeMemory()
+ sidecarWakeSeed(ctx, reg, s, mem) // arming: no sidecar routes yet
+
+ // create + wake, both before this box's next tick.
+ minted := wakeSidecarRoute("bot.eleven.sidecar.internal", "bdm", "eleven", "bot")
+ if err := reg.Upsert(minted); err != nil {
+ t.Fatalf("upsert: %v", err)
+ }
+ touchSidecarWake(t, ctx, s, "bdm", "eleven", "bot")
+
+ var mu sync.Mutex
+ var woken []string
+ wake := func(rt registry.Route) error {
+ mu.Lock()
+ woken = append(woken, rt.Host)
+ mu.Unlock()
+ return nil
+ }
+ moved, errs := sidecarWakeLeg(ctx, []registry.Route{minted}, s, mem, wake)
+ if len(errs) != 0 {
+ t.Fatalf("unexpected read errors: %v", errs)
+ }
+ if len(moved) != 1 || moved[0].Host != "bot.eleven.sidecar.internal" {
+ t.Fatalf("leg reported %v, want the just-created sidecar — the CLI already told the member this wake was accepted", moved)
+ }
+ waitForCond(t, "the just-created sidecar to be woken", func() bool {
+ mu.Lock()
+ defer mu.Unlock()
+ return slices.Equal(woken, []string{"bot.eleven.sidecar.internal"})
+ })
+}
+
+// The other side of the same coin, end to end: an agent RESTART on a box
+// whose sidecars have been woken many times before wakes nothing at all.
+func TestSidecarWakeLegAfterAnAgentRestartReplaysNothing(t *testing.T) {
+ ctx, s := genKit(t)
+ reg := restartTestRegistry(t)
+ routes := []registry.Route{
+ wakeSidecarRoute("bot.eleven.sidecar.internal", "bdm", "eleven", "bot"),
+ wakeSidecarRoute("other.eleven.sidecar.internal", "bdm", "eleven", "other"),
+ }
+ for _, rt := range routes {
+ if err := reg.Upsert(rt); err != nil {
+ t.Fatalf("upsert %s: %v", rt.Host, err)
+ }
+ }
+ touchSidecarWake(t, ctx, s, "bdm", "eleven", "bot") // history
+ touchSidecarWake(t, ctx, s, "bdm", "eleven", "other") // history
+
+ mem := newSidecarWakeMemory() // a fresh process
+ sidecarWakeSeed(ctx, reg, s, mem)
+ if len(mem.etags) != 2 {
+ t.Fatalf("the arming pass recorded %d etags, want 2 — a baseline that records nothing is no baseline at all", len(mem.etags))
+ }
+
+ var mu sync.Mutex
+ var woken []string
+ wake := func(rt registry.Route) error {
+ mu.Lock()
+ woken = append(woken, rt.Host)
+ mu.Unlock()
+ return nil
+ }
+ moved, errs := sidecarWakeLeg(ctx, routes, s, mem, wake)
+ if len(errs) != 0 {
+ t.Fatalf("unexpected read errors: %v", errs)
+ }
+ if len(moved) != 0 {
+ t.Fatalf("an agent restart replayed %v — every wake these sidecars ever received would fire at once", moved)
+ }
+ time.Sleep(50 * time.Millisecond) // the dispatch is asynchronous; give it a chance to be wrong
+ mu.Lock()
+ defer mu.Unlock()
+ if len(woken) != 0 {
+ t.Fatalf("woke %v on an agent restart, want none", woken)
+ }
+}
+
// TestSidecarWakeLegReturnsBeforeASlowWakeCompletes: the glued leg itself
// must not block on WakeWorker either — the property dispatchSidecarWakes
// provides must survive being called from inside sidecarWakeLeg.
func TestSidecarWakeLegReturnsBeforeASlowWakeCompletes(t *testing.T) {
ctx, s := genKit(t)
routes := []registry.Route{wakeSidecarRoute("bot.eleven.sidecar.internal", "bdm", "eleven", "bot")}
- cache := map[[3]string]string{}
- sidecarWakeChanges(ctx, routes, s, cache) // seed
+ mem := newSidecarWakeMemory()
+ sidecarWakeChanges(ctx, routes, s, mem) // seed
touchSidecarWake(t, ctx, s, "bdm", "eleven", "bot")
slow := make(chan struct{})
@@ -388,7 +548,7 @@ func TestSidecarWakeLegReturnsBeforeASlowWakeCompletes(t *testing.T) {
var moved []registry.Route
done := make(chan struct{})
go func() {
- moved, _ = sidecarWakeLeg(ctx, routes, s, cache, wake)
+ moved, _ = sidecarWakeLeg(ctx, routes, s, mem, wake)
close(done)
}()
select {