amadan
Add the sidecar wake generation object and its box-side pass
4 files changed,
+425
−0
internal/host/sidecarwake.go+109 −0internal/host/sidecarwake_test.go+223 −0internal/release/sidecarwake.go+45 −0internal/release/sidecarwake_test.go+48 −0
diff --git a/internal/host/sidecarwake.go b/internal/host/sidecarwake.go| new file mode 100644 |
| index 000000000..cbff12df9 |
| --- /dev/null |
| +++ b/internal/host/sidecarwake.go |
| @@ -0,0 +1,109 @@ |
| +// The box half of a sidecar wake request: when a sidecar's wake object |
| +// (release.SidecarWakeKey) MOVES, wake exactly that sidecar. Pull-based like |
| +// the restart signal it is modelled on (restart.go) — no push channel, no |
| +// box access, and the box need not even be reachable from anywhere for this |
| +// to work. |
| +// |
| +// It exists because a sidecar has no socket and nothing dials it: an |
| +// instance wakes on its next inbound request, but a hibernating sidecar has |
| +// no "next request" to wake on, so waking one from outside the box needs an |
| +// explicit signal a pull-based box can notice on its own. |
| +package host |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "fmt" |
| + "log" |
| + "os" |
| + |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// sidecarsServed maps every sidecar route this box serves to the (account, |
| +// app, name) triple its wake object lives at — name comes from Slug, the |
| +// field a sidecar route carries its short name on (registry.Route's own |
| +// doc comment; provision's reconciler stamps it from the sidecar record). |
| +// A sidecar route with no Slug cannot be resolved to a wake key and is |
| +// skipped: it has nothing this pass can act on. |
| +func sidecarsServed(routes []registry.Route) map[[3]string]registry.Route { |
| + out := make(map[[3]string]registry.Route) |
| + for _, rt := range routes { |
| + if !rt.IsSidecar() || rt.Slug == "" { |
| + continue |
| + } |
| + out[[3]string{release.RouteAccount(rt), rt.App, rt.Slug}] = rt |
| + } |
| + return out |
| +} |
| + |
| +// sidecarWakePass is sidecar-wake's counterpart to restartGenPass |
| +// (restart.go): read the wake-object etag of every sidecar this box serves |
| +// and, for each one that MOVED, call wake with that sidecar's route. |
| +// |
| +// - First sight of a sidecar records silently and wakes nothing. A box |
| +// (or an agent process) that has just started must not wake every |
| +// sidecar it serves because it has never seen these etags before — that |
| +// is the seed-don't-replay rule the serial watcher already applies |
| +// (agent.go's fastTick), and it is the whole safety property of this |
| +// feature. Without it, an agent restart would replay every wake this |
| +// app's sidecars have ever received — a thundering herd on boot, and |
| +// for a hibernating fleet a real cost, not a cosmetic one. |
| +// - An unchanged etag does nothing. |
| +// - Absent that stays absent does nothing, forever: a sidecar nobody has |
| +// ever woken this way has no object and never will until someone asks. |
| +// - Any movement wakes: the etag changed, or absent -> present (the |
| +// sidecar's first ever wake request). |
| +// - A wake object naming a sidecar this box does not serve is never even |
| +// read: the pass only ever builds keys from sidecarsServed, which comes |
| +// from THIS box's own route table. Such an object, however it moves in |
| +// the bucket, produces neither a wake nor an error here — it is simply |
| +// never asked about. |
| +// - A read error for one sidecar leaves its cache entry untouched and the |
| +// rest of the pass keeps going; the error comes back for the caller to |
| +// aggregate into its own log line, same discipline as restartGenPass. |
| +// - A sidecar route that vanishes from the route set loses its cache |
| +// entry: a re-added sidecar is a first sight again, which is right, |
| +// because a route that has just been (re-)minted is starting fresh |
| +// anyway. |
| +// |
| +// The cache is mutated in place and is expected to live for the process |
| +// lifetime, exactly like restartGenPass's — there is nowhere else to |
| +// remember what "changed" means between two requests that may be days |
| +// apart. wake may be nil (no activator armed on this box): a working no-op. |
| +func sidecarWakePass(ctx context.Context, routes []registry.Route, store release.Store, cache map[[3]string]string, wake func(rt registry.Route) error) (woken []string, readErrs []error) { |
| + served := sidecarsServed(routes) |
| + live := make(map[[3]string]bool, len(served)) |
| + for k, rt := range served { |
| + live[k] = true |
| + _, etag, err := store.Get(ctx, release.SidecarWakeKey(k[0], k[1], k[2])) |
| + switch { |
| + case errors.Is(err, os.ErrNotExist): |
| + etag = genAbsent |
| + case err != nil: |
| + readErrs = append(readErrs, fmt.Errorf("%s/%s/%s: %w", k[0], k[1], k[2], err)) |
| + continue // cache entry untouched: unknown this tick, not changed |
| + } |
| + prev, seen := cache[k] |
| + cache[k] = etag |
| + if !seen || prev == etag { |
| + continue |
| + } |
| + if wake == nil { |
| + continue |
| + } |
| + if err := wake(rt); err != nil { |
| + log.Printf("agent: sidecar wake requested: wake %s: %v", rt.Host, err) |
| + continue |
| + } |
| + woken = append(woken, rt.Host) |
| + log.Printf("agent: sidecar wake requested: woke %s", rt.Host) |
| + } |
| + for k := range cache { |
| + if !live[k] { |
| + delete(cache, k) |
| + } |
| + } |
| + return woken, readErrs |
| +} |
diff --git a/internal/host/sidecarwake_test.go b/internal/host/sidecarwake_test.go| new file mode 100644 |
| index 000000000..efae9b48a |
| --- /dev/null |
| +++ b/internal/host/sidecarwake_test.go |
| @@ -0,0 +1,223 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "os" |
| + "slices" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// wakeSidecarRoute is sidecarRoute (restart_test.go) with Slug set — the |
| +// field a sidecar route carries its short name on (registry.Route.Slug's |
| +// doc comment; provision's reconciler stamps it from the sidecar record). |
| +// sidecarWakePass keys entirely off it, so a fixture that omits it is |
| +// useless here even though restart_test.go's sidecarRoute never needed it. |
| +func wakeSidecarRoute(host, acct, app, slug string) registry.Route { |
| + rt := sidecarRoute(host, acct, app) |
| + rt.Slug = slug |
| + return rt |
| +} |
| + |
| +// touchSidecarWake writes a sidecar-wake object directly, standing in for |
| +// the console door Task 12 has not built yet on this branch — exactly the |
| +// role racingRestartPutStore's raw JSON plays in restartgen_test.go. Every |
| +// call produces a BYTE-DIFFERENT body (an incrementing generation), because |
| +// DirStore's etag is content-derived and a byte-identical rewrite would not |
| +// move it. |
| +func touchSidecarWake(t *testing.T, ctx context.Context, s release.Store, acct, app, name string, gen int) { |
| + t.Helper() |
| + body, err := json.Marshal(release.SidecarWake{ |
| + Schema: 1, Generation: gen, At: "2026-08-17T23:05:00Z", |
| + By: "member:bdm", Reason: "test wake", |
| + }) |
| + if err != nil { |
| + t.Fatalf("marshal sidecar wake: %v", err) |
| + } |
| + key := release.SidecarWakeKey(acct, app, name) |
| + _, etag, err := s.Get(ctx, key) |
| + if err != nil && !errors.Is(err, os.ErrNotExist) { |
| + t.Fatalf("read sidecar wake %s: %v", key, err) |
| + } |
| + if errors.Is(err, os.ErrNotExist) { |
| + etag = "" |
| + } |
| + if _, err := s.Put(ctx, key, body, etag); err != nil { |
| + t.Fatalf("put sidecar wake %s: %v", key, err) |
| + } |
| +} |
| + |
| +func TestSidecarWakePassEtagMoveWakesExactlyThatSidecar(t *testing.T) { |
| + ctx, s := genKit(t) |
| + routes := []registry.Route{ |
| + wakeSidecarRoute("bot.eleven.sidecar.internal", "bdm", "eleven", "bot"), |
| + wakeSidecarRoute("other.eleven.sidecar.internal", "bdm", "eleven", "other"), |
| + } |
| + cache := map[[3]string]string{} |
| + // Seed: first sight of both, silently. |
| + sidecarWakePass(ctx, routes, s, cache, nil) |
| + |
| + touchSidecarWake(t, ctx, s, "bdm", "eleven", "bot", 1) |
| + var woken []string |
| + wake := func(rt registry.Route) error { woken = append(woken, rt.Host); return nil } |
| + got, errs := sidecarWakePass(ctx, routes, s, cache, wake) |
| + if len(errs) != 0 { |
| + t.Fatalf("unexpected read errors: %v", errs) |
| + } |
| + if !slices.Equal(got, []string{"bot.eleven.sidecar.internal"}) { |
| + t.Fatalf("sidecarWakePass returned %v, want exactly the moved sidecar", got) |
| + } |
| + if !slices.Equal(woken, []string{"bot.eleven.sidecar.internal"}) { |
| + t.Fatalf("woken = %v, want exactly the named sidecar and nothing else", woken) |
| + } |
| +} |
| + |
| +func TestSidecarWakePassUnchangedEtagWakesNothing(t *testing.T) { |
| + ctx, s := genKit(t) |
| + routes := []registry.Route{wakeSidecarRoute("bot.eleven.sidecar.internal", "bdm", "eleven", "bot")} |
| + touchSidecarWake(t, ctx, s, "bdm", "eleven", "bot", 1) |
| + cache := map[[3]string]string{} |
| + sidecarWakePass(ctx, routes, s, cache, nil) // seed |
| + |
| + var woken []string |
| + wake := func(rt registry.Route) error { woken = append(woken, rt.Host); return nil } |
| + got, errs := sidecarWakePass(ctx, routes, s, cache, wake) |
| + if len(errs) != 0 { |
| + t.Fatalf("unexpected read errors: %v", errs) |
| + } |
| + if len(got) != 0 || len(woken) != 0 { |
| + t.Fatalf("unchanged etag woke %v (returned %v), want none", woken, got) |
| + } |
| +} |
| + |
| +// TestSidecarWakePassFirstSightDoesNotReplay is the no-replay-on-absence |
| +// rule: an unseeded cache must not treat "I have never looked at this |
| +// before" as movement, or every agent restart would wake every sidecar |
| +// this app has ever asked to wake — a thundering herd on boot. |
| +func TestSidecarWakePassFirstSightDoesNotReplay(t *testing.T) { |
| + ctx, s := genKit(t) |
| + // Wake object already exists BEFORE this box ever looks — the shape a |
| + // long-lived wake request or a box that just (re)started sees. |
| + touchSidecarWake(t, ctx, s, "bdm", "eleven", "bot", 1) |
| + routes := []registry.Route{wakeSidecarRoute("bot.eleven.sidecar.internal", "bdm", "eleven", "bot")} |
| + cache := map[[3]string]string{} |
| + |
| + var woken []string |
| + wake := func(rt registry.Route) error { woken = append(woken, rt.Host); return nil } |
| + got, errs := sidecarWakePass(ctx, routes, s, cache, wake) |
| + if len(errs) != 0 { |
| + t.Fatalf("unexpected read errors: %v", errs) |
| + } |
| + if len(got) != 0 || len(woken) != 0 { |
| + t.Fatalf("first sight woke %v (returned %v) — a box that just started must not replay every past wake", woken, got) |
| + } |
| + want := touchedEtag(t, ctx, s, "bdm", "eleven", "bot") |
| + if got := cache[[3]string{"bdm", "eleven", "bot"}]; got != want { |
| + t.Fatalf("cache etag = %q, want the store's %q", got, want) |
| + } |
| + |
| + // A SECOND wake request after the seed IS movement. |
| + touchSidecarWake(t, ctx, s, "bdm", "eleven", "bot", 2) |
| + got, errs = sidecarWakePass(ctx, routes, s, cache, wake) |
| + if len(errs) != 0 { |
| + t.Fatalf("unexpected read errors: %v", errs) |
| + } |
| + if !slices.Equal(got, []string{"bot.eleven.sidecar.internal"}) || !slices.Equal(woken, []string{"bot.eleven.sidecar.internal"}) { |
| + t.Fatalf("second request returned %v, woken %v, want the sidecar woken exactly once", got, woken) |
| + } |
| +} |
| + |
| +func touchedEtag(t *testing.T, ctx context.Context, s release.Store, acct, app, name string) string { |
| + t.Helper() |
| + _, etag, err := s.Get(ctx, release.SidecarWakeKey(acct, app, name)) |
| + if err != nil { |
| + t.Fatalf("read sidecar wake: %v", err) |
| + } |
| + return etag |
| +} |
| + |
| +// TestSidecarWakePassReadErrorDoesNotSuppressOthers: a transient read |
| +// failure against one sidecar's wake object must not blind the pass to a |
| +// real move on a different one in the same tick. |
| +func TestSidecarWakePassReadErrorDoesNotSuppressOthers(t *testing.T) { |
| + ctx, s0 := genKit(t) |
| + routes := []registry.Route{ |
| + wakeSidecarRoute("bot.eleven.sidecar.internal", "bdm", "eleven", "bot"), |
| + wakeSidecarRoute("worker.titogo.sidecar.internal", "bdm", "titogo", "worker"), |
| + } |
| + cache := map[[3]string]string{} |
| + sidecarWakePass(ctx, routes, s0, cache, nil) // seed both, absent |
| + |
| + touchSidecarWake(t, ctx, s0, "bdm", "eleven", "bot", 1) |
| + touchSidecarWake(t, ctx, s0, "bdm", "titogo", "worker", 1) |
| + failing := &failingKeyStore{Store: s0, failKey: release.SidecarWakeKey("bdm", "eleven", "bot"), err: errors.New("bucket blip")} |
| + |
| + var woken []string |
| + wake := func(rt registry.Route) error { woken = append(woken, rt.Host); return nil } |
| + got, errs := sidecarWakePass(ctx, routes, failing, cache, wake) |
| + if len(errs) != 1 { |
| + t.Fatalf("errs = %v, want exactly one (the failing key)", errs) |
| + } |
| + if !slices.Equal(got, []string{"worker.titogo.sidecar.internal"}) || !slices.Equal(woken, []string{"worker.titogo.sidecar.internal"}) { |
| + t.Fatalf("returned %v, woken %v, want only the unaffected sidecar moved", got, woken) |
| + } |
| +} |
| + |
| +// TestSidecarWakePassIgnoresUnservedSidecar: a wake object naming a sidecar |
| +// this box does not serve is never even read, let alone acted on or |
| +// reported as an error — the pass only ever asks about routes this box |
| +// actually has. |
| +func TestSidecarWakePassIgnoresUnservedSidecar(t *testing.T) { |
| + ctx, s := genKit(t) |
| + // A wake request for a sidecar this box has no route for at all. |
| + touchSidecarWake(t, ctx, s, "bdm", "eleven", "ghost", 1) |
| + routes := []registry.Route{wakeSidecarRoute("bot.eleven.sidecar.internal", "bdm", "eleven", "bot")} |
| + cache := map[[3]string]string{} |
| + sidecarWakePass(ctx, routes, s, cache, nil) // seed |
| + |
| + touchSidecarWake(t, ctx, s, "bdm", "eleven", "ghost", 2) // "ghost" moves again |
| + var woken []string |
| + wake := func(rt registry.Route) error { woken = append(woken, rt.Host); return nil } |
| + got, errs := sidecarWakePass(ctx, routes, s, cache, wake) |
| + if len(errs) != 0 { |
| + t.Fatalf("unexpected read errors: %v", errs) |
| + } |
| + if len(got) != 0 || len(woken) != 0 { |
| + t.Fatalf("an unserved sidecar's wake object produced %v (woken %v), want none", got, woken) |
| + } |
| + if _, ok := cache[[3]string{"bdm", "eleven", "ghost"}]; ok { |
| + t.Fatal("a sidecar this box does not serve must never appear in its cache") |
| + } |
| +} |
| + |
| +// TestSidecarWakePassIgnoresNonSidecarRouteWithMatchingSlug: Slug is not a |
| +// sidecar-only field (registry.Route.Slug's doc comment — it is the |
| +// app-account short name usable on a root domain, stamped on ordinary |
| +// routes too), so an ordinary instance route that happens to carry the same |
| +// Slug as a real sidecar name must never be mistaken for that sidecar and |
| +// must never trigger a read of, or a wake from, its wake object. |
| +func TestSidecarWakePassIgnoresNonSidecarRouteWithMatchingSlug(t *testing.T) { |
| + ctx, s := genKit(t) |
| + touchSidecarWake(t, ctx, s, "bdm", "eleven", "bot", 1) |
| + notSidecar := execRoute("app.example.com", "bdm", "eleven") |
| + notSidecar.Slug = "bot" // same name a sidecar could carry, wrong Kind |
| + routes := []registry.Route{notSidecar} |
| + cache := map[[3]string]string{} |
| + |
| + var woken []string |
| + wake := func(rt registry.Route) error { woken = append(woken, rt.Host); return nil } |
| + got, errs := sidecarWakePass(ctx, routes, s, cache, wake) |
| + if len(errs) != 0 { |
| + t.Fatalf("unexpected read errors: %v", errs) |
| + } |
| + if len(got) != 0 || len(woken) != 0 { |
| + t.Fatalf("a non-sidecar route with a matching Slug produced %v (woken %v), want none", got, woken) |
| + } |
| + if len(cache) != 0 { |
| + t.Fatalf("cache = %v, want empty — a non-sidecar route must never seed a sidecar cache entry", cache) |
| + } |
| +} |
diff --git a/internal/release/sidecarwake.go b/internal/release/sidecarwake.go| new file mode 100644 |
| index 000000000..5a5cabdd6 |
| --- /dev/null |
| +++ b/internal/release/sidecarwake.go |
| @@ -0,0 +1,45 @@ |
| +// The wake signal for a hibernating sidecar: one tiny CAS-guarded object per |
| +// sidecar, watched by etag, exactly one door over from restartgen.go's |
| +// restart signal and appconfig's config-gen object — the same mechanism a |
| +// third time, because "tell a box to act, with no push channel and no box |
| +// access" is one problem no matter what the box is being told to do. |
| +// |
| +// Bucket layout addition: |
| +// |
| +// apps/<acct>/<app>/sidecar-wake/<name>.json CAS-guarded, schema-1 |
| +// |
| +// It is a SIBLING prefix under the app, deliberately not a key under |
| +// sidecars/: provision.ParseSidecarKey treats anything but exactly one |
| +// non-empty segment after "sidecars/" as not-a-record, and a wake object |
| +// living inside that prefix would either have to be taught to that parser |
| +// (widening what it accepts) or silently misparse as a sidecar name of |
| +// "<name>.json" — either way the family's "exactly one segment" rule stops |
| +// meaning what it says. A sibling prefix costs nothing and keeps that rule |
| +// exactly as strict as it reads. |
| +package release |
| + |
| +// SidecarWake is the object a wake request touches. Change detection is by |
| +// ETAG, never by Generation: a monotonic counter that resets — a bucket |
| +// restored from backup, an object rewritten by a script that does not know |
| +// the current value — must not wedge a box's cache into believing every |
| +// future request lands on a value it has "already seen". Generation, At, By |
| +// and Reason exist so a human debugging "why did this sidecar wake?" has an |
| +// answer; a box reading this object for its own purposes reads only its |
| +// etag and never unmarshals the body at all. |
| +// |
| +// It carries no authority: the worst a forged touch can do is wake a |
| +// sidecar that was going to run under its own signed release anyway. Waking |
| +// is not deciding what runs, the same way a restart-generation touch isn't. |
| +type SidecarWake struct { |
| + Schema int `json:"schema"` // 1 |
| + Generation int `json:"generation"` |
| + At string `json:"at"` |
| + By string `json:"by"` |
| + Reason string `json:"reason"` |
| +} |
| + |
| +// SidecarWakeKey is the bucket key of name's wake object within app, derived |
| +// from AppPrefix like every other per-app key. |
| +func SidecarWakeKey(acct, app, name string) string { |
| + return AppPrefix(acct, app) + "sidecar-wake/" + name + ".json" |
| +} |
diff --git a/internal/release/sidecarwake_test.go b/internal/release/sidecarwake_test.go| new file mode 100644 |
| index 000000000..1a3abfadb |
| --- /dev/null |
| +++ b/internal/release/sidecarwake_test.go |
| @@ -0,0 +1,48 @@ |
| +package release |
| + |
| +import ( |
| + "encoding/json" |
| + "testing" |
| +) |
| + |
| +// TestSidecarWakeKeyIsAnAppSibling pins the key's shape: a SIBLING prefix |
| +// under the app's own account-scoped prefix, deliberately not a key under |
| +// sidecars/ — provision.ParseSidecarKey's "exactly one non-empty segment |
| +// after sidecars/" rule stays exactly as strict as it reads only if nothing |
| +// else ever lands under that prefix. |
| +func TestSidecarWakeKeyIsAnAppSibling(t *testing.T) { |
| + got, want := SidecarWakeKey("bdm", "eleven", "bot"), "apps/bdm/eleven/sidecar-wake/bot.json" |
| + if got != want { |
| + t.Fatalf("SidecarWakeKey = %q, want %q", got, want) |
| + } |
| +} |
| + |
| +// TestSidecarWakeJSONShape pins the wire shape against the exact example the |
| +// design settled on, so a field rename or a json-tag typo fails loudly |
| +// instead of silently breaking every box's etag comparison (which never |
| +// parses the body at all) while still breaking a human reading the object |
| +// during an incident. |
| +func TestSidecarWakeJSONShape(t *testing.T) { |
| + w := SidecarWake{ |
| + Schema: 1, |
| + Generation: 3, |
| + At: "2026-08-17T23:05:00Z", |
| + By: "member:bdm", |
| + Reason: "operator wake", |
| + } |
| + raw, err := json.Marshal(w) |
| + if err != nil { |
| + t.Fatalf("marshal: %v", err) |
| + } |
| + const want = `{"schema":1,"generation":3,"at":"2026-08-17T23:05:00Z","by":"member:bdm","reason":"operator wake"}` |
| + if string(raw) != want { |
| + t.Fatalf("SidecarWake JSON = %s, want %s", raw, want) |
| + } |
| + var back SidecarWake |
| + if err := json.Unmarshal(raw, &back); err != nil { |
| + t.Fatalf("unmarshal: %v", err) |
| + } |
| + if back != w { |
| + t.Fatalf("round trip = %+v, want %+v", back, w) |
| + } |
| +} |