carlos / platform Public

feat: fast-adopt canary channels every few seconds, not hourly

Splits two axes that were conflated: bake window (canary/* already
bypasses this) and poll latency (it didn't -- a canary release still sat
unnoticed for up to an hour, waiting on the same hourly+jitter cycle as
stable). Adds canaryAdoptLoop, a second loop scoped strictly to routes
following a canary/* channel, sweeping every ~2s instead of hourly.
Deliberately NOT fleet-wide: canary channels are individually created and
inherently low-cardinality (one deliberate rollout at a time), so this
can't turn into a poll storm the way fast-polling every route would.
Stable/beta stay on the existing hourly loop -- bake-gated for a day or
three regardless, so faster polling buys nothing there.

release.Pass is now a thin wrapper around a new release.PassFilter(...,
extra) -- zero signature change for Pass's 9 existing call sites (agent.go,
cli.go's `carlos adopt`, and 7 test call sites), the filter is purely
additive.

Solves the real use case that surfaced it: pushing a fix to one Tito
customer's instance via canary, and having it land in seconds instead of
up to an hour.

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>

Paul Campbell 2 months ago 34d744f49fa8d648f0149b407f847e3a90c5bd36 parent 0b8caea

4 files changed, +121 −9

internal/host/agent.go +51 −7
internal/host/host_test.go +20 −0
internal/release/adopt.go +19 −2
internal/release/adopt_test.go +31 −0

diff --git a/internal/host/agent.go b/internal/host/agent.go

index 0277c1d7a..abdc6a214 100644

--- a/internal/host/agent.go

+++ b/internal/host/agent.go

@@ -1,11 +1,13 @@

// The `carlos agent` verb — the single-box role: edge (TLS + host routing) +

-// hibernation activator + hourly release adoption + the fleet status page's

-// 5-minute self-report composed in one process. The composition is new

-// here, but each part is a port: the edge from Eleven's router.go, the

-// activator from lchat activator.go, the adopt loop is release.Pass on the

-// cadence of Eleven's updater timer, and the status loop is

-// internal/status's CollectBox/WriteBox/BuildFleet/PublishFleet run on a

-// timer with one publisher elected via internal/lease.

+// hibernation activator + hourly release adoption (plus a canary/*-only

+// fast-adopt sweep every few seconds, canaryAdoptLoop) + the fleet status

+// page's 5-minute self-report composed in one process. The composition is

+// new here, but each part is a port: the edge from Eleven's router.go, the

+// activator from lchat activator.go, the adopt loop is release.Pass (or,

+// for canaries, release.PassFilter) on the cadence of Eleven's updater

+// timer, and the status loop is internal/status's

+// CollectBox/WriteBox/BuildFleet/PublishFleet run on a timer with one

+// publisher elected via internal/lease.

// Fleet status page config surface (every env var any part of this file

// reads for it, gathered in one place since no single function owns the

@@ -133,6 +135,7 @@ func RunAgent(args []string) {

}

go adoptLoop(reg)

+ go canaryAdoptLoop(reg)

go statusLoop(reg)

go domainsLoop(reg, peekAcmeCache(args))

@@ -197,6 +200,47 @@ func adoptLoop(reg *registry.Registry) {

}

+// canaryAdoptLoop mirrors adoptLoop but sweeps only routes following a

+// canary/* channel, every few seconds instead of hourly — "I promoted a

+// canary, why is my box still on the old version for the next 40 minutes"

+// with a real fix instead of an operator forcing `carlos adopt` by hand.

+// Bounded deliberately to canary/*: those channels are individually

+// created and inherently low-cardinality (one deliberate rollout at a

+// time, never the whole fleet — see release.PassFilter's doc comment and

+// bake.go's "canary/* has no window"), so this can't grow into a

+// fleet-wide poll storm the way sweeping every route this fast would.

+// Stable/beta routes stay on adoptLoop's hourly cadence: they're

+// bake-gated for a day or three regardless, so faster polling would buy

+// nothing there.

+func canaryAdoptLoop(reg *registry.Registry) {

+ ctx := context.Background()

+ store, err := openDeployStore(ctx)

+ if err != nil {

+ return // adoptLoop already logged this

+ }

+ pub, err := release.ParsePublicKey(os.Getenv("CARLOS_RELEASE_PUBKEY"), "CARLOS_RELEASE_PUBKEY")

+ if err != nil {

+ return // ditto

+ }

+ optDir := release.OptDir()

+ log.Printf("agent: canary adopt loop armed (2s + jitter, canary/* routes only)")

+ for {

+ time.Sleep(2*time.Second + rand.N(time.Second))

+ if err := release.PassFilter(ctx, reg, store, pub, optDir, restartRoute, isCanaryRoute); err != nil {

+ log.Printf("agent: canary adopt pass: %v", err)

+ }

+// isCanaryRoute reports whether rt follows a canary/* channel — the same

+// two-segment convention bake.go's DefaultHolds wildcard matches (a bare

+// "canary" with no slash is NOT covered, same as bake.go: it falls through

+// to the unknown-channel default there, so it shouldn't fast-poll here

+// either).

+func isCanaryRoute(rt registry.Route) bool {

+ return strings.HasPrefix(rt.Channel, "canary/")

// peekAcmeCache pre-parses the agent args for -acme-cache the same way

// peekRegistryPath peeks -registry: the edge does the authoritative parse,

// this just lets the domains loop stamp cert expiry from the SAME cache dir

diff --git a/internal/host/host_test.go b/internal/host/host_test.go

index 44b880ede..ea0a3b211 100644

--- a/internal/host/host_test.go

+++ b/internal/host/host_test.go

@@ -365,3 +365,23 @@ func TestBootstrapIdempotent(t *testing.T) {

}

+func TestIsCanaryRoute(t *testing.T) {

+ cases := []struct {

+ channel string

+ want bool

+ }{

+ {"canary/x", true},

+ {"canary/pr-142", true},

+ {"stable", false},

+ {"beta", false},

+ {"canary", false}, // no slash — same convention as bake.go's DefaultHolds wildcard

+ {"", false},

+ }

+ for _, c := range cases {

+ rt := registry.Route{Channel: c.channel}

+ if got := isCanaryRoute(rt); got != c.want {

+ t.Errorf("isCanaryRoute(Channel=%q) = %v, want %v", c.channel, got, c.want)

+ }

diff --git a/internal/release/adopt.go b/internal/release/adopt.go

index 87d0baf5c..c8dc86c86 100644

--- a/internal/release/adopt.go

+++ b/internal/release/adopt.go

@@ -39,17 +39,34 @@ var adoptPlatform = runtime.GOOS + "-" + runtime.GOARCH

// hosts are processed AFTER every awake host, so an awake host discovers a

// boot-broken build (and poisons it) before any sleeper is moved onto it.

func Pass(ctx context.Context, reg *registry.Registry, s Store, pub ed25519.PublicKey, optDir string, restart func(host string) error) error {

+ return PassFilter(ctx, reg, s, pub, optDir, restart, nil)

+// PassFilter is Pass narrowed to the routes extra additionally accepts

+// (nil extra makes it identical to Pass) — the seam a faster, narrower

+// sweep composes on top of the full one without duplicating the group

+// resolution, bake-clock, staging, or restart logic. Built for the

+// canary-only fast-adopt loop (internal/host/agent.go): canary channels

+// are individually created and inherently low-cardinality (one deliberate

+// rollout at a time, never the whole fleet — see bake.go's "canary/* has

+// no window"), so sweeping just them every few seconds instead of hourly

+// can't turn into a fleet-wide poll storm the way sweeping every route

+// that fast would.

+func PassFilter(ctx context.Context, reg *registry.Registry, s Store, pub ed25519.PublicKey, optDir string, restart func(host string) error, extra func(registry.Route) bool) error {

routes, err := reg.List()

if err != nil {

return err

}

+ adopts := func(rt registry.Route) bool {

+ return adoptable(rt) && (extra == nil || extra(rt))

+ }

// Resolve every distinct (App, Channel) once. Verification failures kill

// the group (nil entry), not the pass.

type groupKey struct{ app, channel string }

pointers := map[groupKey]*ChannelPointer{}

for _, rt := range routes {

- if !adoptable(rt) {

+ if !adopts(rt) {

continue

}

k := groupKey{rt.App, routeChannel(rt)}

@@ -96,7 +113,7 @@ func Pass(ctx context.Context, reg *registry.Registry, s Store, pub ed25519.Publ

// sleeper is moved onto it.

var hibernating []registry.Route

for _, rt := range routes {

- if !adoptable(rt) {

+ if !adopts(rt) {

continue

}

if rt.Hibernate {

diff --git a/internal/release/adopt_test.go b/internal/release/adopt_test.go

index ad98a5847..c8b3940ed 100644

--- a/internal/release/adopt_test.go

+++ b/internal/release/adopt_test.go

@@ -240,3 +240,34 @@ func TestPassMissingPlatformArtifactSkips(t *testing.T) {

t.Fatalf("live = %q, want unmoved (no artifact for this platform)", v)

}

+func TestPassFilterOnlyTouchesRoutesExtraAccepts(t *testing.T) {

+ s := NewDirStore(t.TempDir())

+ ctx := context.Background()

+ priv, pub := testKeys(t)

+ opt := t.TempDir()

+ reg := testRegistry(t)

+ shipRelease(t, s, "myapp", "v1", adoptPlatform, []byte("build one"))

+ if err := Promote(ctx, s, priv, "myapp", "canary/x", "v1"); err != nil {

+ t.Fatalf("Promote canary/x: %v", err)

+ }

+ if err := Promote(ctx, s, priv, "myapp", "stable", "v1"); err != nil {

+ t.Fatalf("Promote stable: %v", err)

+ }

+ addRoute(t, reg, "canary.example.com", "myapp", "canary/x", false)

+ addRoute(t, reg, "stable.example.com", "myapp", "stable", false)

+ canaryOnly := func(rt registry.Route) bool { return rt.Channel == "canary/x" }

+ restart := func(string) error { return nil }

+ if err := PassFilter(ctx, reg, s, pub, opt, restart, canaryOnly); err != nil {

+ t.Fatalf("PassFilter: %v", err)

+ }

+ if v, _ := LiveVersion(opt, "canary.example.com"); v != "v1" {

+ t.Errorf("canary.example.com live = %q, want v1 (extra accepts it)", v)

+ }

+ if v, _ := LiveVersion(opt, "stable.example.com"); v != "" {

+ t.Errorf("stable.example.com live = %q, want unmoved — PassFilter must not touch a route extra rejects", v)

+ }