amadan
release+host: Materialize stages the channel head at cold wake (closes #77)
A cold wake of an exec-backed route with nothing staged 503s ("no live
binary") until the hourly adopt pass comes round. release.MaterializeHost
closes that window: it IS the adopt pass, filtered to one host with
restart=nil (the host is cold — the activator execing it is the restart),
so signature verification, the bake hold, staging and the Version stamp are
all the pass's own. A second staging path is exactly the drift this refuses
to create.
Success is judged by the live symlink, not the pass error: PassFilter logs
per-route trouble rather than returning it, and "bake-held", "nothing
promoted" and "no such host" all correctly leave the link absent — so an
absent link is MaterializeHost's error. A cold wake must never become a
bake bypass; the 503 is then correct and says why.
The agent wires it through materializeHook, which degrades to nil with no
release store or no CARLOS_RELEASE_PUBKEY — nil is today's behavior exactly.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
4 files changed,
+299
−4
internal/host/agent.go+36 −4internal/host/agent_test.go+93 −0internal/release/materialize.go+42 −0internal/release/materialize_test.go+128 −0
diff --git a/internal/host/agent.go b/internal/host/agent.go| index 53743493c..433fba9f9 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -144,10 +144,11 @@ func RunAgent(args []string) { |
| // never touch the bucket — one opt-in switch for the whole activator. |
| svc = activator.New(*cfg, &activator.SplitBackend{ |
| Exec: &activator.ExecBackend{ |
| - Cfg: *cfg, |
| - OptDir: release.OptDir(), |
| - DataDir: dd, |
| - Cache: cache, |
| + Cfg: *cfg, |
| + OptDir: release.OptDir(), |
| + DataDir: dd, |
| + Cache: cache, |
| + Materialize: materializeHook(reg, dd), |
| }, |
| Unit: &activator.UnitBackend{OptDir: release.OptDir()}, |
| }, reg) // the registry satisfies activator.DueStore directly |
| @@ -203,6 +204,37 @@ func peekRegistryPath(args []string, dd string) string { |
| return *regPath |
| } |
| +// materializeHook builds the ExecBackend's cold-wake staging hook, or nil |
| +// when this box has no release store/pubkey — nil keeps today's behavior |
| +// ("no live binary for <host>" wake failure), the safe degrade for a box |
| +// whose routes are versioned by hand or not at all. |
| +// |
| +// The hook is the adopt pass scoped to one host (release.MaterializeHost), |
| +// so a cold wake with nothing staged pays one slow request instead of 503ing |
| +// until the hourly pass comes round (issue #77). Everything the pass refuses |
| +// it still refuses here: a bake-held head is NOT materialized, and the wake |
| +// fails with that reason rather than the head being installed early. |
| +// |
| +// Store and key are resolved ONCE, at agent start, exactly like adoptLoop's: |
| +// a wake is on a request's critical path, and re-reading env plus re-opening |
| +// an S3 client per cold wake would put that work there for nothing. |
| +func materializeHook(reg *registry.Registry, dd string) func(context.Context, string) error { |
| + store, err := openDeployStore(context.Background()) |
| + if err != nil { |
| + log.Printf("agent: cold-wake materialize disabled: %v", err) |
| + return nil |
| + } |
| + pub, err := release.ParsePublicKey(os.Getenv("CARLOS_RELEASE_PUBKEY"), "CARLOS_RELEASE_PUBKEY") |
| + if err != nil { |
| + log.Printf("agent: cold-wake materialize disabled: %v", err) |
| + return nil |
| + } |
| + optDir := release.OptDir() |
| + return func(ctx context.Context, host string) error { |
| + return release.MaterializeHost(ctx, reg, store, pub, optDir, dd, host) |
| + } |
| +} |
| + |
| // adoptLoop runs a release-adoption pass roughly hourly (plus jitter, so a |
| // fleet never hits the store in the same second), forever. Missing deployment |
| // config disables the loop with one log line — an agent with no release store |
diff --git a/internal/host/agent_test.go b/internal/host/agent_test.go| index 3bddab9a9..66e320903 100644 |
| --- a/internal/host/agent_test.go |
| +++ b/internal/host/agent_test.go |
| @@ -2,7 +2,11 @@ package host |
| import ( |
| "context" |
| + "crypto/ed25519" |
| + "encoding/base64" |
| "errors" |
| + "path/filepath" |
| + "runtime" |
| "testing" |
| "github.com/carlosframework/platform/internal/appconfig" |
| @@ -386,3 +390,92 @@ func TestGenPollPassPrunesUnitAppsToo(t *testing.T) { |
| t.Fatalf("unexpected read errors: %v", errs) |
| } |
| } |
| + |
| +// ---- cold-wake materialize hook (carlos-deploy spec §5) --------------------- |
| + |
| +// TestMaterializeHookDegradesToNil pins the safe degrade: a box with no |
| +// release store, or no (or a broken) release pubkey, gets a NIL hook — which |
| +// is today's behavior exactly, the ExecBackend's "no live binary" wake |
| +// failure. A non-nil hook built from a half-configured box would turn every |
| +// cold wake into a staging error that names the wrong problem. |
| +func TestMaterializeHookDegradesToNil(t *testing.T) { |
| + reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| + if err != nil { |
| + t.Fatalf("open registry: %v", err) |
| + } |
| + pub, _, err := ed25519.GenerateKey(nil) |
| + if err != nil { |
| + t.Fatalf("generate key: %v", err) |
| + } |
| + pubB64 := base64.StdEncoding.EncodeToString(pub) |
| + |
| + for _, c := range []struct{ name, dir, pub string }{ |
| + {"no store, no key", "", ""}, |
| + {"store but no key", t.TempDir(), ""}, |
| + {"store but unparseable key", t.TempDir(), "not-base64!!"}, |
| + {"key but no store", "", pubB64}, |
| + } { |
| + t.Run(c.name, func(t *testing.T) { |
| + t.Setenv("CARLOS_DEPLOYMENT_BUCKET", "") |
| + t.Setenv("CARLOS_DEPLOYMENT_DIR", c.dir) |
| + t.Setenv("CARLOS_RELEASE_PUBKEY", c.pub) |
| + if hook := materializeHook(reg, t.TempDir()); hook != nil { |
| + t.Fatal("hook must be nil on a box that cannot adopt — nil is today's behavior") |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// TestMaterializeHookStagesAColdRoute exercises the wired path end to end |
| +// through the very function the ExecBackend calls: a fully configured box, a |
| +// route with nothing staged, and after one hook call the live symlink exists |
| +// and names the channel head. This is what retires "no live binary" as a |
| +// wake-failure class (issue #77). |
| +func TestMaterializeHookStagesAColdRoute(t *testing.T) { |
| + dir := t.TempDir() |
| + optDir := t.TempDir() |
| + s := release.NewDirStore(dir) |
| + ctx := context.Background() |
| + plat := runtime.GOOS + "-" + runtime.GOARCH |
| + |
| + pub, priv, err := ed25519.GenerateKey(nil) |
| + if err != nil { |
| + t.Fatalf("generate key: %v", err) |
| + } |
| + reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| + if err != nil { |
| + t.Fatalf("open registry: %v", err) |
| + } |
| + if err := reg.Upsert(registry.Route{ |
| + Host: "cold.example.com", Kind: "instance", App: "coldapp", |
| + Channel: "canary/dev", Status: "ready", |
| + }); err != nil { |
| + t.Fatalf("upsert: %v", err) |
| + } |
| + m := release.Manifest{ |
| + App: "coldapp", Version: "abc1234", Kind: "binary", |
| + Artifacts: map[string]release.Artifact{plat: {}}, |
| + } |
| + if err := release.Ship(ctx, s, "bab", m, map[string][]byte{plat: []byte("cold build")}); err != nil { |
| + t.Fatalf("Ship: %v", err) |
| + } |
| + if err := release.Promote(ctx, s, priv, "bab", "coldapp", "canary/dev", "abc1234"); err != nil { |
| + t.Fatalf("Promote: %v", err) |
| + } |
| + |
| + t.Setenv("CARLOS_DEPLOYMENT_BUCKET", "") |
| + t.Setenv("CARLOS_DEPLOYMENT_DIR", dir) |
| + t.Setenv("CARLOS_RELEASE_PUBKEY", base64.StdEncoding.EncodeToString(pub)) |
| + t.Setenv("CARLOS_OPT_DIR", optDir) |
| + |
| + hook := materializeHook(reg, t.TempDir()) |
| + if hook == nil { |
| + t.Fatal("a fully configured box must get a hook") |
| + } |
| + if err := hook(ctx, "cold.example.com"); err != nil { |
| + t.Fatalf("hook: %v", err) |
| + } |
| + if v, err := release.LiveVersion(optDir, "cold.example.com"); err != nil || v != "abc1234" { |
| + t.Fatalf("live = %q, %v; want abc1234", v, err) |
| + } |
| +} |
diff --git a/internal/release/materialize.go b/internal/release/materialize.go| new file mode 100644 |
| index 000000000..026146a43 |
| --- /dev/null |
| +++ b/internal/release/materialize.go |
| @@ -0,0 +1,42 @@ |
| +package release |
| + |
| +import ( |
| + "context" |
| + "crypto/ed25519" |
| + "fmt" |
| + |
| + "github.com/carlosframework/platform/internal/registry" |
| +) |
| + |
| +// MaterializeHost stages host's channel head and points its live symlink — |
| +// the cold-wake, on-demand form of the adopt pass (carlos-deploy spec §5). |
| +// It IS the adopt pass, filtered to one host with restart=nil (the host is |
| +// cold; the activator execing it is the "restart"): bake holds, signature |
| +// verification, staging, and the Version stamp are all the pass's own — |
| +// a second staging path is exactly the drift this refuses to create. |
| +// |
| +// Success is judged by the symlink, not the pass error: PassFilter logs |
| +// per-route trouble rather than returning it (only foundation failures — an |
| +// unreadable registry, an unwritable bake clock — come back as errors), and |
| +// "held by a bake window", "nothing promoted for this channel" and "no such |
| +// host" all correctly leave the link absent. A caller about to exec the |
| +// binary needs the link, so an absent link is this function's error. |
| +// |
| +// dataDir is where the bake clock lives (BakeDir; /data in production), so |
| +// the window a cold wake observes is the same one the hourly root pass |
| +// measures — a cold wake must never become a bake bypass by keeping its own |
| +// clock. |
| +func MaterializeHost(ctx context.Context, reg *registry.Registry, s Store, pub ed25519.PublicKey, optDir, dataDir, host string) error { |
| + only := func(rt registry.Route) bool { return rt.Host == host } |
| + if err := PassFilter(ctx, reg, s, pub, optDir, nil, only, BakeDir(dataDir)); err != nil { |
| + return fmt.Errorf("materialize %s: %w", host, err) |
| + } |
| + v, err := LiveVersion(optDir, host) |
| + if err != nil { |
| + return fmt.Errorf("materialize %s: %w", host, err) |
| + } |
| + if v == "" { |
| + return fmt.Errorf("materialize %s: no adoptable head (nothing promoted for its channel, or the head is bake-held)", host) |
| + } |
| + return nil |
| +} |
diff --git a/internal/release/materialize_test.go b/internal/release/materialize_test.go| new file mode 100644 |
| index 000000000..8f0b8b6bd |
| --- /dev/null |
| +++ b/internal/release/materialize_test.go |
| @@ -0,0 +1,128 @@ |
| +package release |
| + |
| +import ( |
| + "context" |
| + "crypto/ed25519" |
| + "strings" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/registry" |
| +) |
| + |
| +// matFixture is the shape every test below needs: a dir store with a shipped |
| +// release, a signed channel pointer, a registry holding the route(s), a fresh |
| +// optDir with NOTHING staged, and a dataDir for the bake clock. Built inline |
| +// the way adopt_test.go builds its fixtures (testKeys/shipRelease/addRoute/ |
| +// testRegistry) — this package has no shared fixture builder and inventing a |
| +// parallel harness for three tests would be the drift, not the reuse. |
| +type matFixture struct { |
| + store Store |
| + priv ed25519.PrivateKey |
| + pub ed25519.PublicKey |
| + reg *registry.Registry |
| + optDir string |
| + dataDir string |
| +} |
| + |
| +func newMatFixture(t *testing.T) *matFixture { |
| + t.Helper() |
| + priv, pub := testKeys(t) |
| + return &matFixture{ |
| + store: NewDirStore(t.TempDir()), |
| + priv: priv, |
| + pub: pub, |
| + reg: testRegistry(t), |
| + optDir: t.TempDir(), |
| + dataDir: t.TempDir(), |
| + } |
| +} |
| + |
| +// TestMaterializeHostStagesChannelHead is the cold-wake case the activator |
| +// hits: an exec-backed route with nothing staged at all. MaterializeHost must |
| +// leave the live symlink pointing at the channel head — and must touch ONLY |
| +// the named host, since the wake it serves is that host's alone. |
| +func TestMaterializeHostStagesChannelHead(t *testing.T) { |
| + env := newMatFixture(t) |
| + ctx := context.Background() |
| + |
| + // canary/* has no bake window, so the head is immediately adoptable. |
| + shipRelease(t, env.store, "bab", "matapp", "abc1234", adoptPlatform, []byte("cold wake build")) |
| + if err := Promote(ctx, env.store, env.priv, "bab", "matapp", "canary/dev", "abc1234"); err != nil { |
| + t.Fatalf("Promote: %v", err) |
| + } |
| + addRoute(t, env.reg, "mat.test", "matapp", "canary/dev", false) |
| + // A sibling on the same channel, equally unstaged: waking one host must not |
| + // stage the fleet. |
| + addRoute(t, env.reg, "other.test", "matapp", "canary/dev", false) |
| + |
| + if err := MaterializeHost(ctx, env.reg, env.store, env.pub, env.optDir, env.dataDir, "mat.test"); err != nil { |
| + t.Fatalf("MaterializeHost: %v", err) |
| + } |
| + |
| + v, err := LiveVersion(env.optDir, "mat.test") |
| + if err != nil || v != "abc1234" { |
| + t.Fatalf("live = %q, %v; want abc1234", v, err) |
| + } |
| + if v, _ := LiveVersion(env.optDir, "other.test"); v != "" { |
| + t.Fatalf("sibling host was materialized too (%q) — a wake stages one host", v) |
| + } |
| +} |
| + |
| +// TestMaterializeHostRefusesHeldHead: a cold wake must NOT become a bake |
| +// bypass (spec §5). The app is declared production and the route follows beta |
| +// (24h hold) with a head promoted moments ago, so the pass correctly stages |
| +// nothing — and MaterializeHost must report that as an error, because the |
| +// caller's next move is exec'ing a binary that isn't there. |
| +func TestMaterializeHostRefusesHeldHead(t *testing.T) { |
| + env := newMatFixture(t) |
| + ctx := context.Background() |
| + |
| + shipRelease(t, env.store, "bab", "matapp", "abc1234", adoptPlatform, []byte("held build")) |
| + if err := Promote(ctx, env.store, env.priv, "bab", "matapp", "beta", "abc1234"); err != nil { |
| + t.Fatalf("Promote: %v", err) |
| + } |
| + if err := SetProduction(ctx, env.store, env.priv, "bab", "matapp", true, "test"); err != nil { |
| + t.Fatalf("SetProduction: %v", err) |
| + } |
| + addRoute(t, env.reg, "mat.test", "matapp", "beta", false) |
| + |
| + err := MaterializeHost(ctx, env.reg, env.store, env.pub, env.optDir, env.dataDir, "mat.test") |
| + if err == nil { |
| + t.Fatal("materialize of a bake-held head must refuse") |
| + } |
| + if !strings.Contains(err.Error(), "mat.test") { |
| + t.Errorf("error %q does not name the host", err) |
| + } |
| + if v, _ := LiveVersion(env.optDir, "mat.test"); v != "" { |
| + t.Fatalf("held head was staged anyway: %q", v) |
| + } |
| +} |
| + |
| +// TestMaterializeHostUnknownHost: a host the registry has never heard of |
| +// matches no route, so the pass is a no-op and the symlink stays absent. That |
| +// must surface as an error rather than a silent success the activator would |
| +// read as "go ahead and exec". |
| +func TestMaterializeHostUnknownHost(t *testing.T) { |
| + env := newMatFixture(t) |
| + if err := MaterializeHost(context.Background(), env.reg, env.store, env.pub, env.optDir, env.dataDir, "nope.test"); err == nil { |
| + t.Fatal("unknown host must error") |
| + } |
| +} |
| + |
| +// TestMaterializeHostNothingPromoted: the route exists and the app has |
| +// shipped, but its channel has no pointer — nothing to stage, and the same |
| +// error contract as a held head (the link is what is judged, not the pass). |
| +func TestMaterializeHostNothingPromoted(t *testing.T) { |
| + env := newMatFixture(t) |
| + ctx := context.Background() |
| + |
| + shipRelease(t, env.store, "bab", "matapp", "abc1234", adoptPlatform, []byte("never promoted")) |
| + addRoute(t, env.reg, "mat.test", "matapp", "canary/dev", false) |
| + |
| + if err := MaterializeHost(ctx, env.reg, env.store, env.pub, env.optDir, env.dataDir, "mat.test"); err == nil { |
| + t.Fatal("a channel with nothing promoted must error, not report success") |
| + } |
| + if v, _ := LiveVersion(env.optDir, "mat.test"); v != "" { |
| + t.Fatalf("live = %q, want absent", v) |
| + } |
| +} |