amadan

carlos / platform Public

agent: backfill the instance key into existing <host>.env files at startup (L21)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude 4c9256bb33a00bc5cd8bc7f128bb5d6bbe3c5332 parent 1fbc04c
3 files changed, +173 −0
  • internal/host/agent.go +9 −0
  • internal/host/instancekeys.go +84 −0
  • internal/host/instancekeys_test.go +80 −0
diff --git a/internal/host/agent.go b/internal/host/agent.go
index 8ac99978b..3f6626576 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -485,6 +485,15 @@ func runAgent(args []string, buildClaims ClaimFactory) {
pass := newLayoutPass(reg, svc, dd, eb.NestedOK, swept)
intent := startupRecoveryIntent(ctx, dd, openDeployStore, metricsBoxLabel(os.Getenv, readMachineID()))
recoverLayoutAtStart(ctx, pass, intent)
+ // L21: every instance already on this box gets its key
+ // after layout recovery has settled every journal and
+ // before the layout pass is published or the activator
+ // wakes anything, so nothing moves a file under it.
+ if routes, err := reg.List(); err != nil {
+ log.Printf("agent: instance keys: list routes: %v (each host's next start mints its key)", err)
+ } else {
+ backfillInstanceKeys(routes, eb)
+ }
armedLayoutPass.Store(pass)
},
svc.Start)
diff --git a/internal/host/instancekeys.go b/internal/host/instancekeys.go
new file mode 100644
index 000000000..44fb2311b
--- /dev/null
+++ b/internal/host/instancekeys.go
@@ -0,0 +1,84 @@
+package host
+
+import (
+ "log"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// The instance-key backfill (L21, plan 2026-10-10-instance-key-provision.md
+// ruling 7). Every launch path mints CARLOS_INSTANCE_KEY before it reads the
+// env, so no process is ever started without one. The backfill gives every
+// instance already on a box its key now, at agent start, rather than at its
+// next start, so the rollout can be checked box by box before any app adopts
+// the key: a host that the backfill names "present" or "minted" is ready.
+//
+// It runs only where the legacy activator writes <host>.env itself. With
+// host claims on, every write to a host's installation runs under the
+// claim's guard, and the claim's launch source already mints the key under
+// that guard before every cold admission (claim_snapshot.go
+// EnsureClaimAdminToken), so a claim box is not backfilled here.
+
+// instanceKeyBackfiller is the activator surface the backfill needs.
+type instanceKeyBackfiller interface {
+ BackfillInstanceKey(registry.Route) (activator.InstanceKeyOutcome, error)
+}
+
+// instanceKeyRoute selects the rows the backfill writes for: ready,
+// activator-spawned instances, the rows execSyncable delivers config to,
+// without sidecars and without replicas (a replica is never minted a key,
+// ruling 26). A held row (moving, landing, restoring, releasing) belongs to
+// its operation and is skipped, as is a provisioning row whose first start
+// will mint.
+func instanceKeyRoute(rt registry.Route) bool {
+ return rt.Kind == "instance" && rt.Host != "" && rt.Socket != "" && !rt.Platform && !rt.Replica &&
+ (rt.Status == "" || rt.Status == "ready") &&
+ rt.Backing != registry.BackingUnit && rt.Tunnel == "" && rt.Unit == ""
+}
+
+// instanceKeyBackfill counts one pass's outcomes.
+type instanceKeyBackfill struct {
+ Minted, Present, FromConfig, NoEnvFile, Failed, Skipped int
+}
+
+// backfillInstanceKeys runs the backfill over routes. It never fails the
+// agent: a host it cannot write is logged and counted, and that host's next
+// start mints its key (or refuses, loudly, if the same fault is still there).
+// Nothing logged names a key.
+//
+// The error is read before the outcome: a refusal (a fenced or conflicting
+// row) answers InstanceKeyPresent together with its error, and counting it
+// as present would call a host ready whose key was never written.
+func backfillInstanceKeys(routes []registry.Route, b instanceKeyBackfiller) instanceKeyBackfill {
+ var st instanceKeyBackfill
+ for _, rt := range routes {
+ if !instanceKeyRoute(rt) {
+ st.Skipped++
+ continue
+ }
+ got, err := b.BackfillInstanceKey(rt)
+ if err != nil {
+ st.Failed++
+ log.Printf("agent: instance key for %s: %v (its next start mints it)", rt.Host, err)
+ continue
+ }
+ switch got {
+ case activator.InstanceKeyMinted:
+ st.Minted++
+ case activator.InstanceKeyPresent:
+ st.Present++
+ case activator.InstanceKeyFromConfig:
+ st.FromConfig++
+ case activator.InstanceKeyNoEnvFile:
+ st.NoEnvFile++
+ case activator.InstanceKeyReplica:
+ // Unreachable: instanceKeyRoute drops every replica row. Kept
+ // so the counts always sum to the rows listed.
+ st.Skipped++
+ }
+ }
+ log.Printf("agent: instance keys: %d minted, %d already present, %d set by app configuration, %d never started here, %d failed, %d rows not instances",
+ st.Minted, st.Present, st.FromConfig, st.NoEnvFile, st.Failed, st.Skipped)
+ return st
+}
diff --git a/internal/host/instancekeys_test.go b/internal/host/instancekeys_test.go
new file mode 100644
index 000000000..e06dfafea
--- /dev/null
+++ b/internal/host/instancekeys_test.go
@@ -0,0 +1,80 @@
+package host
+
+import (
+ "errors"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+type fakeKeyBackfiller struct {
+ got map[string]activator.InstanceKeyOutcome
+ err map[string]error
+ asked []string
+}
+
+func (f *fakeKeyBackfiller) BackfillInstanceKey(rt registry.Route) (activator.InstanceKeyOutcome, error) {
+ f.asked = append(f.asked, rt.Host)
+ return f.got[rt.Host], f.err[rt.Host]
+}
+
+func TestBackfillInstanceKeysWritesOnlyReadyActivatorInstances(t *testing.T) {
+ ready := func(host string) registry.Route {
+ return registry.Route{Host: host, Kind: "instance", App: "shop", Status: "ready", Socket: "/run/carlos/" + host + ".sock"}
+ }
+ held := ready("moving.example.com")
+ held.Status = "moving"
+ landing := ready("landing.example.com")
+ landing.Status = "landing"
+ provisioning := ready("new.example.com")
+ provisioning.Status = "provisioning"
+ unit := ready("unit.example.com")
+ unit.Backing = registry.BackingUnit
+ alias := ready("alias.example.com")
+ alias.Platform = true
+ sidecar := ready("worker.example.com")
+ sidecar.Kind = "sidecar"
+ addr := ready("addr.example.com")
+ addr.Socket, addr.Addr = "", "127.0.0.1:8444"
+ replica := ready("shop-au.example.com")
+ replica.Replica = true
+ f := &fakeKeyBackfiller{
+ got: map[string]activator.InstanceKeyOutcome{"shop.example.com": activator.InstanceKeyMinted, "old.example.com": activator.InstanceKeyPresent, "cfg.example.com": activator.InstanceKeyFromConfig},
+ err: map[string]error{"bad.example.com": errors.New("read: permission denied")},
+ }
+ st := backfillInstanceKeys([]registry.Route{ready("shop.example.com"), ready("old.example.com"), ready("cfg.example.com"), ready("bad.example.com"), held, landing, provisioning, unit, alias, sidecar, addr, replica}, f)
+ if len(f.asked) != 4 {
+ t.Fatalf("asked %v, want only the four ready activator instances", f.asked)
+ }
+ if st != (instanceKeyBackfill{Minted: 1, Present: 1, FromConfig: 1, Failed: 1, Skipped: 8}) {
+ t.Fatalf("stats = %+v", st)
+ }
+}
+
+// A refusal (a fenced or conflicting row) answers InstanceKeyPresent
+// together with its error (activator.BackfillInstanceKey). The error
+// decides: the row is a failure, never "present", or the rollout check
+// would read a host whose key was never written as ready.
+func TestBackfillInstanceKeysCountsARefusalAsFailedNotPresent(t *testing.T) {
+ rt := registry.Route{Host: "shop.example.com", Kind: "instance", Status: "ready", Socket: "/run/carlos/shop.example.com.sock"}
+ f := &fakeKeyBackfiller{
+ got: map[string]activator.InstanceKeyOutcome{rt.Host: activator.InstanceKeyPresent},
+ err: map[string]error{rt.Host: errors.New("not backfilled: layout migration in flight")},
+ }
+ st := backfillInstanceKeys([]registry.Route{rt}, f)
+ if st != (instanceKeyBackfill{Failed: 1}) {
+ t.Fatalf("stats = %+v, want the refusal counted as failed only", st)
+ }
+}
+
+// A host with no own env file has never been started on this box: nothing
+// is created, and the pass counts it on its own line.
+func TestBackfillInstanceKeysCountsAHostNeverStartedHere(t *testing.T) {
+ rt := registry.Route{Host: "shop.example.com", Kind: "instance", Status: "ready", Socket: "/run/carlos/shop.example.com.sock"}
+ f := &fakeKeyBackfiller{got: map[string]activator.InstanceKeyOutcome{rt.Host: activator.InstanceKeyNoEnvFile}}
+ st := backfillInstanceKeys([]registry.Route{rt}, f)
+ if st != (instanceKeyBackfill{NoEnvFile: 1}) {
+ t.Fatalf("stats = %+v", st)
+ }
+}